1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2009-2010 The FreeBSD Foundation
5 * All rights reserved.
6 *
7 * This software was developed by Pawel Jakub Dawidek under sponsorship from
8 * the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34
35 #include <sys/types.h>
36 #include <sys/wait.h>
37
38 #include <errno.h>
39 #include <pthread.h>
40 #include <signal.h>
41 #include <stdio.h>
42 #include <string.h>
43 #include <unistd.h>
44
45 #include "hast.h"
46 #include "hastd.h"
47 #include "hast_checksum.h"
48 #include "hast_compression.h"
49 #include "hast_proto.h"
50 #include "hooks.h"
51 #include "nv.h"
52 #include "pjdlog.h"
53 #include "proto.h"
54 #include "subr.h"
55
56 #include "control.h"
57
58 void
child_cleanup(struct hast_resource * res)59 child_cleanup(struct hast_resource *res)
60 {
61
62 proto_close(res->hr_ctrl);
63 res->hr_ctrl = NULL;
64 if (res->hr_event != NULL) {
65 proto_close(res->hr_event);
66 res->hr_event = NULL;
67 }
68 if (res->hr_conn != NULL) {
69 proto_close(res->hr_conn);
70 res->hr_conn = NULL;
71 }
72 res->hr_workerpid = 0;
73 }
74
75 static void
control_set_role_common(struct hastd_config * cfg,struct nv * nvout,uint8_t role,struct hast_resource * res,const char * name,unsigned int no)76 control_set_role_common(struct hastd_config *cfg, struct nv *nvout,
77 uint8_t role, struct hast_resource *res, const char *name, unsigned int no)
78 {
79 int oldrole;
80
81 /* Name is always needed. */
82 if (name != NULL)
83 nv_add_string(nvout, name, "resource%u", no);
84
85 if (res == NULL) {
86 PJDLOG_ASSERT(cfg != NULL);
87 PJDLOG_ASSERT(name != NULL);
88
89 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
90 if (strcmp(res->hr_name, name) == 0)
91 break;
92 }
93 if (res == NULL) {
94 nv_add_int16(nvout, EHAST_NOENTRY, "error%u", no);
95 return;
96 }
97 }
98 PJDLOG_ASSERT(res != NULL);
99
100 /* Send previous role back. */
101 nv_add_string(nvout, role2str(res->hr_role), "role%u", no);
102
103 /* Nothing changed, return here. */
104 if (role == res->hr_role)
105 return;
106
107 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
108 pjdlog_info("Role changed to %s.", role2str(role));
109
110 /* Change role to the new one. */
111 oldrole = res->hr_role;
112 res->hr_role = role;
113 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
114
115 /*
116 * If previous role was primary or secondary we have to kill process
117 * doing that work.
118 */
119 if (res->hr_workerpid != 0) {
120 if (kill(res->hr_workerpid, SIGTERM) == -1) {
121 pjdlog_errno(LOG_WARNING,
122 "Unable to kill worker process %u",
123 (unsigned int)res->hr_workerpid);
124 } else if (waitpid(res->hr_workerpid, NULL, 0) !=
125 res->hr_workerpid) {
126 pjdlog_errno(LOG_WARNING,
127 "Error while waiting for worker process %u",
128 (unsigned int)res->hr_workerpid);
129 } else {
130 pjdlog_debug(1, "Worker process %u stopped.",
131 (unsigned int)res->hr_workerpid);
132 }
133 child_cleanup(res);
134 }
135
136 /* Start worker process if we are changing to primary. */
137 if (role == HAST_ROLE_PRIMARY)
138 hastd_primary(res);
139 pjdlog_prefix_set("%s", "");
140 hook_exec(res->hr_exec, "role", res->hr_name, role2str(oldrole),
141 role2str(res->hr_role), NULL);
142 }
143
144 void
control_set_role(struct hast_resource * res,uint8_t role)145 control_set_role(struct hast_resource *res, uint8_t role)
146 {
147
148 control_set_role_common(NULL, NULL, role, res, NULL, 0);
149 }
150
151 static void
control_status_worker(struct hast_resource * res,struct nv * nvout,unsigned int no)152 control_status_worker(struct hast_resource *res, struct nv *nvout,
153 unsigned int no)
154 {
155 struct nv *cnvin, *cnvout;
156 const char *str;
157 int error;
158
159 cnvin = NULL;
160
161 /*
162 * Prepare and send command to worker process.
163 */
164 cnvout = nv_alloc();
165 nv_add_uint8(cnvout, CONTROL_STATUS, "cmd");
166 error = nv_error(cnvout);
167 if (error != 0) {
168 pjdlog_common(LOG_ERR, 0, error,
169 "Unable to prepare control header");
170 goto end;
171 }
172 if (hast_proto_send(res, res->hr_ctrl, cnvout, NULL, 0) == -1) {
173 error = errno;
174 pjdlog_errno(LOG_ERR, "Unable to send control header");
175 goto end;
176 }
177
178 /*
179 * Receive response.
180 */
181 if (hast_proto_recv_hdr(res->hr_ctrl, &cnvin) == -1) {
182 error = errno;
183 pjdlog_errno(LOG_ERR, "Unable to receive control header");
184 goto end;
185 }
186
187 error = nv_get_int16(cnvin, "error");
188 if (error != 0)
189 goto end;
190
191 if ((str = nv_get_string(cnvin, "status")) == NULL) {
192 error = ENOENT;
193 pjdlog_errno(LOG_ERR, "Field 'status' is missing.");
194 goto end;
195 }
196 nv_add_string(nvout, str, "status%u", no);
197 nv_add_uint64(nvout, nv_get_uint64(cnvin, "dirty"), "dirty%u", no);
198 nv_add_uint32(nvout, nv_get_uint32(cnvin, "extentsize"),
199 "extentsize%u", no);
200 nv_add_uint32(nvout, nv_get_uint32(cnvin, "keepdirty"),
201 "keepdirty%u", no);
202 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read"),
203 "stat_read%u", no);
204 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write"),
205 "stat_write%u", no);
206 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete"),
207 "stat_delete%u", no);
208 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush"),
209 "stat_flush%u", no);
210 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_activemap_update"),
211 "stat_activemap_update%u", no);
212 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read_error"),
213 "stat_read_error%u", no);
214 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write_error"),
215 "stat_write_error%u", no);
216 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete_error"),
217 "stat_delete_error%u", no);
218 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush_error"),
219 "stat_flush_error%u", no);
220 nv_add_uint64(nvout, nv_get_uint64(cnvin, "idle_queue_size"),
221 "idle_queue_size%u", no);
222 nv_add_uint64(nvout, nv_get_uint64(cnvin, "local_queue_size"),
223 "local_queue_size%u", no);
224 nv_add_uint64(nvout, nv_get_uint64(cnvin, "send_queue_size"),
225 "send_queue_size%u", no);
226 nv_add_uint64(nvout, nv_get_uint64(cnvin, "recv_queue_size"),
227 "recv_queue_size%u", no);
228 nv_add_uint64(nvout, nv_get_uint64(cnvin, "done_queue_size"),
229 "done_queue_size%u", no);
230 end:
231 if (cnvin != NULL)
232 nv_free(cnvin);
233 if (cnvout != NULL)
234 nv_free(cnvout);
235 if (error != 0)
236 nv_add_int16(nvout, error, "error");
237 }
238
239 static void
control_status(struct hastd_config * cfg,struct nv * nvout,struct hast_resource * res,const char * name,unsigned int no)240 control_status(struct hastd_config *cfg, struct nv *nvout,
241 struct hast_resource *res, const char *name, unsigned int no)
242 {
243
244 PJDLOG_ASSERT(cfg != NULL);
245 PJDLOG_ASSERT(nvout != NULL);
246 PJDLOG_ASSERT(name != NULL);
247
248 /* Name is always needed. */
249 nv_add_string(nvout, name, "resource%u", no);
250
251 if (res == NULL) {
252 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
253 if (strcmp(res->hr_name, name) == 0)
254 break;
255 }
256 if (res == NULL) {
257 nv_add_int16(nvout, EHAST_NOENTRY, "error%u", no);
258 return;
259 }
260 }
261 PJDLOG_ASSERT(res != NULL);
262 nv_add_string(nvout, res->hr_provname, "provname%u", no);
263 nv_add_string(nvout, res->hr_localpath, "localpath%u", no);
264 nv_add_string(nvout, res->hr_remoteaddr, "remoteaddr%u", no);
265 if (res->hr_sourceaddr[0] != '\0')
266 nv_add_string(nvout, res->hr_sourceaddr, "sourceaddr%u", no);
267 switch (res->hr_replication) {
268 case HAST_REPLICATION_FULLSYNC:
269 nv_add_string(nvout, "fullsync", "replication%u", no);
270 break;
271 case HAST_REPLICATION_MEMSYNC:
272 nv_add_string(nvout, "memsync", "replication%u", no);
273 break;
274 case HAST_REPLICATION_ASYNC:
275 nv_add_string(nvout, "async", "replication%u", no);
276 break;
277 default:
278 nv_add_string(nvout, "unknown", "replication%u", no);
279 break;
280 }
281 nv_add_string(nvout, checksum_name(res->hr_checksum),
282 "checksum%u", no);
283 nv_add_string(nvout, compression_name(res->hr_compression),
284 "compression%u", no);
285 nv_add_string(nvout, role2str(res->hr_role), "role%u", no);
286 nv_add_int32(nvout, res->hr_workerpid, "workerpid%u", no);
287
288 switch (res->hr_role) {
289 case HAST_ROLE_PRIMARY:
290 PJDLOG_ASSERT(res->hr_workerpid != 0);
291 /* FALLTHROUGH */
292 case HAST_ROLE_SECONDARY:
293 if (res->hr_workerpid != 0)
294 break;
295 /* FALLTHROUGH */
296 default:
297 return;
298 }
299
300 /*
301 * If we are here, it means that we have a worker process, which we
302 * want to ask some questions.
303 */
304 control_status_worker(res, nvout, no);
305 }
306
307 void
control_handle(struct hastd_config * cfg)308 control_handle(struct hastd_config *cfg)
309 {
310 struct proto_conn *conn;
311 struct nv *nvin, *nvout;
312 unsigned int ii;
313 const char *str;
314 uint8_t cmd, role;
315 int error;
316
317 if (proto_accept(cfg->hc_controlconn, &conn) == -1) {
318 pjdlog_errno(LOG_ERR, "Unable to accept control connection");
319 return;
320 }
321
322 cfg->hc_controlin = conn;
323 nvin = nvout = NULL;
324 role = HAST_ROLE_UNDEF;
325
326 if (hast_proto_recv_hdr(conn, &nvin) == -1) {
327 pjdlog_errno(LOG_ERR, "Unable to receive control header");
328 nvin = NULL;
329 goto close;
330 }
331
332 /* Obtain command code. 0 means that nv_get_uint8() failed. */
333 cmd = nv_get_uint8(nvin, "cmd");
334 if (cmd == 0) {
335 pjdlog_error("Control header is missing 'cmd' field.");
336 goto close;
337 }
338
339 /* Allocate outgoing nv structure. */
340 nvout = nv_alloc();
341 if (nvout == NULL) {
342 pjdlog_error("Unable to allocate header for control response.");
343 goto close;
344 }
345
346 error = 0;
347
348 str = nv_get_string(nvin, "resource0");
349 if (str == NULL) {
350 pjdlog_error("Control header is missing 'resource0' field.");
351 error = EHAST_INVALID;
352 goto fail;
353 }
354 if (cmd == HASTCTL_CMD_SETROLE) {
355 role = nv_get_uint8(nvin, "role");
356 switch (role) {
357 case HAST_ROLE_INIT:
358 case HAST_ROLE_PRIMARY:
359 case HAST_ROLE_SECONDARY:
360 break;
361 default:
362 pjdlog_error("Invalid role received (%hhu).", role);
363 error = EHAST_INVALID;
364 goto fail;
365 }
366 }
367 if (strcmp(str, "all") == 0) {
368 struct hast_resource *res;
369
370 /* All configured resources. */
371
372 ii = 0;
373 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
374 switch (cmd) {
375 case HASTCTL_CMD_SETROLE:
376 control_set_role_common(cfg, nvout, role, res,
377 res->hr_name, ii++);
378 break;
379 case HASTCTL_CMD_STATUS:
380 control_status(cfg, nvout, res, res->hr_name,
381 ii++);
382 break;
383 default:
384 pjdlog_error("Invalid command received (%hhu).",
385 cmd);
386 error = EHAST_UNIMPLEMENTED;
387 goto fail;
388 }
389 }
390 } else {
391 /* Only selected resources. */
392
393 for (ii = 0; ; ii++) {
394 str = nv_get_string(nvin, "resource%u", ii);
395 if (str == NULL)
396 break;
397 switch (cmd) {
398 case HASTCTL_CMD_SETROLE:
399 control_set_role_common(cfg, nvout, role, NULL,
400 str, ii);
401 break;
402 case HASTCTL_CMD_STATUS:
403 control_status(cfg, nvout, NULL, str, ii);
404 break;
405 default:
406 pjdlog_error("Invalid command received (%hhu).",
407 cmd);
408 error = EHAST_UNIMPLEMENTED;
409 goto fail;
410 }
411 }
412 }
413 if (nv_error(nvout) != 0)
414 goto close;
415 fail:
416 if (error != 0)
417 nv_add_int16(nvout, error, "error");
418
419 if (hast_proto_send(NULL, conn, nvout, NULL, 0) == -1)
420 pjdlog_errno(LOG_ERR, "Unable to send control response");
421 close:
422 if (nvin != NULL)
423 nv_free(nvin);
424 if (nvout != NULL)
425 nv_free(nvout);
426 proto_close(conn);
427 cfg->hc_controlin = NULL;
428 }
429
430 /*
431 * Thread handles control requests from the parent.
432 */
433 void *
ctrl_thread(void * arg)434 ctrl_thread(void *arg)
435 {
436 struct hast_resource *res = arg;
437 struct nv *nvin, *nvout;
438 uint8_t cmd;
439
440 for (;;) {
441 if (hast_proto_recv_hdr(res->hr_ctrl, &nvin) == -1) {
442 if (sigexit_received)
443 pthread_exit(NULL);
444 pjdlog_errno(LOG_ERR,
445 "Unable to receive control message");
446 kill(getpid(), SIGTERM);
447 pthread_exit(NULL);
448 }
449 cmd = nv_get_uint8(nvin, "cmd");
450 if (cmd == 0) {
451 pjdlog_error("Control message is missing 'cmd' field.");
452 nv_free(nvin);
453 continue;
454 }
455 nvout = nv_alloc();
456 switch (cmd) {
457 case CONTROL_STATUS:
458 if (res->hr_remotein != NULL &&
459 res->hr_remoteout != NULL) {
460 nv_add_string(nvout, "complete", "status");
461 } else {
462 nv_add_string(nvout, "degraded", "status");
463 }
464 nv_add_uint32(nvout, (uint32_t)res->hr_extentsize,
465 "extentsize");
466 if (res->hr_role == HAST_ROLE_PRIMARY) {
467 nv_add_uint32(nvout,
468 (uint32_t)res->hr_keepdirty, "keepdirty");
469 nv_add_uint64(nvout,
470 (uint64_t)(activemap_ndirty(res->hr_amp) *
471 res->hr_extentsize), "dirty");
472 } else {
473 nv_add_uint32(nvout, (uint32_t)0, "keepdirty");
474 nv_add_uint64(nvout, (uint64_t)0, "dirty");
475 }
476 nv_add_uint64(nvout, res->hr_stat_read, "stat_read");
477 nv_add_uint64(nvout, res->hr_stat_write, "stat_write");
478 nv_add_uint64(nvout, res->hr_stat_delete,
479 "stat_delete");
480 nv_add_uint64(nvout, res->hr_stat_flush, "stat_flush");
481 nv_add_uint64(nvout, res->hr_stat_activemap_update,
482 "stat_activemap_update");
483 nv_add_uint64(nvout, res->hr_stat_read_error,
484 "stat_read_error");
485 nv_add_uint64(nvout, res->hr_stat_write_error +
486 res->hr_stat_activemap_write_error,
487 "stat_write_error");
488 nv_add_uint64(nvout, res->hr_stat_delete_error,
489 "stat_delete_error");
490 nv_add_uint64(nvout, res->hr_stat_flush_error +
491 res->hr_stat_activemap_flush_error,
492 "stat_flush_error");
493 res->output_status_aux(nvout);
494 nv_add_int16(nvout, 0, "error");
495 break;
496 case CONTROL_RELOAD:
497 /*
498 * When parent receives SIGHUP and discovers that
499 * something related to us has changes, it sends reload
500 * message to us.
501 */
502 PJDLOG_ASSERT(res->hr_role == HAST_ROLE_PRIMARY);
503 primary_config_reload(res, nvin);
504 nv_add_int16(nvout, 0, "error");
505 break;
506 default:
507 nv_add_int16(nvout, EINVAL, "error");
508 break;
509 }
510 nv_free(nvin);
511 if (nv_error(nvout) != 0) {
512 pjdlog_error("Unable to create answer on control message.");
513 nv_free(nvout);
514 continue;
515 }
516 if (hast_proto_send(NULL, res->hr_ctrl, nvout, NULL, 0) == -1) {
517 pjdlog_errno(LOG_ERR,
518 "Unable to send reply to control message");
519 }
520 nv_free(nvout);
521 }
522 /* NOTREACHED */
523 return (NULL);
524 }
525