1 /*-
2 * Copyright (c) 2014-2018 Netflix, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 /*
28 * Author: Lawrence Stewart <lstewart@netflix.com>
29 */
30
31 #include <sys/cdefs.h>
32 #include <sys/param.h>
33 #include <sys/arb.h>
34 #include <sys/ctype.h>
35 #include <sys/errno.h>
36 #include <sys/hash.h>
37 #include <sys/limits.h>
38 #include <sys/malloc.h>
39 #include <sys/qmath.h>
40 #include <sys/sbuf.h>
41 #if defined(DIAGNOSTIC)
42 #include <sys/tree.h>
43 #endif
44 #include <sys/stats.h> /* Must come after qmath.h and arb.h */
45 #include <sys/stddef.h>
46 #include <sys/stdint.h>
47 #include <sys/time.h>
48
49 #ifdef _KERNEL
50 #include <sys/kernel.h>
51 #include <sys/lock.h>
52 #include <sys/rwlock.h>
53 #include <sys/sysctl.h>
54 #include <sys/systm.h>
55 #else /* ! _KERNEL */
56 #include <pthread.h>
57 #include <stdbool.h>
58 #include <stdio.h>
59 #include <stdlib.h>
60 #include <string.h>
61 #endif /* _KERNEL */
62
63 struct voistatdata_voistate {
64 /* Previous VOI value for diff calculation. */
65 struct voistatdata_numeric prev;
66 };
67
68 #define VS_VSDVALID 0x0001 /* Stat's voistatdata updated at least once. */
69 struct voistat {
70 int8_t stype; /* Type of stat e.g. VS_STYPE_SUM. */
71 enum vsd_dtype dtype : 8; /* Data type of this stat's data. */
72 uint16_t data_off; /* Blob offset for this stat's data. */
73 uint16_t dsz; /* Size of stat's data. */
74 #define VS_EBITS 8
75 uint16_t errs : VS_EBITS;/* Non-wrapping error count. */
76 uint16_t flags : 16 - VS_EBITS;
77 };
78 /* The voistat error count is capped to avoid wrapping. */
79 #define VS_INCERRS(vs) do { \
80 if ((vs)->errs < (1U << VS_EBITS) - 1) \
81 (vs)->errs++; \
82 } while (0)
83
84 /*
85 * Ideas for flags:
86 * - Global or entity specific (global would imply use of counter(9)?)
87 * - Whether to reset stats on read or not
88 * - Signal an overflow?
89 * - Compressed voistat array
90 */
91 #define VOI_REQSTATE 0x0001 /* VOI requires VS_STYPE_VOISTATE. */
92 struct voi {
93 int16_t id; /* VOI id. */
94 enum vsd_dtype dtype : 8; /* Data type of the VOI itself. */
95 int8_t voistatmaxid; /* Largest allocated voistat index. */
96 uint16_t stats_off; /* Blob offset for this VOIs stats. */
97 uint16_t flags;
98 };
99
100 /*
101 * Memory for the entire blob is allocated as a slab and then offsets are
102 * maintained to carve up the slab into sections holding different data types.
103 *
104 * Ideas for flags:
105 * - Compressed voi array (trade off memory usage vs search time)
106 * - Units of offsets (default bytes, flag for e.g. vm_page/KiB/Mib)
107 */
108 struct statsblobv1 {
109 uint8_t abi;
110 uint8_t endian;
111 uint16_t flags;
112 uint16_t maxsz;
113 uint16_t cursz;
114 /* Fields from here down are opaque to consumers. */
115 uint32_t tplhash; /* Base template hash ID. */
116 uint16_t stats_off; /* voistat array blob offset. */
117 uint16_t statsdata_off; /* voistatdata array blob offset. */
118 sbintime_t created; /* Blob creation time. */
119 sbintime_t lastrst; /* Time of last reset. */
120 struct voi vois[]; /* Array indexed by [voi_id]. */
121 } __aligned(sizeof(void *));
122 _Static_assert(offsetof(struct statsblobv1, cursz) +
123 SIZEOF_MEMBER(struct statsblobv1, cursz) ==
124 offsetof(struct statsblob, opaque),
125 "statsblobv1 ABI mismatch");
126
127 struct statsblobv1_tpl {
128 struct metablob *mb;
129 struct statsblobv1 *sb;
130 };
131
132 /* Context passed to iterator callbacks. */
133 struct sb_iter_ctx {
134 void *usrctx; /* Caller supplied context. */
135 uint32_t flags; /* Flags for current iteration. */
136 int16_t vslot; /* struct voi slot index. */
137 int8_t vsslot; /* struct voistat slot index. */
138 };
139
140 struct sb_tostrcb_ctx {
141 struct sbuf *buf;
142 struct statsblob_tpl *tpl;
143 enum sb_str_fmt fmt;
144 uint32_t flags;
145 };
146
147 struct sb_visitcb_ctx {
148 stats_blob_visitcb_t cb;
149 void *usrctx;
150 };
151
152 /* Stats blob iterator callback. */
153 typedef int (*stats_v1_blob_itercb_t)(struct statsblobv1 *sb, struct voi *v,
154 struct voistat *vs, struct sb_iter_ctx *ctx);
155
156 #ifdef _KERNEL
157 static struct rwlock tpllistlock;
158 RW_SYSINIT(stats_tpl_list, &tpllistlock, "Stat template list lock");
159 #define TPL_LIST_RLOCK() rw_rlock(&tpllistlock)
160 #define TPL_LIST_RUNLOCK() rw_runlock(&tpllistlock)
161 #define TPL_LIST_WLOCK() rw_wlock(&tpllistlock)
162 #define TPL_LIST_WUNLOCK() rw_wunlock(&tpllistlock)
163 #define TPL_LIST_LOCK_ASSERT() rw_assert(&tpllistlock, RA_LOCKED)
164 #define TPL_LIST_RLOCK_ASSERT() rw_assert(&tpllistlock, RA_RLOCKED)
165 #define TPL_LIST_WLOCK_ASSERT() rw_assert(&tpllistlock, RA_WLOCKED)
166 MALLOC_DEFINE(M_STATS, "stats(9) related memory", "stats(9) related memory");
167 #define stats_free(ptr) free((ptr), M_STATS)
168 #else /* ! _KERNEL */
169 static void stats_constructor(void);
170 static void stats_destructor(void);
171 static pthread_rwlock_t tpllistlock;
172 #define TPL_LIST_UNLOCK() pthread_rwlock_unlock(&tpllistlock)
173 #define TPL_LIST_RLOCK() pthread_rwlock_rdlock(&tpllistlock)
174 #define TPL_LIST_RUNLOCK() TPL_LIST_UNLOCK()
175 #define TPL_LIST_WLOCK() pthread_rwlock_wrlock(&tpllistlock)
176 #define TPL_LIST_WUNLOCK() TPL_LIST_UNLOCK()
177 #define TPL_LIST_LOCK_ASSERT() do { } while (0)
178 #define TPL_LIST_RLOCK_ASSERT() do { } while (0)
179 #define TPL_LIST_WLOCK_ASSERT() do { } while (0)
180 #ifdef NDEBUG
181 #define KASSERT(cond, msg) do {} while (0)
182 #define stats_abort() do {} while (0)
183 #else /* ! NDEBUG */
184 #define KASSERT(cond, msg) do { \
185 if (!(cond)) { \
186 panic msg; \
187 } \
188 } while (0)
189 #define stats_abort() abort()
190 #endif /* NDEBUG */
191 #define stats_free(ptr) free(ptr)
192 #define panic(fmt, ...) do { \
193 fprintf(stderr, (fmt), ##__VA_ARGS__); \
194 stats_abort(); \
195 } while (0)
196 #endif /* _KERNEL */
197
198 #define SB_V1_MAXSZ 65535
199
200 /* Obtain a blob offset pointer. */
201 #define BLOB_OFFSET(sb, off) ((void *)(((uint8_t *)(sb)) + (off)))
202
203 /*
204 * Number of VOIs in the blob's vois[] array. By virtue of struct voi being a
205 * power of 2 size, we can shift instead of divide. The shift amount must be
206 * updated if sizeof(struct voi) ever changes, which the assert should catch.
207 */
208 #define NVOIS(sb) ((int32_t)((((struct statsblobv1 *)(sb))->stats_off - \
209 sizeof(struct statsblobv1)) >> 3))
210 _Static_assert(sizeof(struct voi) == 8, "statsblobv1 voi ABI mismatch");
211
212 /* Try restrict names to alphanumeric and underscore to simplify JSON compat. */
213 const char *vs_stype2name[VS_NUM_STYPES] = {
214 [VS_STYPE_VOISTATE] = "VOISTATE",
215 [VS_STYPE_SUM] = "SUM",
216 [VS_STYPE_MAX] = "MAX",
217 [VS_STYPE_MIN] = "MIN",
218 [VS_STYPE_HIST] = "HIST",
219 [VS_STYPE_TDGST] = "TDGST",
220 };
221
222 const char *vs_stype2desc[VS_NUM_STYPES] = {
223 [VS_STYPE_VOISTATE] = "VOI related state data (not a real stat)",
224 [VS_STYPE_SUM] = "Simple arithmetic accumulator",
225 [VS_STYPE_MAX] = "Maximum observed VOI value",
226 [VS_STYPE_MIN] = "Minimum observed VOI value",
227 [VS_STYPE_HIST] = "Histogram of observed VOI values",
228 [VS_STYPE_TDGST] = "t-digest of observed VOI values",
229 };
230
231 const char *vsd_dtype2name[VSD_NUM_DTYPES] = {
232 [VSD_DTYPE_VOISTATE] = "VOISTATE",
233 [VSD_DTYPE_INT_S32] = "INT_S32",
234 [VSD_DTYPE_INT_U32] = "INT_U32",
235 [VSD_DTYPE_INT_S64] = "INT_S64",
236 [VSD_DTYPE_INT_U64] = "INT_U64",
237 [VSD_DTYPE_INT_SLONG] = "INT_SLONG",
238 [VSD_DTYPE_INT_ULONG] = "INT_ULONG",
239 [VSD_DTYPE_Q_S32] = "Q_S32",
240 [VSD_DTYPE_Q_U32] = "Q_U32",
241 [VSD_DTYPE_Q_S64] = "Q_S64",
242 [VSD_DTYPE_Q_U64] = "Q_U64",
243 [VSD_DTYPE_CRHIST32] = "CRHIST32",
244 [VSD_DTYPE_DRHIST32] = "DRHIST32",
245 [VSD_DTYPE_DVHIST32] = "DVHIST32",
246 [VSD_DTYPE_CRHIST64] = "CRHIST64",
247 [VSD_DTYPE_DRHIST64] = "DRHIST64",
248 [VSD_DTYPE_DVHIST64] = "DVHIST64",
249 [VSD_DTYPE_TDGSTCLUST32] = "TDGSTCLUST32",
250 [VSD_DTYPE_TDGSTCLUST64] = "TDGSTCLUST64",
251 };
252
253 const size_t vsd_dtype2size[VSD_NUM_DTYPES] = {
254 [VSD_DTYPE_VOISTATE] = sizeof(struct voistatdata_voistate),
255 [VSD_DTYPE_INT_S32] = sizeof(struct voistatdata_int32),
256 [VSD_DTYPE_INT_U32] = sizeof(struct voistatdata_int32),
257 [VSD_DTYPE_INT_S64] = sizeof(struct voistatdata_int64),
258 [VSD_DTYPE_INT_U64] = sizeof(struct voistatdata_int64),
259 [VSD_DTYPE_INT_SLONG] = sizeof(struct voistatdata_intlong),
260 [VSD_DTYPE_INT_ULONG] = sizeof(struct voistatdata_intlong),
261 [VSD_DTYPE_Q_S32] = sizeof(struct voistatdata_q32),
262 [VSD_DTYPE_Q_U32] = sizeof(struct voistatdata_q32),
263 [VSD_DTYPE_Q_S64] = sizeof(struct voistatdata_q64),
264 [VSD_DTYPE_Q_U64] = sizeof(struct voistatdata_q64),
265 [VSD_DTYPE_CRHIST32] = sizeof(struct voistatdata_crhist32),
266 [VSD_DTYPE_DRHIST32] = sizeof(struct voistatdata_drhist32),
267 [VSD_DTYPE_DVHIST32] = sizeof(struct voistatdata_dvhist32),
268 [VSD_DTYPE_CRHIST64] = sizeof(struct voistatdata_crhist64),
269 [VSD_DTYPE_DRHIST64] = sizeof(struct voistatdata_drhist64),
270 [VSD_DTYPE_DVHIST64] = sizeof(struct voistatdata_dvhist64),
271 [VSD_DTYPE_TDGSTCLUST32] = sizeof(struct voistatdata_tdgstclust32),
272 [VSD_DTYPE_TDGSTCLUST64] = sizeof(struct voistatdata_tdgstclust64),
273 };
274
275 static const bool vsd_compoundtype[VSD_NUM_DTYPES] = {
276 [VSD_DTYPE_VOISTATE] = true,
277 [VSD_DTYPE_INT_S32] = false,
278 [VSD_DTYPE_INT_U32] = false,
279 [VSD_DTYPE_INT_S64] = false,
280 [VSD_DTYPE_INT_U64] = false,
281 [VSD_DTYPE_INT_SLONG] = false,
282 [VSD_DTYPE_INT_ULONG] = false,
283 [VSD_DTYPE_Q_S32] = false,
284 [VSD_DTYPE_Q_U32] = false,
285 [VSD_DTYPE_Q_S64] = false,
286 [VSD_DTYPE_Q_U64] = false,
287 [VSD_DTYPE_CRHIST32] = true,
288 [VSD_DTYPE_DRHIST32] = true,
289 [VSD_DTYPE_DVHIST32] = true,
290 [VSD_DTYPE_CRHIST64] = true,
291 [VSD_DTYPE_DRHIST64] = true,
292 [VSD_DTYPE_DVHIST64] = true,
293 [VSD_DTYPE_TDGSTCLUST32] = true,
294 [VSD_DTYPE_TDGSTCLUST64] = true,
295 };
296
297 const struct voistatdata_numeric numeric_limits[2][VSD_DTYPE_Q_U64 + 1] = {
298 [LIM_MIN] = {
299 [VSD_DTYPE_VOISTATE] = {},
300 [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MIN}},
301 [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = 0}},
302 [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MIN}},
303 [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = 0}},
304 [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MIN}},
305 [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = 0}},
306 [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMINVAL(INT32_MIN)}},
307 [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = 0}},
308 [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMINVAL(INT64_MIN)}},
309 [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = 0}},
310 },
311 [LIM_MAX] = {
312 [VSD_DTYPE_VOISTATE] = {},
313 [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MAX}},
314 [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = UINT32_MAX}},
315 [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MAX}},
316 [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = UINT64_MAX}},
317 [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MAX}},
318 [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = ULONG_MAX}},
319 [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMAXVAL(INT32_MAX)}},
320 [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = Q_IFMAXVAL(UINT32_MAX)}},
321 [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMAXVAL(INT64_MAX)}},
322 [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = Q_IFMAXVAL(UINT64_MAX)}},
323 }
324 };
325
326 /* tpllistlock protects tpllist and ntpl */
327 static uint32_t ntpl;
328 static struct statsblob_tpl **tpllist;
329
330 static inline void * stats_realloc(void *ptr, size_t oldsz, size_t newsz,
331 int flags);
332 //static void stats_v1_blob_finalise(struct statsblobv1 *sb);
333 static int stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id,
334 uint32_t flags);
335 static int stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes,
336 int newvoistatbytes, int newvoistatdatabytes);
337 static void stats_v1_blob_iter(struct statsblobv1 *sb,
338 stats_v1_blob_itercb_t icb, void *usrctx, uint32_t flags);
339 static inline int stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype,
340 struct voistatdata_tdgst *tdgst, s64q_t x, uint64_t weight, int attempt);
341
342 static inline int
ctd32cmp(const struct voistatdata_tdgstctd32 * c1,const struct voistatdata_tdgstctd32 * c2)343 ctd32cmp(const struct voistatdata_tdgstctd32 *c1, const struct voistatdata_tdgstctd32 *c2)
344 {
345
346 KASSERT(Q_PRECEQ(c1->mu, c2->mu),
347 ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__,
348 Q_RELPREC(c1->mu, c2->mu)));
349
350 return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1);
351 }
352 ARB_GENERATE_STATIC(ctdth32, voistatdata_tdgstctd32, ctdlnk, ctd32cmp);
353
354 static inline int
ctd64cmp(const struct voistatdata_tdgstctd64 * c1,const struct voistatdata_tdgstctd64 * c2)355 ctd64cmp(const struct voistatdata_tdgstctd64 *c1, const struct voistatdata_tdgstctd64 *c2)
356 {
357
358 KASSERT(Q_PRECEQ(c1->mu, c2->mu),
359 ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__,
360 Q_RELPREC(c1->mu, c2->mu)));
361
362 return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1);
363 }
364 ARB_GENERATE_STATIC(ctdth64, voistatdata_tdgstctd64, ctdlnk, ctd64cmp);
365
366 #ifdef DIAGNOSTIC
367 RB_GENERATE_STATIC(rbctdth32, voistatdata_tdgstctd32, rblnk, ctd32cmp);
368 RB_GENERATE_STATIC(rbctdth64, voistatdata_tdgstctd64, rblnk, ctd64cmp);
369 #endif
370
371 static inline sbintime_t
stats_sbinuptime(void)372 stats_sbinuptime(void)
373 {
374 sbintime_t sbt;
375 #ifdef _KERNEL
376
377 sbt = sbinuptime();
378 #else /* ! _KERNEL */
379 struct timespec tp;
380
381 clock_gettime(CLOCK_MONOTONIC_FAST, &tp);
382 sbt = tstosbt(tp);
383 #endif /* _KERNEL */
384
385 return (sbt);
386 }
387
388 static inline void *
stats_realloc(void * ptr,size_t oldsz,size_t newsz,int flags)389 stats_realloc(void *ptr, size_t oldsz, size_t newsz, int flags)
390 {
391
392 #ifdef _KERNEL
393 /* Default to M_NOWAIT if neither M_NOWAIT or M_WAITOK are set. */
394 if (!(flags & (M_WAITOK | M_NOWAIT)))
395 flags |= M_NOWAIT;
396 ptr = realloc(ptr, newsz, M_STATS, flags);
397 #else /* ! _KERNEL */
398 ptr = realloc(ptr, newsz);
399 if ((flags & M_ZERO) && ptr != NULL) {
400 if (oldsz == 0)
401 memset(ptr, '\0', newsz);
402 else if (newsz > oldsz)
403 memset(BLOB_OFFSET(ptr, oldsz), '\0', newsz - oldsz);
404 }
405 #endif /* _KERNEL */
406
407 return (ptr);
408 }
409
410 static inline char *
stats_strdup(const char * s,int flags)411 stats_strdup(const char *s,
412 #ifdef _KERNEL
413 int flags)
414 {
415 char *copy;
416 size_t len;
417
418 if (!(flags & (M_WAITOK | M_NOWAIT)))
419 flags |= M_NOWAIT;
420
421 len = strlen(s) + 1;
422 if ((copy = malloc(len, M_STATS, flags)) != NULL)
423 bcopy(s, copy, len);
424
425 return (copy);
426 #else
427 int flags __unused)
428 {
429 return (strdup(s));
430 #endif
431 }
432
433 static inline void
434 stats_tpl_update_hash(struct statsblob_tpl *tpl)
435 {
436
437 TPL_LIST_WLOCK_ASSERT();
438 tpl->mb->tplhash = hash32_str(tpl->mb->tplname, 0);
439 for (int voi_id = 0; voi_id < NVOIS(tpl->sb); voi_id++) {
440 if (tpl->mb->voi_meta[voi_id].name != NULL)
441 tpl->mb->tplhash = hash32_str(
442 tpl->mb->voi_meta[voi_id].name, tpl->mb->tplhash);
443 }
444 tpl->mb->tplhash = hash32_buf(tpl->sb, tpl->sb->cursz,
445 tpl->mb->tplhash);
446 }
447
448 static inline uint64_t
449 stats_pow_u64(uint64_t base, uint64_t exp)
450 {
451 uint64_t result = 1;
452
453 while (exp) {
454 if (exp & 1)
455 result *= base;
456 exp >>= 1;
457 base *= base;
458 }
459
460 return (result);
461 }
462
463 static inline int
464 stats_vss_hist_bkt_hlpr(struct vss_hist_hlpr_info *info, uint32_t curbkt,
465 struct voistatdata_numeric *bkt_lb, struct voistatdata_numeric *bkt_ub)
466 {
467 uint64_t step = 0;
468 int error = 0;
469
470 switch (info->scheme) {
471 case BKT_LIN:
472 step = info->lin.stepinc;
473 break;
474 case BKT_EXP:
475 step = stats_pow_u64(info->exp.stepbase,
476 info->exp.stepexp + curbkt);
477 break;
478 case BKT_LINEXP:
479 {
480 uint64_t curstepexp = 1;
481
482 switch (info->voi_dtype) {
483 case VSD_DTYPE_INT_S32:
484 while ((int32_t)stats_pow_u64(info->linexp.stepbase,
485 curstepexp) <= bkt_lb->int32.s32)
486 curstepexp++;
487 break;
488 case VSD_DTYPE_INT_U32:
489 while ((uint32_t)stats_pow_u64(info->linexp.stepbase,
490 curstepexp) <= bkt_lb->int32.u32)
491 curstepexp++;
492 break;
493 case VSD_DTYPE_INT_S64:
494 while ((int64_t)stats_pow_u64(info->linexp.stepbase,
495 curstepexp) <= bkt_lb->int64.s64)
496 curstepexp++;
497 break;
498 case VSD_DTYPE_INT_U64:
499 while ((uint64_t)stats_pow_u64(info->linexp.stepbase,
500 curstepexp) <= bkt_lb->int64.u64)
501 curstepexp++;
502 break;
503 case VSD_DTYPE_INT_SLONG:
504 while ((long)stats_pow_u64(info->linexp.stepbase,
505 curstepexp) <= bkt_lb->intlong.slong)
506 curstepexp++;
507 break;
508 case VSD_DTYPE_INT_ULONG:
509 while ((unsigned long)stats_pow_u64(info->linexp.stepbase,
510 curstepexp) <= bkt_lb->intlong.ulong)
511 curstepexp++;
512 break;
513 case VSD_DTYPE_Q_S32:
514 while ((s32q_t)stats_pow_u64(info->linexp.stepbase,
515 curstepexp) <= Q_GIVAL(bkt_lb->q32.sq32))
516 break;
517 case VSD_DTYPE_Q_U32:
518 while ((u32q_t)stats_pow_u64(info->linexp.stepbase,
519 curstepexp) <= Q_GIVAL(bkt_lb->q32.uq32))
520 break;
521 case VSD_DTYPE_Q_S64:
522 while ((s64q_t)stats_pow_u64(info->linexp.stepbase,
523 curstepexp) <= Q_GIVAL(bkt_lb->q64.sq64))
524 curstepexp++;
525 break;
526 case VSD_DTYPE_Q_U64:
527 while ((u64q_t)stats_pow_u64(info->linexp.stepbase,
528 curstepexp) <= Q_GIVAL(bkt_lb->q64.uq64))
529 curstepexp++;
530 break;
531 default:
532 break;
533 }
534
535 step = stats_pow_u64(info->linexp.stepbase, curstepexp) /
536 info->linexp.linstepdiv;
537 if (step == 0)
538 step = 1;
539 break;
540 }
541 default:
542 break;
543 }
544
545 if (info->scheme == BKT_USR) {
546 *bkt_lb = info->usr.bkts[curbkt].lb;
547 *bkt_ub = info->usr.bkts[curbkt].ub;
548 } else if (step != 0) {
549 switch (info->voi_dtype) {
550 case VSD_DTYPE_INT_S32:
551 bkt_ub->int32.s32 += (int32_t)step;
552 break;
553 case VSD_DTYPE_INT_U32:
554 bkt_ub->int32.u32 += (uint32_t)step;
555 break;
556 case VSD_DTYPE_INT_S64:
557 bkt_ub->int64.s64 += (int64_t)step;
558 break;
559 case VSD_DTYPE_INT_U64:
560 bkt_ub->int64.u64 += (uint64_t)step;
561 break;
562 case VSD_DTYPE_INT_SLONG:
563 bkt_ub->intlong.slong += (long)step;
564 break;
565 case VSD_DTYPE_INT_ULONG:
566 bkt_ub->intlong.ulong += (unsigned long)step;
567 break;
568 case VSD_DTYPE_Q_S32:
569 error = Q_QADDI(&bkt_ub->q32.sq32, step);
570 break;
571 case VSD_DTYPE_Q_U32:
572 error = Q_QADDI(&bkt_ub->q32.uq32, step);
573 break;
574 case VSD_DTYPE_Q_S64:
575 error = Q_QADDI(&bkt_ub->q64.sq64, step);
576 break;
577 case VSD_DTYPE_Q_U64:
578 error = Q_QADDI(&bkt_ub->q64.uq64, step);
579 break;
580 default:
581 break;
582 }
583 } else { /* info->scheme != BKT_USR && step == 0 */
584 return (EINVAL);
585 }
586
587 return (error);
588 }
589
590 static uint32_t
591 stats_vss_hist_nbkts_hlpr(struct vss_hist_hlpr_info *info)
592 {
593 struct voistatdata_numeric bkt_lb, bkt_ub;
594 uint32_t nbkts;
595 int done;
596
597 if (info->scheme == BKT_USR) {
598 /* XXXLAS: Setting info->{lb,ub} from macro is tricky. */
599 info->lb = info->usr.bkts[0].lb;
600 info->ub = info->usr.bkts[info->usr.nbkts - 1].lb;
601 }
602
603 nbkts = 0;
604 done = 0;
605 bkt_ub = info->lb;
606
607 do {
608 bkt_lb = bkt_ub;
609 if (stats_vss_hist_bkt_hlpr(info, nbkts++, &bkt_lb, &bkt_ub))
610 return (0);
611
612 if (info->scheme == BKT_USR)
613 done = (nbkts == info->usr.nbkts);
614 else {
615 switch (info->voi_dtype) {
616 case VSD_DTYPE_INT_S32:
617 done = (bkt_ub.int32.s32 > info->ub.int32.s32);
618 break;
619 case VSD_DTYPE_INT_U32:
620 done = (bkt_ub.int32.u32 > info->ub.int32.u32);
621 break;
622 case VSD_DTYPE_INT_S64:
623 done = (bkt_ub.int64.s64 > info->ub.int64.s64);
624 break;
625 case VSD_DTYPE_INT_U64:
626 done = (bkt_ub.int64.u64 > info->ub.int64.u64);
627 break;
628 case VSD_DTYPE_INT_SLONG:
629 done = (bkt_ub.intlong.slong >
630 info->ub.intlong.slong);
631 break;
632 case VSD_DTYPE_INT_ULONG:
633 done = (bkt_ub.intlong.ulong >
634 info->ub.intlong.ulong);
635 break;
636 case VSD_DTYPE_Q_S32:
637 done = Q_QGTQ(bkt_ub.q32.sq32,
638 info->ub.q32.sq32);
639 break;
640 case VSD_DTYPE_Q_U32:
641 done = Q_QGTQ(bkt_ub.q32.uq32,
642 info->ub.q32.uq32);
643 break;
644 case VSD_DTYPE_Q_S64:
645 done = Q_QGTQ(bkt_ub.q64.sq64,
646 info->ub.q64.sq64);
647 break;
648 case VSD_DTYPE_Q_U64:
649 done = Q_QGTQ(bkt_ub.q64.uq64,
650 info->ub.q64.uq64);
651 break;
652 default:
653 return (0);
654 }
655 }
656 } while (!done);
657
658 if (info->flags & VSD_HIST_LBOUND_INF)
659 nbkts++;
660 if (info->flags & VSD_HIST_UBOUND_INF)
661 nbkts++;
662
663 return (nbkts);
664 }
665
666 int
667 stats_vss_hist_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
668 struct vss_hist_hlpr_info *info)
669 {
670 struct voistatdata_hist *hist;
671 struct voistatdata_numeric bkt_lb, bkt_ub, *lbinfbktlb, *lbinfbktub,
672 *ubinfbktlb, *ubinfbktub;
673 uint32_t bkt, nbkts, nloop;
674
675 if (vss == NULL || info == NULL || (info->flags &
676 (VSD_HIST_LBOUND_INF|VSD_HIST_UBOUND_INF) && (info->hist_dtype ==
677 VSD_DTYPE_DVHIST32 || info->hist_dtype == VSD_DTYPE_DVHIST64)))
678 return (EINVAL);
679
680 info->voi_dtype = voi_dtype;
681
682 if ((nbkts = stats_vss_hist_nbkts_hlpr(info)) == 0)
683 return (EINVAL);
684
685 switch (info->hist_dtype) {
686 case VSD_DTYPE_CRHIST32:
687 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist32, nbkts);
688 break;
689 case VSD_DTYPE_DRHIST32:
690 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist32, nbkts);
691 break;
692 case VSD_DTYPE_DVHIST32:
693 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist32, nbkts);
694 break;
695 case VSD_DTYPE_CRHIST64:
696 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist64, nbkts);
697 break;
698 case VSD_DTYPE_DRHIST64:
699 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist64, nbkts);
700 break;
701 case VSD_DTYPE_DVHIST64:
702 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist64, nbkts);
703 break;
704 default:
705 return (EINVAL);
706 }
707
708 vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
709 if (vss->iv == NULL)
710 return (ENOMEM);
711
712 hist = (struct voistatdata_hist *)vss->iv;
713 bkt_ub = info->lb;
714
715 for (bkt = (info->flags & VSD_HIST_LBOUND_INF), nloop = 0;
716 bkt < nbkts;
717 bkt++, nloop++) {
718 bkt_lb = bkt_ub;
719 if (stats_vss_hist_bkt_hlpr(info, nloop, &bkt_lb, &bkt_ub))
720 return (EINVAL);
721
722 switch (info->hist_dtype) {
723 case VSD_DTYPE_CRHIST32:
724 VSD(crhist32, hist)->bkts[bkt].lb = bkt_lb;
725 break;
726 case VSD_DTYPE_DRHIST32:
727 VSD(drhist32, hist)->bkts[bkt].lb = bkt_lb;
728 VSD(drhist32, hist)->bkts[bkt].ub = bkt_ub;
729 break;
730 case VSD_DTYPE_DVHIST32:
731 VSD(dvhist32, hist)->bkts[bkt].val = bkt_lb;
732 break;
733 case VSD_DTYPE_CRHIST64:
734 VSD(crhist64, hist)->bkts[bkt].lb = bkt_lb;
735 break;
736 case VSD_DTYPE_DRHIST64:
737 VSD(drhist64, hist)->bkts[bkt].lb = bkt_lb;
738 VSD(drhist64, hist)->bkts[bkt].ub = bkt_ub;
739 break;
740 case VSD_DTYPE_DVHIST64:
741 VSD(dvhist64, hist)->bkts[bkt].val = bkt_lb;
742 break;
743 default:
744 return (EINVAL);
745 }
746 }
747
748 lbinfbktlb = lbinfbktub = ubinfbktlb = ubinfbktub = NULL;
749
750 switch (info->hist_dtype) {
751 case VSD_DTYPE_CRHIST32:
752 lbinfbktlb = &VSD(crhist32, hist)->bkts[0].lb;
753 ubinfbktlb = &VSD(crhist32, hist)->bkts[nbkts - 1].lb;
754 break;
755 case VSD_DTYPE_DRHIST32:
756 lbinfbktlb = &VSD(drhist32, hist)->bkts[0].lb;
757 lbinfbktub = &VSD(drhist32, hist)->bkts[0].ub;
758 ubinfbktlb = &VSD(drhist32, hist)->bkts[nbkts - 1].lb;
759 ubinfbktub = &VSD(drhist32, hist)->bkts[nbkts - 1].ub;
760 break;
761 case VSD_DTYPE_CRHIST64:
762 lbinfbktlb = &VSD(crhist64, hist)->bkts[0].lb;
763 ubinfbktlb = &VSD(crhist64, hist)->bkts[nbkts - 1].lb;
764 break;
765 case VSD_DTYPE_DRHIST64:
766 lbinfbktlb = &VSD(drhist64, hist)->bkts[0].lb;
767 lbinfbktub = &VSD(drhist64, hist)->bkts[0].ub;
768 ubinfbktlb = &VSD(drhist64, hist)->bkts[nbkts - 1].lb;
769 ubinfbktub = &VSD(drhist64, hist)->bkts[nbkts - 1].ub;
770 break;
771 case VSD_DTYPE_DVHIST32:
772 case VSD_DTYPE_DVHIST64:
773 break;
774 default:
775 return (EINVAL);
776 }
777
778 if ((info->flags & VSD_HIST_LBOUND_INF) && lbinfbktlb) {
779 *lbinfbktlb = numeric_limits[LIM_MIN][info->voi_dtype];
780 /*
781 * Assignment from numeric_limit array for Q types assigns max
782 * possible integral/fractional value for underlying data type,
783 * but we must set control bits for this specific histogram per
784 * the user's choice of fractional bits, which we extract from
785 * info->lb.
786 */
787 if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
788 info->voi_dtype == VSD_DTYPE_Q_U32) {
789 /* Signedness doesn't matter for setting control bits. */
790 Q_SCVAL(lbinfbktlb->q32.sq32,
791 Q_GCVAL(info->lb.q32.sq32));
792 } else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
793 info->voi_dtype == VSD_DTYPE_Q_U64) {
794 /* Signedness doesn't matter for setting control bits. */
795 Q_SCVAL(lbinfbktlb->q64.sq64,
796 Q_GCVAL(info->lb.q64.sq64));
797 }
798 if (lbinfbktub)
799 *lbinfbktub = info->lb;
800 }
801 if ((info->flags & VSD_HIST_UBOUND_INF) && ubinfbktlb) {
802 *ubinfbktlb = bkt_lb;
803 if (ubinfbktub) {
804 *ubinfbktub = numeric_limits[LIM_MAX][info->voi_dtype];
805 if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
806 info->voi_dtype == VSD_DTYPE_Q_U32) {
807 Q_SCVAL(ubinfbktub->q32.sq32,
808 Q_GCVAL(info->lb.q32.sq32));
809 } else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
810 info->voi_dtype == VSD_DTYPE_Q_U64) {
811 Q_SCVAL(ubinfbktub->q64.sq64,
812 Q_GCVAL(info->lb.q64.sq64));
813 }
814 }
815 }
816
817 return (0);
818 }
819
820 int
821 stats_vss_tdgst_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
822 struct vss_tdgst_hlpr_info *info)
823 {
824 struct voistatdata_tdgst *tdgst;
825 struct ctdth32 *ctd32tree;
826 struct ctdth64 *ctd64tree;
827 struct voistatdata_tdgstctd32 *ctd32;
828 struct voistatdata_tdgstctd64 *ctd64;
829
830 info->voi_dtype = voi_dtype;
831
832 switch (info->tdgst_dtype) {
833 case VSD_DTYPE_TDGSTCLUST32:
834 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust32, info->nctds);
835 break;
836 case VSD_DTYPE_TDGSTCLUST64:
837 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust64, info->nctds);
838 break;
839 default:
840 return (EINVAL);
841 }
842
843 vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
844 if (vss->iv == NULL)
845 return (ENOMEM);
846
847 tdgst = (struct voistatdata_tdgst *)vss->iv;
848
849 switch (info->tdgst_dtype) {
850 case VSD_DTYPE_TDGSTCLUST32:
851 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
852 ARB_INIT(ctd32, ctdlnk, ctd32tree, info->nctds) {
853 Q_INI(&ctd32->mu, 0, 0, info->prec);
854 }
855 break;
856 case VSD_DTYPE_TDGSTCLUST64:
857 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
858 ARB_INIT(ctd64, ctdlnk, ctd64tree, info->nctds) {
859 Q_INI(&ctd64->mu, 0, 0, info->prec);
860 }
861 break;
862 default:
863 return (EINVAL);
864 }
865
866 return (0);
867 }
868
869 int
870 stats_vss_numeric_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
871 struct vss_numeric_hlpr_info *info)
872 {
873 struct voistatdata_numeric iv;
874
875 switch (vss->stype) {
876 case VS_STYPE_SUM:
877 iv = stats_ctor_vsd_numeric(0);
878 break;
879 case VS_STYPE_MIN:
880 iv = numeric_limits[LIM_MAX][voi_dtype];
881 break;
882 case VS_STYPE_MAX:
883 iv = numeric_limits[LIM_MIN][voi_dtype];
884 break;
885 default:
886 return (EINVAL);
887 }
888
889 vss->iv = stats_realloc(NULL, 0, vsd_dtype2size[voi_dtype], 0);
890 if (vss->iv == NULL)
891 return (ENOMEM);
892
893 vss->vs_dtype = voi_dtype;
894 vss->vsdsz = vsd_dtype2size[voi_dtype];
895 switch (voi_dtype) {
896 case VSD_DTYPE_INT_S32:
897 *((int32_t *)vss->iv) = iv.int32.s32;
898 break;
899 case VSD_DTYPE_INT_U32:
900 *((uint32_t *)vss->iv) = iv.int32.u32;
901 break;
902 case VSD_DTYPE_INT_S64:
903 *((int64_t *)vss->iv) = iv.int64.s64;
904 break;
905 case VSD_DTYPE_INT_U64:
906 *((uint64_t *)vss->iv) = iv.int64.u64;
907 break;
908 case VSD_DTYPE_INT_SLONG:
909 *((long *)vss->iv) = iv.intlong.slong;
910 break;
911 case VSD_DTYPE_INT_ULONG:
912 *((unsigned long *)vss->iv) = iv.intlong.ulong;
913 break;
914 case VSD_DTYPE_Q_S32:
915 *((s32q_t *)vss->iv) = Q_SCVAL(iv.q32.sq32,
916 Q_CTRLINI(info->prec));
917 break;
918 case VSD_DTYPE_Q_U32:
919 *((u32q_t *)vss->iv) = Q_SCVAL(iv.q32.uq32,
920 Q_CTRLINI(info->prec));
921 break;
922 case VSD_DTYPE_Q_S64:
923 *((s64q_t *)vss->iv) = Q_SCVAL(iv.q64.sq64,
924 Q_CTRLINI(info->prec));
925 break;
926 case VSD_DTYPE_Q_U64:
927 *((u64q_t *)vss->iv) = Q_SCVAL(iv.q64.uq64,
928 Q_CTRLINI(info->prec));
929 break;
930 default:
931 break;
932 }
933
934 return (0);
935 }
936
937 int
938 stats_vss_hlpr_init(enum vsd_dtype voi_dtype, uint32_t nvss,
939 struct voistatspec *vss)
940 {
941 int i, ret;
942
943 for (i = nvss - 1; i >= 0; i--) {
944 if (vss[i].hlpr && (ret = vss[i].hlpr(voi_dtype, &vss[i],
945 vss[i].hlprinfo)) != 0)
946 return (ret);
947 }
948
949 return (0);
950 }
951
952 void
953 stats_vss_hlpr_cleanup(uint32_t nvss, struct voistatspec *vss)
954 {
955 int i;
956
957 for (i = nvss - 1; i >= 0; i--) {
958 if (vss[i].hlpr) {
959 stats_free((void *)vss[i].iv);
960 vss[i].iv = NULL;
961 }
962 }
963 }
964
965 int
966 stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl)
967 {
968 int error;
969
970 error = 0;
971
972 TPL_LIST_WLOCK();
973 if (tpl_id < 0 || tpl_id >= (int)ntpl) {
974 error = ENOENT;
975 } else {
976 *tpl = tpllist[tpl_id];
977 /* XXXLAS: Acquire refcount on tpl. */
978 }
979 TPL_LIST_WUNLOCK();
980
981 return (error);
982 }
983
984 int
985 stats_tpl_fetch_allocid(const char *name, uint32_t hash)
986 {
987 int i, tpl_id;
988
989 tpl_id = -ESRCH;
990
991 TPL_LIST_RLOCK();
992 for (i = ntpl - 1; i >= 0; i--) {
993 if (name != NULL) {
994 if (strlen(name) == strlen(tpllist[i]->mb->tplname) &&
995 strncmp(name, tpllist[i]->mb->tplname,
996 TPL_MAX_NAME_LEN) == 0 && (!hash || hash ==
997 tpllist[i]->mb->tplhash)) {
998 tpl_id = i;
999 break;
1000 }
1001 } else if (hash == tpllist[i]->mb->tplhash) {
1002 tpl_id = i;
1003 break;
1004 }
1005 }
1006 TPL_LIST_RUNLOCK();
1007
1008 return (tpl_id);
1009 }
1010
1011 int
1012 stats_tpl_id2name(uint32_t tpl_id, char *buf, size_t len)
1013 {
1014 int error;
1015
1016 error = 0;
1017
1018 TPL_LIST_RLOCK();
1019 if (tpl_id < ntpl) {
1020 if (buf != NULL && len > strlen(tpllist[tpl_id]->mb->tplname))
1021 strlcpy(buf, tpllist[tpl_id]->mb->tplname, len);
1022 else
1023 error = EOVERFLOW;
1024 } else
1025 error = ENOENT;
1026 TPL_LIST_RUNLOCK();
1027
1028 return (error);
1029 }
1030
1031 int
1032 stats_tpl_sample_rollthedice(struct stats_tpl_sample_rate *rates, int nrates,
1033 void *seed_bytes, size_t seed_len)
1034 {
1035 uint32_t cum_pct, rnd_pct;
1036 int i;
1037
1038 cum_pct = 0;
1039
1040 /*
1041 * Choose a pseudorandom or seeded number in range [0,100] and use
1042 * it to make a sampling decision and template selection where required.
1043 * If no seed is supplied, a PRNG is used to generate a pseudorandom
1044 * number so that every selection is independent. If a seed is supplied,
1045 * the caller desires random selection across different seeds, but
1046 * deterministic selection given the same seed. This is achieved by
1047 * hashing the seed and using the hash as the random number source.
1048 *
1049 * XXXLAS: Characterise hash function output distribution.
1050 */
1051 if (seed_bytes == NULL)
1052 rnd_pct = random() / (INT32_MAX / 100);
1053 else
1054 rnd_pct = hash32_buf(seed_bytes, seed_len, 0) /
1055 (UINT32_MAX / 100U);
1056
1057 /*
1058 * We map the randomly selected percentage on to the interval [0,100]
1059 * consisting of the cumulatively summed template sampling percentages.
1060 * The difference between the cumulative sum of all template sampling
1061 * percentages and 100 is treated as a NULL assignment i.e. no stats
1062 * template will be assigned, and -1 returned instead.
1063 */
1064 for (i = 0; i < nrates; i++) {
1065 cum_pct += rates[i].tpl_sample_pct;
1066
1067 KASSERT(cum_pct <= 100, ("%s cum_pct %u > 100", __func__,
1068 cum_pct));
1069 if (rnd_pct > cum_pct || rates[i].tpl_sample_pct == 0)
1070 continue;
1071
1072 return (rates[i].tpl_slot_id);
1073 }
1074
1075 return (-1);
1076 }
1077
1078 int
1079 stats_v1_blob_clone(struct statsblobv1 **dst, size_t dstmaxsz,
1080 struct statsblobv1 *src, uint32_t flags)
1081 {
1082 int error, tmperror;
1083
1084 error = tmperror = 0;
1085
1086 if (src == NULL || dst == NULL ||
1087 src->cursz < sizeof(struct statsblob) ||
1088 ((flags & SB_CLONE_ALLOCDST) &&
1089 (flags & (SB_CLONE_USRDSTNOFAULT | SB_CLONE_USRDST)))) {
1090 error = EINVAL;
1091 } else if (flags & SB_CLONE_ALLOCDST) {
1092 *dst = stats_realloc(NULL, 0, src->cursz, 0);
1093 if (*dst)
1094 (*dst)->maxsz = dstmaxsz = src->cursz;
1095 else
1096 error = ENOMEM;
1097 } else if (*dst == NULL || dstmaxsz < sizeof(struct statsblob)) {
1098 error = EINVAL;
1099 }
1100
1101 if (!error) {
1102 size_t postcurszlen;
1103
1104 /*
1105 * Clone src into dst except for the maxsz field. If dst is too
1106 * small to hold all of src, only copy src's header and return
1107 * EOVERFLOW.
1108 */
1109 #ifdef _KERNEL
1110 if (flags & SB_CLONE_USRDSTNOFAULT)
1111 error = copyout_nofault(src, *dst,
1112 offsetof(struct statsblob, maxsz));
1113 else if (flags & SB_CLONE_USRDST)
1114 error = copyout(src, *dst,
1115 offsetof(struct statsblob, maxsz));
1116 else
1117 #endif
1118 memcpy(*dst, src, offsetof(struct statsblob, maxsz));
1119 #ifdef _KERNEL
1120 if (error != 0)
1121 goto out;
1122 #endif
1123
1124
1125 if (dstmaxsz >= src->cursz) {
1126 postcurszlen = src->cursz -
1127 offsetof(struct statsblob, cursz);
1128 } else {
1129 error = EOVERFLOW;
1130 postcurszlen = sizeof(struct statsblob) -
1131 offsetof(struct statsblob, cursz);
1132 }
1133 #ifdef _KERNEL
1134 if (flags & SB_CLONE_USRDSTNOFAULT)
1135 tmperror = copyout_nofault(&(src->cursz), &((*dst)->cursz),
1136 postcurszlen);
1137 else if (flags & SB_CLONE_USRDST)
1138 tmperror = copyout(&(src->cursz), &((*dst)->cursz),
1139 postcurszlen);
1140 else
1141 #endif
1142 memcpy(&((*dst)->cursz), &(src->cursz), postcurszlen);
1143
1144 error = error ? error : tmperror;
1145 }
1146 #ifdef _KERNEL
1147 out:
1148 #endif
1149
1150 return (error);
1151 }
1152
1153 int
1154 stats_v1_tpl_alloc(const char *name, uint32_t flags __unused)
1155 {
1156 struct statsblobv1_tpl *tpl, **newtpllist;
1157 struct statsblobv1 *tpl_sb;
1158 struct metablob *tpl_mb;
1159 int tpl_id;
1160
1161 if (name != NULL && strlen(name) > TPL_MAX_NAME_LEN)
1162 return (-EINVAL);
1163
1164 if (name != NULL && stats_tpl_fetch_allocid(name, 0) >= 0)
1165 return (-EEXIST);
1166
1167 tpl = stats_realloc(NULL, 0, sizeof(struct statsblobv1_tpl), M_ZERO);
1168 tpl_mb = stats_realloc(NULL, 0, sizeof(struct metablob), M_ZERO);
1169 tpl_sb = stats_realloc(NULL, 0, sizeof(struct statsblobv1), M_ZERO);
1170
1171 if (tpl_mb != NULL && name != NULL)
1172 tpl_mb->tplname = stats_strdup(name, 0);
1173
1174 if (tpl == NULL || tpl_sb == NULL || tpl_mb == NULL ||
1175 tpl_mb->tplname == NULL) {
1176 stats_free(tpl);
1177 stats_free(tpl_sb);
1178 if (tpl_mb != NULL) {
1179 stats_free(tpl_mb->tplname);
1180 stats_free(tpl_mb);
1181 }
1182 return (-ENOMEM);
1183 }
1184
1185 tpl->mb = tpl_mb;
1186 tpl->sb = tpl_sb;
1187
1188 tpl_sb->abi = STATS_ABI_V1;
1189 tpl_sb->endian =
1190 #if BYTE_ORDER == LITTLE_ENDIAN
1191 SB_LE;
1192 #elif BYTE_ORDER == BIG_ENDIAN
1193 SB_BE;
1194 #else
1195 SB_UE;
1196 #endif
1197 tpl_sb->cursz = tpl_sb->maxsz = sizeof(struct statsblobv1);
1198 tpl_sb->stats_off = tpl_sb->statsdata_off = sizeof(struct statsblobv1);
1199
1200 TPL_LIST_WLOCK();
1201 newtpllist = stats_realloc(tpllist, ntpl * sizeof(void *),
1202 (ntpl + 1) * sizeof(void *), 0);
1203 if (newtpllist != NULL) {
1204 tpl_id = ntpl++;
1205 tpllist = (struct statsblob_tpl **)newtpllist;
1206 tpllist[tpl_id] = (struct statsblob_tpl *)tpl;
1207 stats_tpl_update_hash(tpllist[tpl_id]);
1208 } else {
1209 stats_free(tpl);
1210 stats_free(tpl_sb);
1211 if (tpl_mb != NULL) {
1212 stats_free(tpl_mb->tplname);
1213 stats_free(tpl_mb);
1214 }
1215 tpl_id = -ENOMEM;
1216 }
1217 TPL_LIST_WUNLOCK();
1218
1219 return (tpl_id);
1220 }
1221
1222 int
1223 stats_v1_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, const char *voi_name,
1224 enum vsd_dtype voi_dtype, uint32_t nvss, struct voistatspec *vss,
1225 uint32_t flags)
1226 {
1227 struct voi *voi;
1228 struct voistat *tmpstat;
1229 struct statsblobv1 *tpl_sb;
1230 struct metablob *tpl_mb;
1231 int error, i, newstatdataidx, newvoibytes, newvoistatbytes,
1232 newvoistatdatabytes, newvoistatmaxid;
1233 uint32_t nbytes;
1234
1235 if (voi_id < 0 || voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES ||
1236 nvss == 0 || vss == NULL)
1237 return (EINVAL);
1238
1239 error = nbytes = newvoibytes = newvoistatbytes =
1240 newvoistatdatabytes = 0;
1241 newvoistatmaxid = -1;
1242
1243 /* Calculate the number of bytes required for the new voistats. */
1244 for (i = nvss - 1; i >= 0; i--) {
1245 if (vss[i].stype == 0 || vss[i].stype >= VS_NUM_STYPES ||
1246 vss[i].vs_dtype == 0 || vss[i].vs_dtype >= VSD_NUM_DTYPES ||
1247 vss[i].iv == NULL || vss[i].vsdsz == 0)
1248 return (EINVAL);
1249 if ((int)vss[i].stype > newvoistatmaxid)
1250 newvoistatmaxid = vss[i].stype;
1251 newvoistatdatabytes += vss[i].vsdsz;
1252 }
1253
1254 if (flags & SB_VOI_RELUPDATE) {
1255 /* XXXLAS: VOI state bytes may need to vary based on stat types. */
1256 newvoistatdatabytes += sizeof(struct voistatdata_voistate);
1257 }
1258 nbytes += newvoistatdatabytes;
1259
1260 TPL_LIST_WLOCK();
1261 if (tpl_id < ntpl) {
1262 tpl_sb = (struct statsblobv1 *)tpllist[tpl_id]->sb;
1263 tpl_mb = tpllist[tpl_id]->mb;
1264
1265 if (voi_id >= NVOIS(tpl_sb) || tpl_sb->vois[voi_id].id == -1) {
1266 /* Adding a new VOI and associated stats. */
1267 if (voi_id >= NVOIS(tpl_sb)) {
1268 /* We need to grow the tpl_sb->vois array. */
1269 newvoibytes = (voi_id - (NVOIS(tpl_sb) - 1)) *
1270 sizeof(struct voi);
1271 nbytes += newvoibytes;
1272 }
1273 newvoistatbytes =
1274 (newvoistatmaxid + 1) * sizeof(struct voistat);
1275 } else {
1276 /* Adding stats to an existing VOI. */
1277 if (newvoistatmaxid >
1278 tpl_sb->vois[voi_id].voistatmaxid) {
1279 newvoistatbytes = (newvoistatmaxid -
1280 tpl_sb->vois[voi_id].voistatmaxid) *
1281 sizeof(struct voistat);
1282 }
1283 /* XXXLAS: KPI does not yet support expanding VOIs. */
1284 error = EOPNOTSUPP;
1285 }
1286 nbytes += newvoistatbytes;
1287
1288 if (!error && newvoibytes > 0) {
1289 struct voi_meta *voi_meta = tpl_mb->voi_meta;
1290
1291 voi_meta = stats_realloc(voi_meta, voi_meta == NULL ?
1292 0 : NVOIS(tpl_sb) * sizeof(struct voi_meta),
1293 (1 + voi_id) * sizeof(struct voi_meta),
1294 M_ZERO);
1295
1296 if (voi_meta == NULL)
1297 error = ENOMEM;
1298 else
1299 tpl_mb->voi_meta = voi_meta;
1300 }
1301
1302 if (!error) {
1303 /* NB: Resizing can change where tpl_sb points. */
1304 error = stats_v1_blob_expand(&tpl_sb, newvoibytes,
1305 newvoistatbytes, newvoistatdatabytes);
1306 }
1307
1308 if (!error) {
1309 tpl_mb->voi_meta[voi_id].name = stats_strdup(voi_name,
1310 0);
1311 if (tpl_mb->voi_meta[voi_id].name == NULL)
1312 error = ENOMEM;
1313 }
1314
1315 if (!error) {
1316 /* Update the template list with the resized pointer. */
1317 tpllist[tpl_id]->sb = (struct statsblob *)tpl_sb;
1318
1319 /* Update the template. */
1320 voi = &tpl_sb->vois[voi_id];
1321
1322 if (voi->id < 0) {
1323 /* VOI is new and needs to be initialised. */
1324 voi->id = voi_id;
1325 voi->dtype = voi_dtype;
1326 voi->stats_off = tpl_sb->stats_off;
1327 if (flags & SB_VOI_RELUPDATE)
1328 voi->flags |= VOI_REQSTATE;
1329 } else {
1330 /*
1331 * XXXLAS: When this else block is written, the
1332 * "KPI does not yet support expanding VOIs"
1333 * error earlier in this function can be
1334 * removed. What is required here is to shuffle
1335 * the voistat array such that the new stats for
1336 * the voi are contiguous, which will displace
1337 * stats for other vois that reside after the
1338 * voi being updated. The other vois then need
1339 * to have their stats_off adjusted post
1340 * shuffle.
1341 */
1342 }
1343
1344 voi->voistatmaxid = newvoistatmaxid;
1345 newstatdataidx = 0;
1346
1347 if (voi->flags & VOI_REQSTATE) {
1348 /* Initialise the voistate stat in slot 0. */
1349 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off);
1350 tmpstat->stype = VS_STYPE_VOISTATE;
1351 tmpstat->flags = 0;
1352 tmpstat->dtype = VSD_DTYPE_VOISTATE;
1353 newstatdataidx = tmpstat->dsz =
1354 sizeof(struct voistatdata_numeric);
1355 tmpstat->data_off = tpl_sb->statsdata_off;
1356 }
1357
1358 for (i = 0; (uint32_t)i < nvss; i++) {
1359 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off +
1360 (vss[i].stype * sizeof(struct voistat)));
1361 KASSERT(tmpstat->stype < 0, ("voistat %p "
1362 "already initialised", tmpstat));
1363 tmpstat->stype = vss[i].stype;
1364 tmpstat->flags = vss[i].flags;
1365 tmpstat->dtype = vss[i].vs_dtype;
1366 tmpstat->dsz = vss[i].vsdsz;
1367 tmpstat->data_off = tpl_sb->statsdata_off +
1368 newstatdataidx;
1369 memcpy(BLOB_OFFSET(tpl_sb, tmpstat->data_off),
1370 vss[i].iv, vss[i].vsdsz);
1371 newstatdataidx += vss[i].vsdsz;
1372 }
1373
1374 /* Update the template version hash. */
1375 stats_tpl_update_hash(tpllist[tpl_id]);
1376 /* XXXLAS: Confirm tpl name/hash pair remains unique. */
1377 }
1378 } else
1379 error = EINVAL;
1380 TPL_LIST_WUNLOCK();
1381
1382 return (error);
1383 }
1384
1385 struct statsblobv1 *
1386 stats_v1_blob_alloc(uint32_t tpl_id, uint32_t flags __unused)
1387 {
1388 struct statsblobv1 *sb;
1389 int error;
1390
1391 sb = NULL;
1392
1393 TPL_LIST_RLOCK();
1394 if (tpl_id < ntpl) {
1395 sb = stats_realloc(NULL, 0, tpllist[tpl_id]->sb->maxsz, 0);
1396 if (sb != NULL) {
1397 sb->maxsz = tpllist[tpl_id]->sb->maxsz;
1398 error = stats_v1_blob_init_locked(sb, tpl_id, 0);
1399 } else
1400 error = ENOMEM;
1401
1402 if (error) {
1403 stats_free(sb);
1404 sb = NULL;
1405 }
1406 }
1407 TPL_LIST_RUNLOCK();
1408
1409 return (sb);
1410 }
1411
1412 void
1413 stats_v1_blob_destroy(struct statsblobv1 *sb)
1414 {
1415
1416 stats_free(sb);
1417 }
1418
1419 int
1420 stats_v1_voistat_fetch_dptr(struct statsblobv1 *sb, int32_t voi_id,
1421 enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd,
1422 size_t *retvsdsz)
1423 {
1424 struct voi *v;
1425 struct voistat *vs;
1426
1427 if (retvsd == NULL || sb == NULL || sb->abi != STATS_ABI_V1 ||
1428 voi_id >= NVOIS(sb))
1429 return (EINVAL);
1430
1431 v = &sb->vois[voi_id];
1432 if ((__typeof(v->voistatmaxid))stype > v->voistatmaxid)
1433 return (EINVAL);
1434
1435 vs = BLOB_OFFSET(sb, v->stats_off + (stype * sizeof(struct voistat)));
1436 *retvsd = BLOB_OFFSET(sb, vs->data_off);
1437 if (retdtype != NULL)
1438 *retdtype = vs->dtype;
1439 if (retvsdsz != NULL)
1440 *retvsdsz = vs->dsz;
1441
1442 return (0);
1443 }
1444
1445 int
1446 stats_v1_blob_init(struct statsblobv1 *sb, uint32_t tpl_id, uint32_t flags)
1447 {
1448 int error;
1449
1450 error = 0;
1451
1452 TPL_LIST_RLOCK();
1453 if (sb == NULL || tpl_id >= ntpl) {
1454 error = EINVAL;
1455 } else {
1456 error = stats_v1_blob_init_locked(sb, tpl_id, flags);
1457 }
1458 TPL_LIST_RUNLOCK();
1459
1460 return (error);
1461 }
1462
1463 static inline int
1464 stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id,
1465 uint32_t flags __unused)
1466 {
1467 int error;
1468
1469 TPL_LIST_RLOCK_ASSERT();
1470 error = (sb->maxsz >= tpllist[tpl_id]->sb->cursz) ? 0 : EOVERFLOW;
1471 KASSERT(!error,
1472 ("sb %d instead of %d bytes", sb->maxsz, tpllist[tpl_id]->sb->cursz));
1473
1474 if (!error) {
1475 memcpy(sb, tpllist[tpl_id]->sb, tpllist[tpl_id]->sb->cursz);
1476 sb->created = sb->lastrst = stats_sbinuptime();
1477 sb->tplhash = tpllist[tpl_id]->mb->tplhash;
1478 }
1479
1480 return (error);
1481 }
1482
1483 static int
1484 stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes,
1485 int newvoistatbytes, int newvoistatdatabytes)
1486 {
1487 struct statsblobv1 *sb;
1488 struct voi *tmpvoi;
1489 struct voistat *tmpvoistat, *voistat_array;
1490 int error, i, idxnewvois, idxnewvoistats, nbytes, nvoistats;
1491
1492 KASSERT(newvoibytes % sizeof(struct voi) == 0,
1493 ("Bad newvoibytes %d", newvoibytes));
1494 KASSERT(newvoistatbytes % sizeof(struct voistat) == 0,
1495 ("Bad newvoistatbytes %d", newvoistatbytes));
1496
1497 error = ((newvoibytes % sizeof(struct voi) == 0) &&
1498 (newvoistatbytes % sizeof(struct voistat) == 0)) ? 0 : EINVAL;
1499 sb = *sbpp;
1500 nbytes = newvoibytes + newvoistatbytes + newvoistatdatabytes;
1501
1502 /*
1503 * XXXLAS: Required until we gain support for flags which alter the
1504 * units of size/offset fields in key structs.
1505 */
1506 if (!error && ((((int)sb->cursz) + nbytes) > SB_V1_MAXSZ))
1507 error = EFBIG;
1508
1509 if (!error && (sb->cursz + nbytes > sb->maxsz)) {
1510 /* Need to expand our blob. */
1511 sb = stats_realloc(sb, sb->maxsz, sb->cursz + nbytes, M_ZERO);
1512 if (sb != NULL) {
1513 sb->maxsz = sb->cursz + nbytes;
1514 *sbpp = sb;
1515 } else
1516 error = ENOMEM;
1517 }
1518
1519 if (!error) {
1520 /*
1521 * Shuffle memory within the expanded blob working from the end
1522 * backwards, leaving gaps for the new voistat and voistatdata
1523 * structs at the beginning of their respective blob regions,
1524 * and for the new voi structs at the end of their blob region.
1525 */
1526 memmove(BLOB_OFFSET(sb, sb->statsdata_off + nbytes),
1527 BLOB_OFFSET(sb, sb->statsdata_off),
1528 sb->cursz - sb->statsdata_off);
1529 memmove(BLOB_OFFSET(sb, sb->stats_off + newvoibytes +
1530 newvoistatbytes), BLOB_OFFSET(sb, sb->stats_off),
1531 sb->statsdata_off - sb->stats_off);
1532
1533 /* First index of new voi/voistat structs to be initialised. */
1534 idxnewvois = NVOIS(sb);
1535 idxnewvoistats = (newvoistatbytes / sizeof(struct voistat)) - 1;
1536
1537 /* Update housekeeping variables and offsets. */
1538 sb->cursz += nbytes;
1539 sb->stats_off += newvoibytes;
1540 sb->statsdata_off += newvoibytes + newvoistatbytes;
1541
1542 /* XXXLAS: Zeroing not strictly needed but aids debugging. */
1543 memset(&sb->vois[idxnewvois], '\0', newvoibytes);
1544 memset(BLOB_OFFSET(sb, sb->stats_off), '\0',
1545 newvoistatbytes);
1546 memset(BLOB_OFFSET(sb, sb->statsdata_off), '\0',
1547 newvoistatdatabytes);
1548
1549 /* Initialise new voi array members and update offsets. */
1550 for (i = 0; i < NVOIS(sb); i++) {
1551 tmpvoi = &sb->vois[i];
1552 if (i >= idxnewvois) {
1553 tmpvoi->id = tmpvoi->voistatmaxid = -1;
1554 } else if (tmpvoi->id > -1) {
1555 tmpvoi->stats_off += newvoibytes +
1556 newvoistatbytes;
1557 }
1558 }
1559
1560 /* Initialise new voistat array members and update offsets. */
1561 nvoistats = (sb->statsdata_off - sb->stats_off) /
1562 sizeof(struct voistat);
1563 voistat_array = BLOB_OFFSET(sb, sb->stats_off);
1564 for (i = 0; i < nvoistats; i++) {
1565 tmpvoistat = &voistat_array[i];
1566 if (i <= idxnewvoistats) {
1567 tmpvoistat->stype = -1;
1568 } else if (tmpvoistat->stype > -1) {
1569 tmpvoistat->data_off += nbytes;
1570 }
1571 }
1572 }
1573
1574 return (error);
1575 }
1576
1577 static void
1578 stats_v1_blob_finalise(struct statsblobv1 *sb __unused)
1579 {
1580
1581 /* XXXLAS: Fill this in. */
1582 }
1583
1584 static void
1585 stats_v1_blob_iter(struct statsblobv1 *sb, stats_v1_blob_itercb_t icb,
1586 void *usrctx, uint32_t flags)
1587 {
1588 struct voi *v;
1589 struct voistat *vs;
1590 struct sb_iter_ctx ctx;
1591 int i, j, firstvoi;
1592
1593 ctx.usrctx = usrctx;
1594 ctx.flags = SB_IT_FIRST_CB;
1595 firstvoi = 1;
1596
1597 for (i = 0; i < NVOIS(sb); i++) {
1598 v = &sb->vois[i];
1599 ctx.vslot = i;
1600 ctx.vsslot = -1;
1601 ctx.flags |= SB_IT_FIRST_VOISTAT;
1602
1603 if (firstvoi)
1604 ctx.flags |= SB_IT_FIRST_VOI;
1605 else if (i == (NVOIS(sb) - 1))
1606 ctx.flags |= SB_IT_LAST_VOI | SB_IT_LAST_CB;
1607
1608 if (v->id < 0 && (flags & SB_IT_NULLVOI)) {
1609 if (icb(sb, v, NULL, &ctx))
1610 return;
1611 firstvoi = 0;
1612 ctx.flags &= ~SB_IT_FIRST_CB;
1613 }
1614
1615 /* If NULL voi, v->voistatmaxid == -1 */
1616 for (j = 0; j <= v->voistatmaxid; j++) {
1617 vs = &((struct voistat *)BLOB_OFFSET(sb,
1618 v->stats_off))[j];
1619 if (vs->stype < 0 &&
1620 !(flags & SB_IT_NULLVOISTAT))
1621 continue;
1622
1623 if (j == v->voistatmaxid) {
1624 ctx.flags |= SB_IT_LAST_VOISTAT;
1625 if (i == (NVOIS(sb) - 1))
1626 ctx.flags |=
1627 SB_IT_LAST_CB;
1628 } else
1629 ctx.flags &= ~SB_IT_LAST_CB;
1630
1631 ctx.vsslot = j;
1632 if (icb(sb, v, vs, &ctx))
1633 return;
1634
1635 ctx.flags &= ~(SB_IT_FIRST_CB | SB_IT_FIRST_VOISTAT |
1636 SB_IT_LAST_VOISTAT);
1637 }
1638 ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI);
1639 }
1640 }
1641
1642 static inline void
1643 stats_voistatdata_tdgst_tostr(enum vsd_dtype voi_dtype __unused,
1644 const struct voistatdata_tdgst *tdgst, enum vsd_dtype tdgst_dtype,
1645 size_t tdgst_dsz __unused, enum sb_str_fmt fmt, struct sbuf *buf, int objdump)
1646 {
1647 const struct ctdth32 *ctd32tree;
1648 const struct ctdth64 *ctd64tree;
1649 const struct voistatdata_tdgstctd32 *ctd32;
1650 const struct voistatdata_tdgstctd64 *ctd64;
1651 const char *fmtstr;
1652 uint64_t smplcnt, compcnt;
1653 int is32bit, qmaxstrlen;
1654 uint16_t maxctds, curctds;
1655
1656 switch (tdgst_dtype) {
1657 case VSD_DTYPE_TDGSTCLUST32:
1658 smplcnt = CONSTVSD(tdgstclust32, tdgst)->smplcnt;
1659 compcnt = CONSTVSD(tdgstclust32, tdgst)->compcnt;
1660 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
1661 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
1662 ctd32tree = &CONSTVSD(tdgstclust32, tdgst)->ctdtree;
1663 ctd32 = (objdump ? ARB_CNODE(ctd32tree, 0) :
1664 ARB_CMIN(ctdth32, ctd32tree));
1665 qmaxstrlen = (ctd32 == NULL) ? 1 : Q_MAXSTRLEN(ctd32->mu, 10);
1666 is32bit = 1;
1667 ctd64tree = NULL;
1668 ctd64 = NULL;
1669 break;
1670 case VSD_DTYPE_TDGSTCLUST64:
1671 smplcnt = CONSTVSD(tdgstclust64, tdgst)->smplcnt;
1672 compcnt = CONSTVSD(tdgstclust64, tdgst)->compcnt;
1673 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
1674 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
1675 ctd64tree = &CONSTVSD(tdgstclust64, tdgst)->ctdtree;
1676 ctd64 = (objdump ? ARB_CNODE(ctd64tree, 0) :
1677 ARB_CMIN(ctdth64, ctd64tree));
1678 qmaxstrlen = (ctd64 == NULL) ? 1 : Q_MAXSTRLEN(ctd64->mu, 10);
1679 is32bit = 0;
1680 ctd32tree = NULL;
1681 ctd32 = NULL;
1682 break;
1683 default:
1684 return;
1685 }
1686
1687 switch (fmt) {
1688 case SB_STRFMT_FREEFORM:
1689 fmtstr = "smplcnt=%ju, compcnt=%ju, maxctds=%hu, nctds=%hu";
1690 break;
1691 case SB_STRFMT_JSON:
1692 default:
1693 fmtstr =
1694 "\"smplcnt\":%ju,\"compcnt\":%ju,\"maxctds\":%hu,"
1695 "\"nctds\":%hu,\"ctds\":[";
1696 break;
1697 }
1698 sbuf_printf(buf, fmtstr, (uintmax_t)smplcnt, (uintmax_t)compcnt,
1699 maxctds, curctds);
1700
1701 while ((is32bit ? NULL != ctd32 : NULL != ctd64)) {
1702 char qstr[qmaxstrlen];
1703
1704 switch (fmt) {
1705 case SB_STRFMT_FREEFORM:
1706 fmtstr = "\n\t\t\t\t";
1707 break;
1708 case SB_STRFMT_JSON:
1709 default:
1710 fmtstr = "{";
1711 break;
1712 }
1713 sbuf_cat(buf, fmtstr);
1714
1715 if (objdump) {
1716 switch (fmt) {
1717 case SB_STRFMT_FREEFORM:
1718 fmtstr = "ctd[%hu].";
1719 break;
1720 case SB_STRFMT_JSON:
1721 default:
1722 fmtstr = "\"ctd\":%hu,";
1723 break;
1724 }
1725 sbuf_printf(buf, fmtstr, is32bit ?
1726 ARB_SELFIDX(ctd32tree, ctd32) :
1727 ARB_SELFIDX(ctd64tree, ctd64));
1728 }
1729
1730 switch (fmt) {
1731 case SB_STRFMT_FREEFORM:
1732 fmtstr = "{mu=";
1733 break;
1734 case SB_STRFMT_JSON:
1735 default:
1736 fmtstr = "\"mu\":";
1737 break;
1738 }
1739 sbuf_cat(buf, fmtstr);
1740 Q_TOSTR((is32bit ? ctd32->mu : ctd64->mu), -1, 10, qstr,
1741 sizeof(qstr));
1742 sbuf_cat(buf, qstr);
1743
1744 switch (fmt) {
1745 case SB_STRFMT_FREEFORM:
1746 fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}";
1747 break;
1748 case SB_STRFMT_JSON:
1749 default:
1750 fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}";
1751 break;
1752 }
1753 sbuf_printf(buf, fmtstr,
1754 is32bit ? ctd32->cnt : (uintmax_t)ctd64->cnt);
1755
1756 if (is32bit)
1757 ctd32 = (objdump ? ARB_CNODE(ctd32tree,
1758 ARB_SELFIDX(ctd32tree, ctd32) + 1) :
1759 ARB_CNEXT(ctdth32, ctd32tree, ctd32));
1760 else
1761 ctd64 = (objdump ? ARB_CNODE(ctd64tree,
1762 ARB_SELFIDX(ctd64tree, ctd64) + 1) :
1763 ARB_CNEXT(ctdth64, ctd64tree, ctd64));
1764
1765 if (fmt == SB_STRFMT_JSON &&
1766 (is32bit ? NULL != ctd32 : NULL != ctd64))
1767 sbuf_putc(buf, ',');
1768 }
1769 if (fmt == SB_STRFMT_JSON)
1770 sbuf_cat(buf, "]");
1771 }
1772
1773 static inline void
1774 stats_voistatdata_hist_tostr(enum vsd_dtype voi_dtype,
1775 const struct voistatdata_hist *hist, enum vsd_dtype hist_dtype,
1776 size_t hist_dsz, enum sb_str_fmt fmt, struct sbuf *buf, int objdump)
1777 {
1778 const struct voistatdata_numeric *bkt_lb, *bkt_ub;
1779 const char *fmtstr;
1780 int is32bit;
1781 uint16_t i, nbkts;
1782
1783 switch (hist_dtype) {
1784 case VSD_DTYPE_CRHIST32:
1785 nbkts = HIST_VSDSZ2NBKTS(crhist32, hist_dsz);
1786 is32bit = 1;
1787 break;
1788 case VSD_DTYPE_DRHIST32:
1789 nbkts = HIST_VSDSZ2NBKTS(drhist32, hist_dsz);
1790 is32bit = 1;
1791 break;
1792 case VSD_DTYPE_DVHIST32:
1793 nbkts = HIST_VSDSZ2NBKTS(dvhist32, hist_dsz);
1794 is32bit = 1;
1795 break;
1796 case VSD_DTYPE_CRHIST64:
1797 nbkts = HIST_VSDSZ2NBKTS(crhist64, hist_dsz);
1798 is32bit = 0;
1799 break;
1800 case VSD_DTYPE_DRHIST64:
1801 nbkts = HIST_VSDSZ2NBKTS(drhist64, hist_dsz);
1802 is32bit = 0;
1803 break;
1804 case VSD_DTYPE_DVHIST64:
1805 nbkts = HIST_VSDSZ2NBKTS(dvhist64, hist_dsz);
1806 is32bit = 0;
1807 break;
1808 default:
1809 return;
1810 }
1811
1812 switch (fmt) {
1813 case SB_STRFMT_FREEFORM:
1814 fmtstr = "nbkts=%hu, ";
1815 break;
1816 case SB_STRFMT_JSON:
1817 default:
1818 fmtstr = "\"nbkts\":%hu,";
1819 break;
1820 }
1821 sbuf_printf(buf, fmtstr, nbkts);
1822
1823 switch (fmt) {
1824 case SB_STRFMT_FREEFORM:
1825 fmtstr = (is32bit ? "oob=%u" : "oob=%ju");
1826 break;
1827 case SB_STRFMT_JSON:
1828 default:
1829 fmtstr = (is32bit ? "\"oob\":%u,\"bkts\":[" :
1830 "\"oob\":%ju,\"bkts\":[");
1831 break;
1832 }
1833 sbuf_printf(buf, fmtstr, is32bit ? VSD_CONSTHIST_FIELDVAL(hist,
1834 hist_dtype, oob) : (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist,
1835 hist_dtype, oob));
1836
1837 for (i = 0; i < nbkts; i++) {
1838 switch (hist_dtype) {
1839 case VSD_DTYPE_CRHIST32:
1840 case VSD_DTYPE_CRHIST64:
1841 bkt_lb = VSD_CONSTCRHIST_FIELDPTR(hist, hist_dtype,
1842 bkts[i].lb);
1843 if (i < nbkts - 1)
1844 bkt_ub = VSD_CONSTCRHIST_FIELDPTR(hist,
1845 hist_dtype, bkts[i + 1].lb);
1846 else
1847 bkt_ub = &numeric_limits[LIM_MAX][voi_dtype];
1848 break;
1849 case VSD_DTYPE_DRHIST32:
1850 case VSD_DTYPE_DRHIST64:
1851 bkt_lb = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype,
1852 bkts[i].lb);
1853 bkt_ub = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype,
1854 bkts[i].ub);
1855 break;
1856 case VSD_DTYPE_DVHIST32:
1857 case VSD_DTYPE_DVHIST64:
1858 bkt_lb = bkt_ub = VSD_CONSTDVHIST_FIELDPTR(hist,
1859 hist_dtype, bkts[i].val);
1860 break;
1861 default:
1862 break;
1863 }
1864
1865 switch (fmt) {
1866 case SB_STRFMT_FREEFORM:
1867 fmtstr = "\n\t\t\t\t";
1868 break;
1869 case SB_STRFMT_JSON:
1870 default:
1871 fmtstr = "{";
1872 break;
1873 }
1874 sbuf_cat(buf, fmtstr);
1875
1876 if (objdump) {
1877 switch (fmt) {
1878 case SB_STRFMT_FREEFORM:
1879 fmtstr = "bkt[%hu].";
1880 break;
1881 case SB_STRFMT_JSON:
1882 default:
1883 fmtstr = "\"bkt\":%hu,";
1884 break;
1885 }
1886 sbuf_printf(buf, fmtstr, i);
1887 }
1888
1889 switch (fmt) {
1890 case SB_STRFMT_FREEFORM:
1891 fmtstr = "{lb=";
1892 break;
1893 case SB_STRFMT_JSON:
1894 default:
1895 fmtstr = "\"lb\":";
1896 break;
1897 }
1898 sbuf_cat(buf, fmtstr);
1899 stats_voistatdata_tostr((const struct voistatdata *)bkt_lb,
1900 voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric),
1901 fmt, buf, objdump);
1902
1903 switch (fmt) {
1904 case SB_STRFMT_FREEFORM:
1905 fmtstr = ",ub=";
1906 break;
1907 case SB_STRFMT_JSON:
1908 default:
1909 fmtstr = ",\"ub\":";
1910 break;
1911 }
1912 sbuf_cat(buf, fmtstr);
1913 stats_voistatdata_tostr((const struct voistatdata *)bkt_ub,
1914 voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric),
1915 fmt, buf, objdump);
1916
1917 switch (fmt) {
1918 case SB_STRFMT_FREEFORM:
1919 fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}";
1920 break;
1921 case SB_STRFMT_JSON:
1922 default:
1923 fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}";
1924 break;
1925 }
1926 sbuf_printf(buf, fmtstr, is32bit ?
1927 VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, bkts[i].cnt) :
1928 (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, hist_dtype,
1929 bkts[i].cnt));
1930
1931 if (fmt == SB_STRFMT_JSON && i < nbkts - 1)
1932 sbuf_putc(buf, ',');
1933 }
1934 if (fmt == SB_STRFMT_JSON)
1935 sbuf_cat(buf, "]");
1936 }
1937
1938 int
1939 stats_voistatdata_tostr(const struct voistatdata *vsd, enum vsd_dtype voi_dtype,
1940 enum vsd_dtype vsd_dtype, size_t vsd_sz, enum sb_str_fmt fmt,
1941 struct sbuf *buf, int objdump)
1942 {
1943 const char *fmtstr;
1944
1945 if (vsd == NULL || buf == NULL || voi_dtype >= VSD_NUM_DTYPES ||
1946 vsd_dtype >= VSD_NUM_DTYPES || fmt >= SB_STRFMT_NUM_FMTS)
1947 return (EINVAL);
1948
1949 switch (vsd_dtype) {
1950 case VSD_DTYPE_VOISTATE:
1951 switch (fmt) {
1952 case SB_STRFMT_FREEFORM:
1953 fmtstr = "prev=";
1954 break;
1955 case SB_STRFMT_JSON:
1956 default:
1957 fmtstr = "\"prev\":";
1958 break;
1959 }
1960 sbuf_cat(buf, fmtstr);
1961 /*
1962 * Render prev by passing it as *vsd and voi_dtype as vsd_dtype.
1963 */
1964 stats_voistatdata_tostr(
1965 (const struct voistatdata *)&CONSTVSD(voistate, vsd)->prev,
1966 voi_dtype, voi_dtype, vsd_sz, fmt, buf, objdump);
1967 break;
1968 case VSD_DTYPE_INT_S32:
1969 sbuf_printf(buf, "%d", vsd->int32.s32);
1970 break;
1971 case VSD_DTYPE_INT_U32:
1972 sbuf_printf(buf, "%u", vsd->int32.u32);
1973 break;
1974 case VSD_DTYPE_INT_S64:
1975 sbuf_printf(buf, "%jd", (intmax_t)vsd->int64.s64);
1976 break;
1977 case VSD_DTYPE_INT_U64:
1978 sbuf_printf(buf, "%ju", (uintmax_t)vsd->int64.u64);
1979 break;
1980 case VSD_DTYPE_INT_SLONG:
1981 sbuf_printf(buf, "%ld", vsd->intlong.slong);
1982 break;
1983 case VSD_DTYPE_INT_ULONG:
1984 sbuf_printf(buf, "%lu", vsd->intlong.ulong);
1985 break;
1986 case VSD_DTYPE_Q_S32:
1987 {
1988 char qstr[Q_MAXSTRLEN(vsd->q32.sq32, 10)];
1989 Q_TOSTR((s32q_t)vsd->q32.sq32, -1, 10, qstr, sizeof(qstr));
1990 sbuf_cat(buf, qstr);
1991 }
1992 break;
1993 case VSD_DTYPE_Q_U32:
1994 {
1995 char qstr[Q_MAXSTRLEN(vsd->q32.uq32, 10)];
1996 Q_TOSTR((u32q_t)vsd->q32.uq32, -1, 10, qstr, sizeof(qstr));
1997 sbuf_cat(buf, qstr);
1998 }
1999 break;
2000 case VSD_DTYPE_Q_S64:
2001 {
2002 char qstr[Q_MAXSTRLEN(vsd->q64.sq64, 10)];
2003 Q_TOSTR((s64q_t)vsd->q64.sq64, -1, 10, qstr, sizeof(qstr));
2004 sbuf_cat(buf, qstr);
2005 }
2006 break;
2007 case VSD_DTYPE_Q_U64:
2008 {
2009 char qstr[Q_MAXSTRLEN(vsd->q64.uq64, 10)];
2010 Q_TOSTR((u64q_t)vsd->q64.uq64, -1, 10, qstr, sizeof(qstr));
2011 sbuf_cat(buf, qstr);
2012 }
2013 break;
2014 case VSD_DTYPE_CRHIST32:
2015 case VSD_DTYPE_DRHIST32:
2016 case VSD_DTYPE_DVHIST32:
2017 case VSD_DTYPE_CRHIST64:
2018 case VSD_DTYPE_DRHIST64:
2019 case VSD_DTYPE_DVHIST64:
2020 stats_voistatdata_hist_tostr(voi_dtype, CONSTVSD(hist, vsd),
2021 vsd_dtype, vsd_sz, fmt, buf, objdump);
2022 break;
2023 case VSD_DTYPE_TDGSTCLUST32:
2024 case VSD_DTYPE_TDGSTCLUST64:
2025 stats_voistatdata_tdgst_tostr(voi_dtype,
2026 CONSTVSD(tdgst, vsd), vsd_dtype, vsd_sz, fmt, buf,
2027 objdump);
2028 break;
2029 default:
2030 break;
2031 }
2032
2033 return (sbuf_error(buf));
2034 }
2035
2036 static void
2037 stats_v1_itercb_tostr_freeform(struct statsblobv1 *sb, struct voi *v,
2038 struct voistat *vs, struct sb_iter_ctx *ctx)
2039 {
2040 struct sb_tostrcb_ctx *sctx;
2041 struct metablob *tpl_mb;
2042 struct sbuf *buf;
2043 void *vsd;
2044 uint8_t dump;
2045
2046 sctx = ctx->usrctx;
2047 buf = sctx->buf;
2048 tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
2049 dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
2050
2051 if (ctx->flags & SB_IT_FIRST_CB) {
2052 sbuf_printf(buf, "struct statsblobv1@%p", sb);
2053 if (dump) {
2054 sbuf_printf(buf, ", abi=%hhu, endian=%hhu, maxsz=%hu, "
2055 "cursz=%hu, created=%jd, lastrst=%jd, flags=0x%04hx, "
2056 "stats_off=%hu, statsdata_off=%hu",
2057 sb->abi, sb->endian, sb->maxsz, sb->cursz,
2058 sb->created, sb->lastrst, sb->flags, sb->stats_off,
2059 sb->statsdata_off);
2060 }
2061 sbuf_printf(buf, ", tplhash=%u", sb->tplhash);
2062 }
2063
2064 if (ctx->flags & SB_IT_FIRST_VOISTAT) {
2065 sbuf_printf(buf, "\n\tvois[%hd]: id=%hd", ctx->vslot, v->id);
2066 if (v->id < 0)
2067 return;
2068 sbuf_printf(buf, ", name=\"%s\"", (tpl_mb == NULL) ? "" :
2069 tpl_mb->voi_meta[v->id].name);
2070 if (dump)
2071 sbuf_printf(buf, ", flags=0x%04hx, dtype=%s, "
2072 "voistatmaxid=%hhd, stats_off=%hu", v->flags,
2073 vsd_dtype2name[v->dtype], v->voistatmaxid, v->stats_off);
2074 }
2075
2076 if (!dump && vs->stype <= 0)
2077 return;
2078
2079 sbuf_printf(buf, "\n\t\tvois[%hd]stat[%hhd]: stype=", v->id, ctx->vsslot);
2080 if (vs->stype < 0) {
2081 sbuf_printf(buf, "%hhd", vs->stype);
2082 return;
2083 } else
2084 sbuf_printf(buf, "%s, errs=%hu", vs_stype2name[vs->stype],
2085 vs->errs);
2086 vsd = BLOB_OFFSET(sb, vs->data_off);
2087 if (dump)
2088 sbuf_printf(buf, ", flags=0x%04x, dtype=%s, dsz=%hu, "
2089 "data_off=%hu", vs->flags, vsd_dtype2name[vs->dtype],
2090 vs->dsz, vs->data_off);
2091
2092 sbuf_printf(buf, "\n\t\t\tvoistatdata: ");
2093 stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
2094 sctx->fmt, buf, dump);
2095 }
2096
2097 static void
2098 stats_v1_itercb_tostr_json(struct statsblobv1 *sb, struct voi *v, struct voistat *vs,
2099 struct sb_iter_ctx *ctx)
2100 {
2101 struct sb_tostrcb_ctx *sctx;
2102 struct metablob *tpl_mb;
2103 struct sbuf *buf;
2104 const char *fmtstr;
2105 void *vsd;
2106 uint8_t dump;
2107
2108 sctx = ctx->usrctx;
2109 buf = sctx->buf;
2110 tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
2111 dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
2112
2113 if (ctx->flags & SB_IT_FIRST_CB) {
2114 sbuf_putc(buf, '{');
2115 if (dump) {
2116 sbuf_printf(buf, "\"abi\":%hhu,\"endian\":%hhu,"
2117 "\"maxsz\":%hu,\"cursz\":%hu,\"created\":%jd,"
2118 "\"lastrst\":%jd,\"flags\":%hu,\"stats_off\":%hu,"
2119 "\"statsdata_off\":%hu,", sb->abi,
2120 sb->endian, sb->maxsz, sb->cursz, sb->created,
2121 sb->lastrst, sb->flags, sb->stats_off,
2122 sb->statsdata_off);
2123 }
2124
2125 if (tpl_mb == NULL)
2126 fmtstr = "\"tplname\":%s,\"tplhash\":%u,\"vois\":{";
2127 else
2128 fmtstr = "\"tplname\":\"%s\",\"tplhash\":%u,\"vois\":{";
2129
2130 sbuf_printf(buf, fmtstr, tpl_mb ? tpl_mb->tplname : "null",
2131 sb->tplhash);
2132 }
2133
2134 if (ctx->flags & SB_IT_FIRST_VOISTAT) {
2135 if (dump) {
2136 sbuf_printf(buf, "\"[%d]\":{\"id\":%d", ctx->vslot,
2137 v->id);
2138 if (v->id < 0) {
2139 sbuf_printf(buf, "},");
2140 return;
2141 }
2142
2143 if (tpl_mb == NULL)
2144 fmtstr = ",\"name\":%s,\"flags\":%hu,"
2145 "\"dtype\":\"%s\",\"voistatmaxid\":%hhd,"
2146 "\"stats_off\":%hu,";
2147 else
2148 fmtstr = ",\"name\":\"%s\",\"flags\":%hu,"
2149 "\"dtype\":\"%s\",\"voistatmaxid\":%hhd,"
2150 "\"stats_off\":%hu,";
2151
2152 sbuf_printf(buf, fmtstr, tpl_mb ?
2153 tpl_mb->voi_meta[v->id].name : "null", v->flags,
2154 vsd_dtype2name[v->dtype], v->voistatmaxid,
2155 v->stats_off);
2156 } else {
2157 if (tpl_mb == NULL) {
2158 sbuf_printf(buf, "\"[%hd]\":{", v->id);
2159 } else {
2160 sbuf_printf(buf, "\"%s\":{",
2161 tpl_mb->voi_meta[v->id].name);
2162 }
2163 }
2164 sbuf_cat(buf, "\"stats\":{");
2165 }
2166
2167 vsd = BLOB_OFFSET(sb, vs->data_off);
2168 if (dump) {
2169 sbuf_printf(buf, "\"[%hhd]\":", ctx->vsslot);
2170 if (vs->stype < 0) {
2171 sbuf_printf(buf, "{\"stype\":-1},");
2172 return;
2173 }
2174 sbuf_printf(buf, "{\"stype\":\"%s\",\"errs\":%hu,\"flags\":%hu,"
2175 "\"dtype\":\"%s\",\"data_off\":%hu,\"voistatdata\":{",
2176 vs_stype2name[vs->stype], vs->errs, vs->flags,
2177 vsd_dtype2name[vs->dtype], vs->data_off);
2178 } else if (vs->stype > 0) {
2179 if (tpl_mb == NULL)
2180 sbuf_printf(buf, "\"[%hhd]\":", vs->stype);
2181 else
2182 sbuf_printf(buf, "\"%s\":", vs_stype2name[vs->stype]);
2183 } else
2184 return;
2185
2186 if ((vs->flags & VS_VSDVALID) || dump) {
2187 if (!dump)
2188 sbuf_printf(buf, "{\"errs\":%hu,", vs->errs);
2189 /* Simple non-compound VSD types need a key. */
2190 if (!vsd_compoundtype[vs->dtype])
2191 sbuf_cat(buf, "\"val\":");
2192 stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
2193 sctx->fmt, buf, dump);
2194 sbuf_cat(buf, dump ? "}}" : "}");
2195 } else
2196 sbuf_cat(buf, dump ? "null}" : "null");
2197
2198 if (ctx->flags & SB_IT_LAST_VOISTAT)
2199 sbuf_cat(buf, "}}");
2200
2201 if (ctx->flags & SB_IT_LAST_CB)
2202 sbuf_cat(buf, "}}");
2203 else
2204 sbuf_putc(buf, ',');
2205 }
2206
2207 static int
2208 stats_v1_itercb_tostr(struct statsblobv1 *sb, struct voi *v, struct voistat *vs,
2209 struct sb_iter_ctx *ctx)
2210 {
2211 struct sb_tostrcb_ctx *sctx;
2212
2213 sctx = ctx->usrctx;
2214
2215 switch (sctx->fmt) {
2216 case SB_STRFMT_FREEFORM:
2217 stats_v1_itercb_tostr_freeform(sb, v, vs, ctx);
2218 break;
2219 case SB_STRFMT_JSON:
2220 stats_v1_itercb_tostr_json(sb, v, vs, ctx);
2221 break;
2222 default:
2223 break;
2224 }
2225
2226 return (sbuf_error(sctx->buf));
2227 }
2228
2229 int
2230 stats_v1_blob_tostr(struct statsblobv1 *sb, struct sbuf *buf,
2231 enum sb_str_fmt fmt, uint32_t flags)
2232 {
2233 struct sb_tostrcb_ctx sctx;
2234 uint32_t iflags;
2235
2236 if (sb == NULL || sb->abi != STATS_ABI_V1 || buf == NULL ||
2237 fmt >= SB_STRFMT_NUM_FMTS)
2238 return (EINVAL);
2239
2240 sctx.buf = buf;
2241 sctx.fmt = fmt;
2242 sctx.flags = flags;
2243
2244 if (flags & SB_TOSTR_META) {
2245 if (stats_tpl_fetch(stats_tpl_fetch_allocid(NULL, sb->tplhash),
2246 &sctx.tpl))
2247 return (EINVAL);
2248 } else
2249 sctx.tpl = NULL;
2250
2251 iflags = 0;
2252 if (flags & SB_TOSTR_OBJDUMP)
2253 iflags |= (SB_IT_NULLVOI | SB_IT_NULLVOISTAT);
2254 stats_v1_blob_iter(sb, stats_v1_itercb_tostr, &sctx, iflags);
2255
2256 return (sbuf_error(buf));
2257 }
2258
2259 static int
2260 stats_v1_itercb_visit(struct statsblobv1 *sb, struct voi *v,
2261 struct voistat *vs, struct sb_iter_ctx *ctx)
2262 {
2263 struct sb_visitcb_ctx *vctx;
2264 struct sb_visit sbv;
2265
2266 vctx = ctx->usrctx;
2267
2268 sbv.tplhash = sb->tplhash;
2269 sbv.voi_id = v->id;
2270 sbv.voi_dtype = v->dtype;
2271 sbv.vs_stype = vs->stype;
2272 sbv.vs_dtype = vs->dtype;
2273 sbv.vs_dsz = vs->dsz;
2274 sbv.vs_data = BLOB_OFFSET(sb, vs->data_off);
2275 sbv.vs_errs = vs->errs;
2276 sbv.flags = ctx->flags & (SB_IT_FIRST_CB | SB_IT_LAST_CB |
2277 SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT |
2278 SB_IT_LAST_VOISTAT);
2279
2280 return (vctx->cb(&sbv, vctx->usrctx));
2281 }
2282
2283 int
2284 stats_v1_blob_visit(struct statsblobv1 *sb, stats_blob_visitcb_t func,
2285 void *usrctx)
2286 {
2287 struct sb_visitcb_ctx vctx;
2288
2289 if (sb == NULL || sb->abi != STATS_ABI_V1 || func == NULL)
2290 return (EINVAL);
2291
2292 vctx.cb = func;
2293 vctx.usrctx = usrctx;
2294
2295 stats_v1_blob_iter(sb, stats_v1_itercb_visit, &vctx, 0);
2296
2297 return (0);
2298 }
2299
2300 static int
2301 stats_v1_icb_reset_voistat(struct statsblobv1 *sb, struct voi *v __unused,
2302 struct voistat *vs, struct sb_iter_ctx *ctx __unused)
2303 {
2304 void *vsd;
2305
2306 if (vs->stype == VS_STYPE_VOISTATE)
2307 return (0);
2308
2309 vsd = BLOB_OFFSET(sb, vs->data_off);
2310
2311 /* Perform the stat type's default reset action. */
2312 switch (vs->stype) {
2313 case VS_STYPE_SUM:
2314 switch (vs->dtype) {
2315 case VSD_DTYPE_Q_S32:
2316 Q_SIFVAL(VSD(q32, vsd)->sq32, 0);
2317 break;
2318 case VSD_DTYPE_Q_U32:
2319 Q_SIFVAL(VSD(q32, vsd)->uq32, 0);
2320 break;
2321 case VSD_DTYPE_Q_S64:
2322 Q_SIFVAL(VSD(q64, vsd)->sq64, 0);
2323 break;
2324 case VSD_DTYPE_Q_U64:
2325 Q_SIFVAL(VSD(q64, vsd)->uq64, 0);
2326 break;
2327 default:
2328 bzero(vsd, vs->dsz);
2329 break;
2330 }
2331 break;
2332 case VS_STYPE_MAX:
2333 switch (vs->dtype) {
2334 case VSD_DTYPE_Q_S32:
2335 Q_SIFVAL(VSD(q32, vsd)->sq32,
2336 Q_IFMINVAL(VSD(q32, vsd)->sq32));
2337 break;
2338 case VSD_DTYPE_Q_U32:
2339 Q_SIFVAL(VSD(q32, vsd)->uq32,
2340 Q_IFMINVAL(VSD(q32, vsd)->uq32));
2341 break;
2342 case VSD_DTYPE_Q_S64:
2343 Q_SIFVAL(VSD(q64, vsd)->sq64,
2344 Q_IFMINVAL(VSD(q64, vsd)->sq64));
2345 break;
2346 case VSD_DTYPE_Q_U64:
2347 Q_SIFVAL(VSD(q64, vsd)->uq64,
2348 Q_IFMINVAL(VSD(q64, vsd)->uq64));
2349 break;
2350 default:
2351 memcpy(vsd, &numeric_limits[LIM_MIN][vs->dtype],
2352 vs->dsz);
2353 break;
2354 }
2355 break;
2356 case VS_STYPE_MIN:
2357 switch (vs->dtype) {
2358 case VSD_DTYPE_Q_S32:
2359 Q_SIFVAL(VSD(q32, vsd)->sq32,
2360 Q_IFMAXVAL(VSD(q32, vsd)->sq32));
2361 break;
2362 case VSD_DTYPE_Q_U32:
2363 Q_SIFVAL(VSD(q32, vsd)->uq32,
2364 Q_IFMAXVAL(VSD(q32, vsd)->uq32));
2365 break;
2366 case VSD_DTYPE_Q_S64:
2367 Q_SIFVAL(VSD(q64, vsd)->sq64,
2368 Q_IFMAXVAL(VSD(q64, vsd)->sq64));
2369 break;
2370 case VSD_DTYPE_Q_U64:
2371 Q_SIFVAL(VSD(q64, vsd)->uq64,
2372 Q_IFMAXVAL(VSD(q64, vsd)->uq64));
2373 break;
2374 default:
2375 memcpy(vsd, &numeric_limits[LIM_MAX][vs->dtype],
2376 vs->dsz);
2377 break;
2378 }
2379 break;
2380 case VS_STYPE_HIST:
2381 {
2382 /* Reset bucket counts. */
2383 struct voistatdata_hist *hist;
2384 int i, is32bit;
2385 uint16_t nbkts;
2386
2387 hist = VSD(hist, vsd);
2388 switch (vs->dtype) {
2389 case VSD_DTYPE_CRHIST32:
2390 nbkts = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
2391 is32bit = 1;
2392 break;
2393 case VSD_DTYPE_DRHIST32:
2394 nbkts = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
2395 is32bit = 1;
2396 break;
2397 case VSD_DTYPE_DVHIST32:
2398 nbkts = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
2399 is32bit = 1;
2400 break;
2401 case VSD_DTYPE_CRHIST64:
2402 nbkts = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
2403 is32bit = 0;
2404 break;
2405 case VSD_DTYPE_DRHIST64:
2406 nbkts = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
2407 is32bit = 0;
2408 break;
2409 case VSD_DTYPE_DVHIST64:
2410 nbkts = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
2411 is32bit = 0;
2412 break;
2413 default:
2414 return (0);
2415 }
2416
2417 bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, oob),
2418 is32bit ? sizeof(uint32_t) : sizeof(uint64_t));
2419 for (i = nbkts - 1; i >= 0; i--) {
2420 bzero(VSD_HIST_FIELDPTR(hist, vs->dtype,
2421 bkts[i].cnt), is32bit ? sizeof(uint32_t) :
2422 sizeof(uint64_t));
2423 }
2424 break;
2425 }
2426 case VS_STYPE_TDGST:
2427 {
2428 /* Reset sample count centroids array/tree. */
2429 struct voistatdata_tdgst *tdgst;
2430 struct ctdth32 *ctd32tree;
2431 struct ctdth64 *ctd64tree;
2432 struct voistatdata_tdgstctd32 *ctd32;
2433 struct voistatdata_tdgstctd64 *ctd64;
2434
2435 tdgst = VSD(tdgst, vsd);
2436 switch (vs->dtype) {
2437 case VSD_DTYPE_TDGSTCLUST32:
2438 VSD(tdgstclust32, tdgst)->smplcnt = 0;
2439 VSD(tdgstclust32, tdgst)->compcnt = 0;
2440 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
2441 ARB_INIT(ctd32, ctdlnk, ctd32tree,
2442 ARB_MAXNODES(ctd32tree)) {
2443 ctd32->cnt = 0;
2444 Q_SIFVAL(ctd32->mu, 0);
2445 }
2446 #ifdef DIAGNOSTIC
2447 RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
2448 #endif
2449 break;
2450 case VSD_DTYPE_TDGSTCLUST64:
2451 VSD(tdgstclust64, tdgst)->smplcnt = 0;
2452 VSD(tdgstclust64, tdgst)->compcnt = 0;
2453 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
2454 ARB_INIT(ctd64, ctdlnk, ctd64tree,
2455 ARB_MAXNODES(ctd64tree)) {
2456 ctd64->cnt = 0;
2457 Q_SIFVAL(ctd64->mu, 0);
2458 }
2459 #ifdef DIAGNOSTIC
2460 RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
2461 #endif
2462 break;
2463 default:
2464 return (0);
2465 }
2466 break;
2467 }
2468 default:
2469 KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
2470 break;
2471 }
2472
2473 vs->errs = 0;
2474 vs->flags &= ~VS_VSDVALID;
2475
2476 return (0);
2477 }
2478
2479 int
2480 stats_v1_blob_snapshot(struct statsblobv1 **dst, size_t dstmaxsz,
2481 struct statsblobv1 *src, uint32_t flags)
2482 {
2483 int error;
2484
2485 if (src != NULL && src->abi == STATS_ABI_V1) {
2486 error = stats_v1_blob_clone(dst, dstmaxsz, src, flags);
2487 if (!error) {
2488 if (flags & SB_CLONE_RSTSRC) {
2489 stats_v1_blob_iter(src,
2490 stats_v1_icb_reset_voistat, NULL, 0);
2491 src->lastrst = stats_sbinuptime();
2492 }
2493 stats_v1_blob_finalise(*dst);
2494 }
2495 } else
2496 error = EINVAL;
2497
2498 return (error);
2499 }
2500
2501 static inline int
2502 stats_v1_voi_update_max(enum vsd_dtype voi_dtype __unused,
2503 struct voistatdata *voival, struct voistat *vs, void *vsd)
2504 {
2505 int error;
2506
2507 KASSERT(vs->dtype < VSD_NUM_DTYPES,
2508 ("Unknown VSD dtype %d", vs->dtype));
2509
2510 error = 0;
2511
2512 switch (vs->dtype) {
2513 case VSD_DTYPE_INT_S32:
2514 if (VSD(int32, vsd)->s32 < voival->int32.s32) {
2515 VSD(int32, vsd)->s32 = voival->int32.s32;
2516 vs->flags |= VS_VSDVALID;
2517 }
2518 break;
2519 case VSD_DTYPE_INT_U32:
2520 if (VSD(int32, vsd)->u32 < voival->int32.u32) {
2521 VSD(int32, vsd)->u32 = voival->int32.u32;
2522 vs->flags |= VS_VSDVALID;
2523 }
2524 break;
2525 case VSD_DTYPE_INT_S64:
2526 if (VSD(int64, vsd)->s64 < voival->int64.s64) {
2527 VSD(int64, vsd)->s64 = voival->int64.s64;
2528 vs->flags |= VS_VSDVALID;
2529 }
2530 break;
2531 case VSD_DTYPE_INT_U64:
2532 if (VSD(int64, vsd)->u64 < voival->int64.u64) {
2533 VSD(int64, vsd)->u64 = voival->int64.u64;
2534 vs->flags |= VS_VSDVALID;
2535 }
2536 break;
2537 case VSD_DTYPE_INT_SLONG:
2538 if (VSD(intlong, vsd)->slong < voival->intlong.slong) {
2539 VSD(intlong, vsd)->slong = voival->intlong.slong;
2540 vs->flags |= VS_VSDVALID;
2541 }
2542 break;
2543 case VSD_DTYPE_INT_ULONG:
2544 if (VSD(intlong, vsd)->ulong < voival->intlong.ulong) {
2545 VSD(intlong, vsd)->ulong = voival->intlong.ulong;
2546 vs->flags |= VS_VSDVALID;
2547 }
2548 break;
2549 case VSD_DTYPE_Q_S32:
2550 if (Q_QLTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
2551 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
2552 voival->q32.sq32)))) {
2553 vs->flags |= VS_VSDVALID;
2554 }
2555 break;
2556 case VSD_DTYPE_Q_U32:
2557 if (Q_QLTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
2558 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
2559 voival->q32.uq32)))) {
2560 vs->flags |= VS_VSDVALID;
2561 }
2562 break;
2563 case VSD_DTYPE_Q_S64:
2564 if (Q_QLTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
2565 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
2566 voival->q64.sq64)))) {
2567 vs->flags |= VS_VSDVALID;
2568 }
2569 break;
2570 case VSD_DTYPE_Q_U64:
2571 if (Q_QLTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
2572 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
2573 voival->q64.uq64)))) {
2574 vs->flags |= VS_VSDVALID;
2575 }
2576 break;
2577 default:
2578 error = EINVAL;
2579 break;
2580 }
2581
2582 return (error);
2583 }
2584
2585 static inline int
2586 stats_v1_voi_update_min(enum vsd_dtype voi_dtype __unused,
2587 struct voistatdata *voival, struct voistat *vs, void *vsd)
2588 {
2589 int error;
2590
2591 KASSERT(vs->dtype < VSD_NUM_DTYPES,
2592 ("Unknown VSD dtype %d", vs->dtype));
2593
2594 error = 0;
2595
2596 switch (vs->dtype) {
2597 case VSD_DTYPE_INT_S32:
2598 if (VSD(int32, vsd)->s32 > voival->int32.s32) {
2599 VSD(int32, vsd)->s32 = voival->int32.s32;
2600 vs->flags |= VS_VSDVALID;
2601 }
2602 break;
2603 case VSD_DTYPE_INT_U32:
2604 if (VSD(int32, vsd)->u32 > voival->int32.u32) {
2605 VSD(int32, vsd)->u32 = voival->int32.u32;
2606 vs->flags |= VS_VSDVALID;
2607 }
2608 break;
2609 case VSD_DTYPE_INT_S64:
2610 if (VSD(int64, vsd)->s64 > voival->int64.s64) {
2611 VSD(int64, vsd)->s64 = voival->int64.s64;
2612 vs->flags |= VS_VSDVALID;
2613 }
2614 break;
2615 case VSD_DTYPE_INT_U64:
2616 if (VSD(int64, vsd)->u64 > voival->int64.u64) {
2617 VSD(int64, vsd)->u64 = voival->int64.u64;
2618 vs->flags |= VS_VSDVALID;
2619 }
2620 break;
2621 case VSD_DTYPE_INT_SLONG:
2622 if (VSD(intlong, vsd)->slong > voival->intlong.slong) {
2623 VSD(intlong, vsd)->slong = voival->intlong.slong;
2624 vs->flags |= VS_VSDVALID;
2625 }
2626 break;
2627 case VSD_DTYPE_INT_ULONG:
2628 if (VSD(intlong, vsd)->ulong > voival->intlong.ulong) {
2629 VSD(intlong, vsd)->ulong = voival->intlong.ulong;
2630 vs->flags |= VS_VSDVALID;
2631 }
2632 break;
2633 case VSD_DTYPE_Q_S32:
2634 if (Q_QGTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
2635 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
2636 voival->q32.sq32)))) {
2637 vs->flags |= VS_VSDVALID;
2638 }
2639 break;
2640 case VSD_DTYPE_Q_U32:
2641 if (Q_QGTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
2642 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
2643 voival->q32.uq32)))) {
2644 vs->flags |= VS_VSDVALID;
2645 }
2646 break;
2647 case VSD_DTYPE_Q_S64:
2648 if (Q_QGTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
2649 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
2650 voival->q64.sq64)))) {
2651 vs->flags |= VS_VSDVALID;
2652 }
2653 break;
2654 case VSD_DTYPE_Q_U64:
2655 if (Q_QGTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
2656 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
2657 voival->q64.uq64)))) {
2658 vs->flags |= VS_VSDVALID;
2659 }
2660 break;
2661 default:
2662 error = EINVAL;
2663 break;
2664 }
2665
2666 return (error);
2667 }
2668
2669 static inline int
2670 stats_v1_voi_update_sum(enum vsd_dtype voi_dtype __unused,
2671 struct voistatdata *voival, struct voistat *vs, void *vsd)
2672 {
2673 int error;
2674
2675 KASSERT(vs->dtype < VSD_NUM_DTYPES,
2676 ("Unknown VSD dtype %d", vs->dtype));
2677
2678 error = 0;
2679
2680 switch (vs->dtype) {
2681 case VSD_DTYPE_INT_S32:
2682 VSD(int32, vsd)->s32 += voival->int32.s32;
2683 break;
2684 case VSD_DTYPE_INT_U32:
2685 VSD(int32, vsd)->u32 += voival->int32.u32;
2686 break;
2687 case VSD_DTYPE_INT_S64:
2688 VSD(int64, vsd)->s64 += voival->int64.s64;
2689 break;
2690 case VSD_DTYPE_INT_U64:
2691 VSD(int64, vsd)->u64 += voival->int64.u64;
2692 break;
2693 case VSD_DTYPE_INT_SLONG:
2694 VSD(intlong, vsd)->slong += voival->intlong.slong;
2695 break;
2696 case VSD_DTYPE_INT_ULONG:
2697 VSD(intlong, vsd)->ulong += voival->intlong.ulong;
2698 break;
2699 case VSD_DTYPE_Q_S32:
2700 error = Q_QADDQ(&VSD(q32, vsd)->sq32, voival->q32.sq32);
2701 break;
2702 case VSD_DTYPE_Q_U32:
2703 error = Q_QADDQ(&VSD(q32, vsd)->uq32, voival->q32.uq32);
2704 break;
2705 case VSD_DTYPE_Q_S64:
2706 error = Q_QADDQ(&VSD(q64, vsd)->sq64, voival->q64.sq64);
2707 break;
2708 case VSD_DTYPE_Q_U64:
2709 error = Q_QADDQ(&VSD(q64, vsd)->uq64, voival->q64.uq64);
2710 break;
2711 default:
2712 error = EINVAL;
2713 break;
2714 }
2715
2716 if (!error)
2717 vs->flags |= VS_VSDVALID;
2718
2719 return (error);
2720 }
2721
2722 static inline int
2723 stats_v1_voi_update_hist(enum vsd_dtype voi_dtype, struct voistatdata *voival,
2724 struct voistat *vs, struct voistatdata_hist *hist)
2725 {
2726 struct voistatdata_numeric *bkt_lb, *bkt_ub;
2727 uint64_t *oob64, *cnt64;
2728 uint32_t *oob32, *cnt32;
2729 int error, i, found, is32bit, has_ub, eq_only;
2730
2731 error = 0;
2732
2733 switch (vs->dtype) {
2734 case VSD_DTYPE_CRHIST32:
2735 i = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
2736 is32bit = 1;
2737 has_ub = eq_only = 0;
2738 oob32 = &VSD(crhist32, hist)->oob;
2739 break;
2740 case VSD_DTYPE_DRHIST32:
2741 i = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
2742 is32bit = has_ub = 1;
2743 eq_only = 0;
2744 oob32 = &VSD(drhist32, hist)->oob;
2745 break;
2746 case VSD_DTYPE_DVHIST32:
2747 i = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
2748 is32bit = eq_only = 1;
2749 has_ub = 0;
2750 oob32 = &VSD(dvhist32, hist)->oob;
2751 break;
2752 case VSD_DTYPE_CRHIST64:
2753 i = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
2754 is32bit = has_ub = eq_only = 0;
2755 oob64 = &VSD(crhist64, hist)->oob;
2756 break;
2757 case VSD_DTYPE_DRHIST64:
2758 i = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
2759 is32bit = eq_only = 0;
2760 has_ub = 1;
2761 oob64 = &VSD(drhist64, hist)->oob;
2762 break;
2763 case VSD_DTYPE_DVHIST64:
2764 i = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
2765 is32bit = has_ub = 0;
2766 eq_only = 1;
2767 oob64 = &VSD(dvhist64, hist)->oob;
2768 break;
2769 default:
2770 return (EINVAL);
2771 }
2772 i--; /* Adjust for 0-based array index. */
2773
2774 /* XXXLAS: Should probably use a better bucket search algorithm. ARB? */
2775 for (found = 0; i >= 0 && !found; i--) {
2776 switch (vs->dtype) {
2777 case VSD_DTYPE_CRHIST32:
2778 bkt_lb = &VSD(crhist32, hist)->bkts[i].lb;
2779 cnt32 = &VSD(crhist32, hist)->bkts[i].cnt;
2780 break;
2781 case VSD_DTYPE_DRHIST32:
2782 bkt_lb = &VSD(drhist32, hist)->bkts[i].lb;
2783 bkt_ub = &VSD(drhist32, hist)->bkts[i].ub;
2784 cnt32 = &VSD(drhist32, hist)->bkts[i].cnt;
2785 break;
2786 case VSD_DTYPE_DVHIST32:
2787 bkt_lb = &VSD(dvhist32, hist)->bkts[i].val;
2788 cnt32 = &VSD(dvhist32, hist)->bkts[i].cnt;
2789 break;
2790 case VSD_DTYPE_CRHIST64:
2791 bkt_lb = &VSD(crhist64, hist)->bkts[i].lb;
2792 cnt64 = &VSD(crhist64, hist)->bkts[i].cnt;
2793 break;
2794 case VSD_DTYPE_DRHIST64:
2795 bkt_lb = &VSD(drhist64, hist)->bkts[i].lb;
2796 bkt_ub = &VSD(drhist64, hist)->bkts[i].ub;
2797 cnt64 = &VSD(drhist64, hist)->bkts[i].cnt;
2798 break;
2799 case VSD_DTYPE_DVHIST64:
2800 bkt_lb = &VSD(dvhist64, hist)->bkts[i].val;
2801 cnt64 = &VSD(dvhist64, hist)->bkts[i].cnt;
2802 break;
2803 default:
2804 return (EINVAL);
2805 }
2806
2807 switch (voi_dtype) {
2808 case VSD_DTYPE_INT_S32:
2809 if (voival->int32.s32 >= bkt_lb->int32.s32) {
2810 if ((eq_only && voival->int32.s32 ==
2811 bkt_lb->int32.s32) ||
2812 (!eq_only && (!has_ub ||
2813 voival->int32.s32 < bkt_ub->int32.s32)))
2814 found = 1;
2815 }
2816 break;
2817 case VSD_DTYPE_INT_U32:
2818 if (voival->int32.u32 >= bkt_lb->int32.u32) {
2819 if ((eq_only && voival->int32.u32 ==
2820 bkt_lb->int32.u32) ||
2821 (!eq_only && (!has_ub ||
2822 voival->int32.u32 < bkt_ub->int32.u32)))
2823 found = 1;
2824 }
2825 break;
2826 case VSD_DTYPE_INT_S64:
2827 if (voival->int64.s64 >= bkt_lb->int64.s64)
2828 if ((eq_only && voival->int64.s64 ==
2829 bkt_lb->int64.s64) ||
2830 (!eq_only && (!has_ub ||
2831 voival->int64.s64 < bkt_ub->int64.s64)))
2832 found = 1;
2833 break;
2834 case VSD_DTYPE_INT_U64:
2835 if (voival->int64.u64 >= bkt_lb->int64.u64)
2836 if ((eq_only && voival->int64.u64 ==
2837 bkt_lb->int64.u64) ||
2838 (!eq_only && (!has_ub ||
2839 voival->int64.u64 < bkt_ub->int64.u64)))
2840 found = 1;
2841 break;
2842 case VSD_DTYPE_INT_SLONG:
2843 if (voival->intlong.slong >= bkt_lb->intlong.slong)
2844 if ((eq_only && voival->intlong.slong ==
2845 bkt_lb->intlong.slong) ||
2846 (!eq_only && (!has_ub ||
2847 voival->intlong.slong <
2848 bkt_ub->intlong.slong)))
2849 found = 1;
2850 break;
2851 case VSD_DTYPE_INT_ULONG:
2852 if (voival->intlong.ulong >= bkt_lb->intlong.ulong)
2853 if ((eq_only && voival->intlong.ulong ==
2854 bkt_lb->intlong.ulong) ||
2855 (!eq_only && (!has_ub ||
2856 voival->intlong.ulong <
2857 bkt_ub->intlong.ulong)))
2858 found = 1;
2859 break;
2860 case VSD_DTYPE_Q_S32:
2861 if (Q_QGEQ(voival->q32.sq32, bkt_lb->q32.sq32))
2862 if ((eq_only && Q_QEQ(voival->q32.sq32,
2863 bkt_lb->q32.sq32)) ||
2864 (!eq_only && (!has_ub ||
2865 Q_QLTQ(voival->q32.sq32,
2866 bkt_ub->q32.sq32))))
2867 found = 1;
2868 break;
2869 case VSD_DTYPE_Q_U32:
2870 if (Q_QGEQ(voival->q32.uq32, bkt_lb->q32.uq32))
2871 if ((eq_only && Q_QEQ(voival->q32.uq32,
2872 bkt_lb->q32.uq32)) ||
2873 (!eq_only && (!has_ub ||
2874 Q_QLTQ(voival->q32.uq32,
2875 bkt_ub->q32.uq32))))
2876 found = 1;
2877 break;
2878 case VSD_DTYPE_Q_S64:
2879 if (Q_QGEQ(voival->q64.sq64, bkt_lb->q64.sq64))
2880 if ((eq_only && Q_QEQ(voival->q64.sq64,
2881 bkt_lb->q64.sq64)) ||
2882 (!eq_only && (!has_ub ||
2883 Q_QLTQ(voival->q64.sq64,
2884 bkt_ub->q64.sq64))))
2885 found = 1;
2886 break;
2887 case VSD_DTYPE_Q_U64:
2888 if (Q_QGEQ(voival->q64.uq64, bkt_lb->q64.uq64))
2889 if ((eq_only && Q_QEQ(voival->q64.uq64,
2890 bkt_lb->q64.uq64)) ||
2891 (!eq_only && (!has_ub ||
2892 Q_QLTQ(voival->q64.uq64,
2893 bkt_ub->q64.uq64))))
2894 found = 1;
2895 break;
2896 default:
2897 break;
2898 }
2899 }
2900
2901 if (found) {
2902 if (is32bit)
2903 *cnt32 += 1;
2904 else
2905 *cnt64 += 1;
2906 } else {
2907 if (is32bit)
2908 *oob32 += 1;
2909 else
2910 *oob64 += 1;
2911 }
2912
2913 vs->flags |= VS_VSDVALID;
2914 return (error);
2915 }
2916
2917 static inline int
2918 stats_v1_vsd_tdgst_compress(enum vsd_dtype vs_dtype,
2919 struct voistatdata_tdgst *tdgst, int attempt)
2920 {
2921 struct ctdth32 *ctd32tree;
2922 struct ctdth64 *ctd64tree;
2923 struct voistatdata_tdgstctd32 *ctd32;
2924 struct voistatdata_tdgstctd64 *ctd64;
2925 uint64_t ebits, idxmask;
2926 uint32_t bitsperidx, nebits;
2927 int error, idx, is32bit, maxctds, remctds, tmperr;
2928
2929 error = 0;
2930
2931 switch (vs_dtype) {
2932 case VSD_DTYPE_TDGSTCLUST32:
2933 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
2934 if (!ARB_FULL(ctd32tree))
2935 return (0);
2936 VSD(tdgstclust32, tdgst)->compcnt++;
2937 maxctds = remctds = ARB_MAXNODES(ctd32tree);
2938 ARB_RESET_TREE(ctd32tree, ctdth32, maxctds);
2939 VSD(tdgstclust32, tdgst)->smplcnt = 0;
2940 is32bit = 1;
2941 ctd64tree = NULL;
2942 ctd64 = NULL;
2943 #ifdef DIAGNOSTIC
2944 RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
2945 #endif
2946 break;
2947 case VSD_DTYPE_TDGSTCLUST64:
2948 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
2949 if (!ARB_FULL(ctd64tree))
2950 return (0);
2951 VSD(tdgstclust64, tdgst)->compcnt++;
2952 maxctds = remctds = ARB_MAXNODES(ctd64tree);
2953 ARB_RESET_TREE(ctd64tree, ctdth64, maxctds);
2954 VSD(tdgstclust64, tdgst)->smplcnt = 0;
2955 is32bit = 0;
2956 ctd32tree = NULL;
2957 ctd32 = NULL;
2958 #ifdef DIAGNOSTIC
2959 RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
2960 #endif
2961 break;
2962 default:
2963 return (EINVAL);
2964 }
2965
2966 /*
2967 * Rebuild the t-digest ARB by pseudorandomly selecting centroids and
2968 * re-inserting the mu/cnt of each as a value and corresponding weight.
2969 */
2970
2971 /*
2972 * XXXCEM: random(9) is currently rand(3), not random(3). rand(3)
2973 * RAND_MAX happens to be approximately 31 bits (range [0,
2974 * 0x7ffffffd]), so the math kinda works out. When/if this portion of
2975 * the code is compiled in userspace, it gets the random(3) behavior,
2976 * which has expected range [0, 0x7fffffff].
2977 */
2978 #define bitsperrand 31
2979 ebits = 0;
2980 nebits = 0;
2981 bitsperidx = fls(maxctds);
2982 KASSERT(bitsperidx <= sizeof(ebits) << 3,
2983 ("%s: bitsperidx=%d, ebits=%d",
2984 __func__, bitsperidx, (int)(sizeof(ebits) << 3)));
2985 idxmask = (UINT64_C(1) << bitsperidx) - 1;
2986
2987 /* Initialise the free list with randomised centroid indices. */
2988 for (; remctds > 0; remctds--) {
2989 while (nebits < bitsperidx) {
2990 ebits |= ((uint64_t)random()) << nebits;
2991 nebits += bitsperrand;
2992 if (nebits > (sizeof(ebits) << 3))
2993 nebits = sizeof(ebits) << 3;
2994 }
2995 idx = ebits & idxmask;
2996 nebits -= bitsperidx;
2997 ebits >>= bitsperidx;
2998
2999 /*
3000 * Select the next centroid to put on the ARB free list. We
3001 * start with the centroid at our randomly selected array index,
3002 * and work our way forwards until finding one (the latter
3003 * aspect reduces re-insertion randomness, but is good enough).
3004 */
3005 do {
3006 if (idx >= maxctds)
3007 idx %= maxctds;
3008
3009 if (is32bit)
3010 ctd32 = ARB_NODE(ctd32tree, idx);
3011 else
3012 ctd64 = ARB_NODE(ctd64tree, idx);
3013 } while ((is32bit ? ARB_ISFREE(ctd32, ctdlnk) :
3014 ARB_ISFREE(ctd64, ctdlnk)) && ++idx);
3015
3016 /* Put the centroid on the ARB free list. */
3017 if (is32bit)
3018 ARB_RETURNFREE(ctd32tree, ctd32, ctdlnk);
3019 else
3020 ARB_RETURNFREE(ctd64tree, ctd64, ctdlnk);
3021 }
3022
3023 /*
3024 * The free list now contains the randomised indices of every centroid.
3025 * Walk the free list from start to end, re-inserting each centroid's
3026 * mu/cnt. The tdgst_add() call may or may not consume the free centroid
3027 * we re-insert values from during each loop iteration, so we must latch
3028 * the index of the next free list centroid before the re-insertion
3029 * call. The previous loop above should have left the centroid pointer
3030 * pointing to the element at the head of the free list.
3031 */
3032 KASSERT((is32bit ?
3033 ARB_FREEIDX(ctd32tree) == ARB_SELFIDX(ctd32tree, ctd32) :
3034 ARB_FREEIDX(ctd64tree) == ARB_SELFIDX(ctd64tree, ctd64)),
3035 ("%s: t-digest ARB@%p free list bug", __func__,
3036 (is32bit ? (void *)ctd32tree : (void *)ctd64tree)));
3037 remctds = maxctds;
3038 while ((is32bit ? ctd32 != NULL : ctd64 != NULL)) {
3039 tmperr = 0;
3040 if (is32bit) {
3041 s64q_t x;
3042
3043 idx = ARB_NEXTFREEIDX(ctd32, ctdlnk);
3044 /* Cloning a s32q_t into a s64q_t should never fail. */
3045 tmperr = Q_QCLONEQ(&x, ctd32->mu);
3046 tmperr = tmperr ? tmperr : stats_v1_vsd_tdgst_add(
3047 vs_dtype, tdgst, x, ctd32->cnt, attempt);
3048 ctd32 = ARB_NODE(ctd32tree, idx);
3049 KASSERT(ctd32 == NULL || ARB_ISFREE(ctd32, ctdlnk),
3050 ("%s: t-digest ARB@%p free list bug", __func__,
3051 ctd32tree));
3052 } else {
3053 idx = ARB_NEXTFREEIDX(ctd64, ctdlnk);
3054 tmperr = stats_v1_vsd_tdgst_add(vs_dtype, tdgst,
3055 ctd64->mu, ctd64->cnt, attempt);
3056 ctd64 = ARB_NODE(ctd64tree, idx);
3057 KASSERT(ctd64 == NULL || ARB_ISFREE(ctd64, ctdlnk),
3058 ("%s: t-digest ARB@%p free list bug", __func__,
3059 ctd64tree));
3060 }
3061 /*
3062 * This process should not produce errors, bugs notwithstanding.
3063 * Just in case, latch any errors and attempt all re-insertions.
3064 */
3065 error = tmperr ? tmperr : error;
3066 remctds--;
3067 }
3068
3069 KASSERT(remctds == 0, ("%s: t-digest ARB@%p free list bug", __func__,
3070 (is32bit ? (void *)ctd32tree : (void *)ctd64tree)));
3071
3072 return (error);
3073 }
3074
3075 static inline int
3076 stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst,
3077 s64q_t x, uint64_t weight, int attempt)
3078 {
3079 #ifdef DIAGNOSTIC
3080 char qstr[Q_MAXSTRLEN(x, 10)];
3081 #endif
3082 struct ctdth32 *ctd32tree;
3083 struct ctdth64 *ctd64tree;
3084 void *closest, *cur, *lb, *ub;
3085 struct voistatdata_tdgstctd32 *ctd32;
3086 struct voistatdata_tdgstctd64 *ctd64;
3087 uint64_t cnt, smplcnt, sum, tmpsum;
3088 s64q_t k, minz, q, z;
3089 int error, is32bit, n;
3090
3091 error = 0;
3092 minz = Q_INI(&z, 0, 0, Q_NFBITS(x));
3093
3094 switch (vs_dtype) {
3095 case VSD_DTYPE_TDGSTCLUST32:
3096 if ((UINT32_MAX - weight) < VSD(tdgstclust32, tdgst)->smplcnt)
3097 error = EOVERFLOW;
3098 smplcnt = VSD(tdgstclust32, tdgst)->smplcnt;
3099 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
3100 is32bit = 1;
3101 ctd64tree = NULL;
3102 ctd64 = NULL;
3103 break;
3104 case VSD_DTYPE_TDGSTCLUST64:
3105 if ((UINT64_MAX - weight) < VSD(tdgstclust64, tdgst)->smplcnt)
3106 error = EOVERFLOW;
3107 smplcnt = VSD(tdgstclust64, tdgst)->smplcnt;
3108 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
3109 is32bit = 0;
3110 ctd32tree = NULL;
3111 ctd32 = NULL;
3112 break;
3113 default:
3114 error = EINVAL;
3115 break;
3116 }
3117
3118 if (error)
3119 return (error);
3120
3121 /*
3122 * Inspired by Ted Dunning's AVLTreeDigest.java
3123 */
3124 do {
3125 #if defined(DIAGNOSTIC)
3126 KASSERT(attempt < 5,
3127 ("%s: Too many attempts", __func__));
3128 #endif
3129 if (attempt >= 5)
3130 return (EAGAIN);
3131
3132 Q_SIFVAL(minz, Q_IFMAXVAL(minz));
3133 closest = ub = NULL;
3134 sum = tmpsum = 0;
3135
3136 if (is32bit)
3137 lb = cur = (void *)(ctd32 = ARB_MIN(ctdth32, ctd32tree));
3138 else
3139 lb = cur = (void *)(ctd64 = ARB_MIN(ctdth64, ctd64tree));
3140
3141 if (lb == NULL) /* Empty tree. */
3142 lb = (is32bit ? (void *)ARB_ROOT(ctd32tree) :
3143 (void *)ARB_ROOT(ctd64tree));
3144
3145 /*
3146 * Find the set of centroids with minimum distance to x and
3147 * compute the sum of counts for all centroids with mean less
3148 * than the first centroid in the set.
3149 */
3150 for (; cur != NULL;
3151 cur = (is32bit ?
3152 (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) :
3153 (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) {
3154 if (is32bit) {
3155 cnt = ctd32->cnt;
3156 KASSERT(Q_PRECEQ(ctd32->mu, x),
3157 ("%s: Q_RELPREC(mu,x)=%d", __func__,
3158 Q_RELPREC(ctd32->mu, x)));
3159 /* Ok to assign as both have same precision. */
3160 z = ctd32->mu;
3161 } else {
3162 cnt = ctd64->cnt;
3163 KASSERT(Q_PRECEQ(ctd64->mu, x),
3164 ("%s: Q_RELPREC(mu,x)=%d", __func__,
3165 Q_RELPREC(ctd64->mu, x)));
3166 /* Ok to assign as both have same precision. */
3167 z = ctd64->mu;
3168 }
3169
3170 error = Q_QSUBQ(&z, x);
3171 #if defined(DIAGNOSTIC)
3172 KASSERT(!error, ("%s: unexpected error %d", __func__,
3173 error));
3174 #endif
3175 if (error)
3176 return (error);
3177
3178 z = Q_QABS(z);
3179 if (Q_QLTQ(z, minz)) {
3180 minz = z;
3181 lb = cur;
3182 sum = tmpsum;
3183 tmpsum += cnt;
3184 } else if (Q_QGTQ(z, minz)) {
3185 ub = cur;
3186 break;
3187 }
3188 }
3189
3190 cur = (is32bit ?
3191 (void *)(ctd32 = (struct voistatdata_tdgstctd32 *)lb) :
3192 (void *)(ctd64 = (struct voistatdata_tdgstctd64 *)lb));
3193
3194 for (n = 0; cur != ub; cur = (is32bit ?
3195 (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) :
3196 (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) {
3197 if (is32bit)
3198 cnt = ctd32->cnt;
3199 else
3200 cnt = ctd64->cnt;
3201
3202 q = Q_CTRLINI(16);
3203 if (smplcnt == 1)
3204 error = Q_QFRACI(&q, 1, 2);
3205 else
3206 /* [ sum + ((cnt - 1) / 2) ] / (smplcnt - 1) */
3207 error = Q_QFRACI(&q, (sum << 1) + cnt - 1,
3208 (smplcnt - 1) << 1);
3209 k = q;
3210 /* k = q x 4 x samplcnt x attempt */
3211 error |= Q_QMULI(&k, 4 * smplcnt * attempt);
3212 /* k = k x (1 - q) */
3213 error |= Q_QSUBI(&q, 1);
3214 q = Q_QABS(q);
3215 error |= Q_QMULQ(&k, q);
3216 #if defined(DIAGNOSTIC)
3217 #if !defined(_KERNEL)
3218 double q_dbl, k_dbl, q2d, k2d;
3219 q2d = Q_Q2D(q);
3220 k2d = Q_Q2D(k);
3221 q_dbl = smplcnt == 1 ? 0.5 :
3222 (sum + ((cnt - 1) / 2.0)) / (double)(smplcnt - 1);
3223 k_dbl = 4 * smplcnt * q_dbl * (1.0 - q_dbl) * attempt;
3224 /*
3225 * If the difference between q and q_dbl is greater than
3226 * the fractional precision of q, something is off.
3227 * NB: q is holding the value of 1 - q
3228 */
3229 q_dbl = 1.0 - q_dbl;
3230 KASSERT((q_dbl > q2d ? q_dbl - q2d : q2d - q_dbl) <
3231 (1.05 * ((double)1 / (double)(1ULL << Q_NFBITS(q)))),
3232 ("Q-type q bad precision"));
3233 KASSERT((k_dbl > k2d ? k_dbl - k2d : k2d - k_dbl) <
3234 1.0 + (0.01 * smplcnt),
3235 ("Q-type k bad precision"));
3236 #endif /* !_KERNEL */
3237 KASSERT(!error, ("%s: unexpected error %d", __func__,
3238 error));
3239 #endif /* DIAGNOSTIC */
3240 if (error)
3241 return (error);
3242 if ((is32bit && ((ctd32->cnt + weight) <=
3243 (uint64_t)Q_GIVAL(k))) ||
3244 (!is32bit && ((ctd64->cnt + weight) <=
3245 (uint64_t)Q_GIVAL(k)))) {
3246 n++;
3247 /* random() produces 31 bits. */
3248 if (random() < (INT32_MAX / n))
3249 closest = cur;
3250 }
3251 sum += cnt;
3252 }
3253 } while (closest == NULL &&
3254 (is32bit ? ARB_FULL(ctd32tree) : ARB_FULL(ctd64tree)) &&
3255 (error = stats_v1_vsd_tdgst_compress(vs_dtype, tdgst,
3256 attempt++)) == 0);
3257
3258 if (error)
3259 return (error);
3260
3261 if (closest != NULL) {
3262 /* Merge with an existing centroid. */
3263 if (is32bit) {
3264 ctd32 = (struct voistatdata_tdgstctd32 *)closest;
3265 error = Q_QSUBQ(&x, ctd32->mu);
3266 /*
3267 * The following calculation "x / (cnt + weight)"
3268 * computes the amount by which to adjust the centroid's
3269 * mu value in order to merge in the VOI sample.
3270 *
3271 * It can underflow (Q_QDIVI() returns ERANGE) when the
3272 * user centroids' fractional precision (which is
3273 * inherited by 'x') is too low to represent the result.
3274 *
3275 * A sophisticated approach to dealing with this issue
3276 * would minimise accumulation of error by tracking
3277 * underflow per centroid and making an adjustment when
3278 * a LSB's worth of underflow has accumulated.
3279 *
3280 * A simpler approach is to let the result underflow
3281 * i.e. merge the VOI sample into the centroid without
3282 * adjusting the centroid's mu, and rely on the user to
3283 * specify their t-digest with sufficient centroid
3284 * fractional precision such that the accumulation of
3285 * error from multiple underflows is of no material
3286 * consequence to the centroid's final value of mu.
3287 *
3288 * For the moment, the latter approach is employed by
3289 * simply ignoring ERANGE here.
3290 *
3291 * XXXLAS: Per-centroid underflow tracking is likely too
3292 * onerous, but it probably makes sense to accumulate a
3293 * single underflow error variable across all centroids
3294 * and report it as part of the digest to provide
3295 * additional visibility into the digest's fidelity.
3296 */
3297 error = error ? error :
3298 Q_QDIVI(&x, ctd32->cnt + weight);
3299 if ((error && error != ERANGE)
3300 || (error = Q_QADDQ(&ctd32->mu, x))) {
3301 #ifdef DIAGNOSTIC
3302 KASSERT(!error, ("%s: unexpected error %d",
3303 __func__, error));
3304 #endif
3305 return (error);
3306 }
3307 ctd32->cnt += weight;
3308 error = ARB_REINSERT(ctdth32, ctd32tree, ctd32) ==
3309 NULL ? 0 : EALREADY;
3310 #ifdef DIAGNOSTIC
3311 RB_REINSERT(rbctdth32,
3312 &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
3313 #endif
3314 } else {
3315 ctd64 = (struct voistatdata_tdgstctd64 *)closest;
3316 error = Q_QSUBQ(&x, ctd64->mu);
3317 error = error ? error :
3318 Q_QDIVI(&x, ctd64->cnt + weight);
3319 /* Refer to is32bit ERANGE discussion above. */
3320 if ((error && error != ERANGE)
3321 || (error = Q_QADDQ(&ctd64->mu, x))) {
3322 KASSERT(!error, ("%s: unexpected error %d",
3323 __func__, error));
3324 return (error);
3325 }
3326 ctd64->cnt += weight;
3327 error = ARB_REINSERT(ctdth64, ctd64tree, ctd64) ==
3328 NULL ? 0 : EALREADY;
3329 #ifdef DIAGNOSTIC
3330 RB_REINSERT(rbctdth64,
3331 &VSD(tdgstclust64, tdgst)->rbctdtree, ctd64);
3332 #endif
3333 }
3334 } else {
3335 /*
3336 * Add a new centroid. If digest compression is working
3337 * correctly, there should always be at least one free.
3338 */
3339 if (is32bit) {
3340 ctd32 = ARB_GETFREE(ctd32tree, ctdlnk);
3341 #ifdef DIAGNOSTIC
3342 KASSERT(ctd32 != NULL,
3343 ("%s: t-digest@%p has no free centroids",
3344 __func__, tdgst));
3345 #endif
3346 if (ctd32 == NULL)
3347 return (EAGAIN);
3348 if ((error = Q_QCPYVALQ(&ctd32->mu, x)))
3349 return (error);
3350 ctd32->cnt = weight;
3351 error = ARB_INSERT(ctdth32, ctd32tree, ctd32) == NULL ?
3352 0 : EALREADY;
3353 #ifdef DIAGNOSTIC
3354 RB_INSERT(rbctdth32,
3355 &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
3356 #endif
3357 } else {
3358 ctd64 = ARB_GETFREE(ctd64tree, ctdlnk);
3359 #ifdef DIAGNOSTIC
3360 KASSERT(ctd64 != NULL,
3361 ("%s: t-digest@%p has no free centroids",
3362 __func__, tdgst));
3363 #endif
3364 if (ctd64 == NULL) /* Should not happen. */
3365 return (EAGAIN);
3366 /* Direct assignment ok as both have same type/prec. */
3367 ctd64->mu = x;
3368 ctd64->cnt = weight;
3369 error = ARB_INSERT(ctdth64, ctd64tree, ctd64) == NULL ?
3370 0 : EALREADY;
3371 #ifdef DIAGNOSTIC
3372 RB_INSERT(rbctdth64, &VSD(tdgstclust64,
3373 tdgst)->rbctdtree, ctd64);
3374 #endif
3375 }
3376 }
3377
3378 if (is32bit)
3379 VSD(tdgstclust32, tdgst)->smplcnt += weight;
3380 else {
3381 VSD(tdgstclust64, tdgst)->smplcnt += weight;
3382
3383 #ifdef DIAGNOSTIC
3384 struct rbctdth64 *rbctdtree =
3385 &VSD(tdgstclust64, tdgst)->rbctdtree;
3386 struct voistatdata_tdgstctd64 *rbctd64;
3387 int i = 0;
3388 ARB_FOREACH(ctd64, ctdth64, ctd64tree) {
3389 rbctd64 = (i == 0 ? RB_MIN(rbctdth64, rbctdtree) :
3390 RB_NEXT(rbctdth64, rbctdtree, rbctd64));
3391
3392 if (i >= ARB_CURNODES(ctd64tree)
3393 || ctd64 != rbctd64
3394 || ARB_MIN(ctdth64, ctd64tree) !=
3395 RB_MIN(rbctdth64, rbctdtree)
3396 || ARB_MAX(ctdth64, ctd64tree) !=
3397 RB_MAX(rbctdth64, rbctdtree)
3398 || ARB_LEFTIDX(ctd64, ctdlnk) !=
3399 ARB_SELFIDX(ctd64tree, RB_LEFT(rbctd64, rblnk))
3400 || ARB_RIGHTIDX(ctd64, ctdlnk) !=
3401 ARB_SELFIDX(ctd64tree, RB_RIGHT(rbctd64, rblnk))
3402 || ARB_PARENTIDX(ctd64, ctdlnk) !=
3403 ARB_SELFIDX(ctd64tree,
3404 RB_PARENT(rbctd64, rblnk))) {
3405 Q_TOSTR(ctd64->mu, -1, 10, qstr, sizeof(qstr));
3406 printf("ARB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
3407 "mu=%s\n",
3408 (int)ARB_SELFIDX(ctd64tree, ctd64),
3409 ARB_PARENTIDX(ctd64, ctdlnk),
3410 ARB_LEFTIDX(ctd64, ctdlnk),
3411 ARB_RIGHTIDX(ctd64, ctdlnk),
3412 ARB_COLOR(ctd64, ctdlnk),
3413 qstr);
3414
3415 Q_TOSTR(rbctd64->mu, -1, 10, qstr,
3416 sizeof(qstr));
3417 struct voistatdata_tdgstctd64 *parent;
3418 parent = RB_PARENT(rbctd64, rblnk);
3419 int rb_color =
3420 parent == NULL ? 0 :
3421 RB_LEFT(parent, rblnk) == rbctd64 ?
3422 (_RB_BITSUP(parent, rblnk) & _RB_L) != 0 :
3423 (_RB_BITSUP(parent, rblnk) & _RB_R) != 0;
3424 printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
3425 "mu=%s\n",
3426 (int)ARB_SELFIDX(ctd64tree, rbctd64),
3427 (int)ARB_SELFIDX(ctd64tree,
3428 RB_PARENT(rbctd64, rblnk)),
3429 (int)ARB_SELFIDX(ctd64tree,
3430 RB_LEFT(rbctd64, rblnk)),
3431 (int)ARB_SELFIDX(ctd64tree,
3432 RB_RIGHT(rbctd64, rblnk)),
3433 rb_color,
3434 qstr);
3435
3436 panic("RB@%p and ARB@%p trees differ\n",
3437 rbctdtree, ctd64tree);
3438 }
3439 i++;
3440 }
3441 #endif /* DIAGNOSTIC */
3442 }
3443
3444 return (error);
3445 }
3446
3447 static inline int
3448 stats_v1_voi_update_tdgst(enum vsd_dtype voi_dtype, struct voistatdata *voival,
3449 struct voistat *vs, struct voistatdata_tdgst *tdgst)
3450 {
3451 s64q_t x;
3452 int error;
3453
3454 error = 0;
3455
3456 switch (vs->dtype) {
3457 case VSD_DTYPE_TDGSTCLUST32:
3458 /* Use same precision as the user's centroids. */
3459 Q_INI(&x, 0, 0, Q_NFBITS(
3460 ARB_CNODE(&VSD(tdgstclust32, tdgst)->ctdtree, 0)->mu));
3461 break;
3462 case VSD_DTYPE_TDGSTCLUST64:
3463 /* Use same precision as the user's centroids. */
3464 Q_INI(&x, 0, 0, Q_NFBITS(
3465 ARB_CNODE(&VSD(tdgstclust64, tdgst)->ctdtree, 0)->mu));
3466 break;
3467 default:
3468 KASSERT(vs->dtype == VSD_DTYPE_TDGSTCLUST32 ||
3469 vs->dtype == VSD_DTYPE_TDGSTCLUST64,
3470 ("%s: vs->dtype(%d) != VSD_DTYPE_TDGSTCLUST<32|64>",
3471 __func__, vs->dtype));
3472 return (EINVAL);
3473 }
3474
3475 /*
3476 * XXXLAS: Should have both a signed and unsigned 'x' variable to avoid
3477 * returning EOVERFLOW if the voival would have fit in a u64q_t.
3478 */
3479 switch (voi_dtype) {
3480 case VSD_DTYPE_INT_S32:
3481 error = Q_QCPYVALI(&x, voival->int32.s32);
3482 break;
3483 case VSD_DTYPE_INT_U32:
3484 error = Q_QCPYVALI(&x, voival->int32.u32);
3485 break;
3486 case VSD_DTYPE_INT_S64:
3487 error = Q_QCPYVALI(&x, voival->int64.s64);
3488 break;
3489 case VSD_DTYPE_INT_U64:
3490 error = Q_QCPYVALI(&x, voival->int64.u64);
3491 break;
3492 case VSD_DTYPE_INT_SLONG:
3493 error = Q_QCPYVALI(&x, voival->intlong.slong);
3494 break;
3495 case VSD_DTYPE_INT_ULONG:
3496 error = Q_QCPYVALI(&x, voival->intlong.ulong);
3497 break;
3498 case VSD_DTYPE_Q_S32:
3499 error = Q_QCPYVALQ(&x, voival->q32.sq32);
3500 break;
3501 case VSD_DTYPE_Q_U32:
3502 error = Q_QCPYVALQ(&x, voival->q32.uq32);
3503 break;
3504 case VSD_DTYPE_Q_S64:
3505 error = Q_QCPYVALQ(&x, voival->q64.sq64);
3506 break;
3507 case VSD_DTYPE_Q_U64:
3508 error = Q_QCPYVALQ(&x, voival->q64.uq64);
3509 break;
3510 default:
3511 error = EINVAL;
3512 break;
3513 }
3514
3515 if (error ||
3516 (error = stats_v1_vsd_tdgst_add(vs->dtype, tdgst, x, 1, 1)))
3517 return (error);
3518
3519 vs->flags |= VS_VSDVALID;
3520 return (0);
3521 }
3522
3523 int
3524 stats_v1_voi_update(struct statsblobv1 *sb, int32_t voi_id,
3525 enum vsd_dtype voi_dtype, struct voistatdata *voival, uint32_t flags)
3526 {
3527 struct voi *v;
3528 struct voistat *vs;
3529 void *statevsd, *vsd;
3530 int error, i, tmperr;
3531
3532 error = 0;
3533
3534 if (sb == NULL || sb->abi != STATS_ABI_V1 || voi_id >= NVOIS(sb) ||
3535 voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || voival == NULL)
3536 return (EINVAL);
3537 v = &sb->vois[voi_id];
3538 if (voi_dtype != v->dtype || v->id < 0 ||
3539 ((flags & SB_VOI_RELUPDATE) && !(v->flags & VOI_REQSTATE)))
3540 return (EINVAL);
3541
3542 vs = BLOB_OFFSET(sb, v->stats_off);
3543 if (v->flags & VOI_REQSTATE)
3544 statevsd = BLOB_OFFSET(sb, vs->data_off);
3545 else
3546 statevsd = NULL;
3547
3548 if (flags & SB_VOI_RELUPDATE) {
3549 switch (voi_dtype) {
3550 case VSD_DTYPE_INT_S32:
3551 voival->int32.s32 +=
3552 VSD(voistate, statevsd)->prev.int32.s32;
3553 break;
3554 case VSD_DTYPE_INT_U32:
3555 voival->int32.u32 +=
3556 VSD(voistate, statevsd)->prev.int32.u32;
3557 break;
3558 case VSD_DTYPE_INT_S64:
3559 voival->int64.s64 +=
3560 VSD(voistate, statevsd)->prev.int64.s64;
3561 break;
3562 case VSD_DTYPE_INT_U64:
3563 voival->int64.u64 +=
3564 VSD(voistate, statevsd)->prev.int64.u64;
3565 break;
3566 case VSD_DTYPE_INT_SLONG:
3567 voival->intlong.slong +=
3568 VSD(voistate, statevsd)->prev.intlong.slong;
3569 break;
3570 case VSD_DTYPE_INT_ULONG:
3571 voival->intlong.ulong +=
3572 VSD(voistate, statevsd)->prev.intlong.ulong;
3573 break;
3574 case VSD_DTYPE_Q_S32:
3575 error = Q_QADDQ(&voival->q32.sq32,
3576 VSD(voistate, statevsd)->prev.q32.sq32);
3577 break;
3578 case VSD_DTYPE_Q_U32:
3579 error = Q_QADDQ(&voival->q32.uq32,
3580 VSD(voistate, statevsd)->prev.q32.uq32);
3581 break;
3582 case VSD_DTYPE_Q_S64:
3583 error = Q_QADDQ(&voival->q64.sq64,
3584 VSD(voistate, statevsd)->prev.q64.sq64);
3585 break;
3586 case VSD_DTYPE_Q_U64:
3587 error = Q_QADDQ(&voival->q64.uq64,
3588 VSD(voistate, statevsd)->prev.q64.uq64);
3589 break;
3590 default:
3591 KASSERT(0, ("Unknown VOI data type %d", voi_dtype));
3592 break;
3593 }
3594 }
3595
3596 if (error)
3597 return (error);
3598
3599 for (i = v->voistatmaxid; i > 0; i--) {
3600 vs = &((struct voistat *)BLOB_OFFSET(sb, v->stats_off))[i];
3601 if (vs->stype < 0)
3602 continue;
3603
3604 vsd = BLOB_OFFSET(sb, vs->data_off);
3605
3606 switch (vs->stype) {
3607 case VS_STYPE_MAX:
3608 tmperr = stats_v1_voi_update_max(voi_dtype, voival,
3609 vs, vsd);
3610 break;
3611 case VS_STYPE_MIN:
3612 tmperr = stats_v1_voi_update_min(voi_dtype, voival,
3613 vs, vsd);
3614 break;
3615 case VS_STYPE_SUM:
3616 tmperr = stats_v1_voi_update_sum(voi_dtype, voival,
3617 vs, vsd);
3618 break;
3619 case VS_STYPE_HIST:
3620 tmperr = stats_v1_voi_update_hist(voi_dtype, voival,
3621 vs, vsd);
3622 break;
3623 case VS_STYPE_TDGST:
3624 tmperr = stats_v1_voi_update_tdgst(voi_dtype, voival,
3625 vs, vsd);
3626 break;
3627 default:
3628 KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
3629 break;
3630 }
3631
3632 if (tmperr) {
3633 error = tmperr;
3634 VS_INCERRS(vs);
3635 }
3636 }
3637
3638 if (statevsd) {
3639 switch (voi_dtype) {
3640 case VSD_DTYPE_INT_S32:
3641 VSD(voistate, statevsd)->prev.int32.s32 =
3642 voival->int32.s32;
3643 break;
3644 case VSD_DTYPE_INT_U32:
3645 VSD(voistate, statevsd)->prev.int32.u32 =
3646 voival->int32.u32;
3647 break;
3648 case VSD_DTYPE_INT_S64:
3649 VSD(voistate, statevsd)->prev.int64.s64 =
3650 voival->int64.s64;
3651 break;
3652 case VSD_DTYPE_INT_U64:
3653 VSD(voistate, statevsd)->prev.int64.u64 =
3654 voival->int64.u64;
3655 break;
3656 case VSD_DTYPE_INT_SLONG:
3657 VSD(voistate, statevsd)->prev.intlong.slong =
3658 voival->intlong.slong;
3659 break;
3660 case VSD_DTYPE_INT_ULONG:
3661 VSD(voistate, statevsd)->prev.intlong.ulong =
3662 voival->intlong.ulong;
3663 break;
3664 case VSD_DTYPE_Q_S32:
3665 error = Q_QCPYVALQ(
3666 &VSD(voistate, statevsd)->prev.q32.sq32,
3667 voival->q32.sq32);
3668 break;
3669 case VSD_DTYPE_Q_U32:
3670 error = Q_QCPYVALQ(
3671 &VSD(voistate, statevsd)->prev.q32.uq32,
3672 voival->q32.uq32);
3673 break;
3674 case VSD_DTYPE_Q_S64:
3675 error = Q_QCPYVALQ(
3676 &VSD(voistate, statevsd)->prev.q64.sq64,
3677 voival->q64.sq64);
3678 break;
3679 case VSD_DTYPE_Q_U64:
3680 error = Q_QCPYVALQ(
3681 &VSD(voistate, statevsd)->prev.q64.uq64,
3682 voival->q64.uq64);
3683 break;
3684 default:
3685 KASSERT(0, ("Unknown VOI data type %d", voi_dtype));
3686 break;
3687 }
3688 }
3689
3690 return (error);
3691 }
3692
3693 #ifdef _KERNEL
3694
3695 static void
3696 stats_init(void *arg)
3697 {
3698
3699 }
3700 SYSINIT(stats, SI_SUB_KDTRACE, SI_ORDER_FIRST, stats_init, NULL);
3701
3702 /*
3703 * Sysctl handler to display the list of available stats templates.
3704 */
3705 static int
3706 stats_tpl_list_available(SYSCTL_HANDLER_ARGS)
3707 {
3708 struct sbuf *s;
3709 int err, i;
3710
3711 err = 0;
3712
3713 /* We can tolerate ntpl being stale, so do not take the lock. */
3714 s = sbuf_new(NULL, NULL, /* +1 per tpl for , */
3715 ntpl * (STATS_TPL_MAX_STR_SPEC_LEN + 1), SBUF_FIXEDLEN);
3716 if (s == NULL)
3717 return (ENOMEM);
3718
3719 TPL_LIST_RLOCK();
3720 for (i = 0; i < ntpl; i++) {
3721 err = sbuf_printf(s, "%s\"%s\":%u", i ? "," : "",
3722 tpllist[i]->mb->tplname, tpllist[i]->mb->tplhash);
3723 if (err) {
3724 /* Sbuf overflow condition. */
3725 err = EOVERFLOW;
3726 break;
3727 }
3728 }
3729 TPL_LIST_RUNLOCK();
3730
3731 if (!err) {
3732 sbuf_finish(s);
3733 err = sysctl_handle_string(oidp, sbuf_data(s), 0, req);
3734 }
3735
3736 sbuf_delete(s);
3737 return (err);
3738 }
3739
3740 /*
3741 * Called by subsystem-specific sysctls to report and/or parse the list of
3742 * templates being sampled and their sampling rates. A stats_tpl_sr_cb_t
3743 * conformant function pointer must be passed in as arg1, which is used to
3744 * interact with the subsystem's stats template sample rates list. If arg2 > 0,
3745 * a zero-initialised allocation of arg2-sized contextual memory is
3746 * heap-allocated and passed in to all subsystem callbacks made during the
3747 * operation of stats_tpl_sample_rates().
3748 *
3749 * XXXLAS: Assumes templates are never removed, which is currently true but may
3750 * need to be reworked in future if dynamic template management becomes a
3751 * requirement e.g. to support kernel module based templates.
3752 */
3753 int
3754 stats_tpl_sample_rates(SYSCTL_HANDLER_ARGS)
3755 {
3756 char kvpair_fmt[16], tplspec_fmt[16];
3757 char tpl_spec[STATS_TPL_MAX_STR_SPEC_LEN];
3758 char tpl_name[TPL_MAX_NAME_LEN + 2]; /* +2 for "" */
3759 stats_tpl_sr_cb_t subsys_cb;
3760 void *subsys_ctx;
3761 char *buf, *new_rates_usr_str, *tpl_name_p;
3762 struct stats_tpl_sample_rate *rates;
3763 struct sbuf *s, _s;
3764 uint32_t cum_pct, pct, tpl_hash;
3765 int err, i, off, len, newlen, nrates;
3766
3767 buf = NULL;
3768 rates = NULL;
3769 err = nrates = 0;
3770 subsys_cb = (stats_tpl_sr_cb_t)arg1;
3771 KASSERT(subsys_cb != NULL, ("%s: subsys_cb == arg1 == NULL", __func__));
3772 if (arg2 > 0)
3773 subsys_ctx = malloc(arg2, M_TEMP, M_WAITOK | M_ZERO);
3774 else
3775 subsys_ctx = NULL;
3776
3777 /* Grab current count of subsystem rates. */
3778 err = subsys_cb(TPL_SR_UNLOCKED_GET, NULL, &nrates, subsys_ctx);
3779 if (err)
3780 goto done;
3781
3782 /* +1 to ensure we can append '\0' post copyin, +5 per rate for =nnn, */
3783 len = max(req->newlen + 1, nrates * (STATS_TPL_MAX_STR_SPEC_LEN + 5));
3784
3785 if (req->oldptr != NULL || req->newptr != NULL)
3786 buf = malloc(len, M_TEMP, M_WAITOK);
3787
3788 if (req->oldptr != NULL) {
3789 if (nrates == 0) {
3790 /* No rates, so return an empty string via oldptr. */
3791 err = SYSCTL_OUT(req, "", 1);
3792 if (err)
3793 goto done;
3794 goto process_new;
3795 }
3796
3797 s = sbuf_new(&_s, buf, len, SBUF_FIXEDLEN | SBUF_INCLUDENUL);
3798
3799 /* Grab locked count of, and ptr to, subsystem rates. */
3800 err = subsys_cb(TPL_SR_RLOCKED_GET, &rates, &nrates,
3801 subsys_ctx);
3802 if (err)
3803 goto done;
3804 TPL_LIST_RLOCK();
3805 for (i = 0; i < nrates && !err; i++) {
3806 err = sbuf_printf(s, "%s\"%s\":%u=%u", i ? "," : "",
3807 tpllist[rates[i].tpl_slot_id]->mb->tplname,
3808 tpllist[rates[i].tpl_slot_id]->mb->tplhash,
3809 rates[i].tpl_sample_pct);
3810 }
3811 TPL_LIST_RUNLOCK();
3812 /* Tell subsystem that we're done with its rates list. */
3813 err = subsys_cb(TPL_SR_RUNLOCK, &rates, &nrates, subsys_ctx);
3814 if (err)
3815 goto done;
3816
3817 err = sbuf_finish(s);
3818 if (err)
3819 goto done; /* We lost a race for buf to be too small. */
3820
3821 /* Return the rendered string data via oldptr. */
3822 err = SYSCTL_OUT(req, sbuf_data(s), sbuf_len(s));
3823 } else {
3824 /* Return the upper bound size for buffer sizing requests. */
3825 err = SYSCTL_OUT(req, NULL, len);
3826 }
3827
3828 process_new:
3829 if (err || req->newptr == NULL)
3830 goto done;
3831
3832 newlen = req->newlen - req->newidx;
3833 err = SYSCTL_IN(req, buf, newlen);
3834 if (err)
3835 goto done;
3836
3837 /*
3838 * Initialise format strings at run time.
3839 *
3840 * Write the max template spec string length into the
3841 * template_spec=percent key-value pair parsing format string as:
3842 * " %<width>[^=]=%u %n"
3843 *
3844 * Write the max template name string length into the tplname:tplhash
3845 * parsing format string as:
3846 * "%<width>[^:]:%u"
3847 *
3848 * Subtract 1 for \0 appended by sscanf().
3849 */
3850 sprintf(kvpair_fmt, " %%%zu[^=]=%%u %%n", sizeof(tpl_spec) - 1);
3851 sprintf(tplspec_fmt, "%%%zu[^:]:%%u", sizeof(tpl_name) - 1);
3852
3853 /*
3854 * Parse each CSV key-value pair specifying a template and its sample
3855 * percentage. Whitespace either side of a key-value pair is ignored.
3856 * Templates can be specified by name, hash, or name and hash per the
3857 * following formats (chars in [] are optional):
3858 * ["]<tplname>["]=<percent>
3859 * :hash=pct
3860 * ["]<tplname>["]:hash=<percent>
3861 */
3862 cum_pct = nrates = 0;
3863 rates = NULL;
3864 buf[newlen] = '\0'; /* buf is at least newlen+1 in size. */
3865 new_rates_usr_str = buf;
3866 while (isspace(*new_rates_usr_str))
3867 new_rates_usr_str++; /* Skip leading whitespace. */
3868 while (*new_rates_usr_str != '\0') {
3869 tpl_name_p = tpl_name;
3870 tpl_name[0] = '\0';
3871 tpl_hash = 0;
3872 off = 0;
3873
3874 /*
3875 * Parse key-value pair which must perform 2 conversions, then
3876 * parse the template spec to extract either name, hash, or name
3877 * and hash depending on the three possible spec formats. The
3878 * tplspec_fmt format specifier parses name or name and hash
3879 * template specs, while the ":%u" format specifier parses
3880 * hash-only template specs. If parsing is successfull, ensure
3881 * the cumulative sampling percentage does not exceed 100.
3882 */
3883 err = EINVAL;
3884 if (2 != sscanf(new_rates_usr_str, kvpair_fmt, tpl_spec, &pct,
3885 &off))
3886 break;
3887 if ((1 > sscanf(tpl_spec, tplspec_fmt, tpl_name, &tpl_hash)) &&
3888 (1 != sscanf(tpl_spec, ":%u", &tpl_hash)))
3889 break;
3890 if ((cum_pct += pct) > 100)
3891 break;
3892 err = 0;
3893
3894 /* Strip surrounding "" from template name if present. */
3895 len = strlen(tpl_name);
3896 if (len > 0) {
3897 if (tpl_name[len - 1] == '"')
3898 tpl_name[--len] = '\0';
3899 if (tpl_name[0] == '"') {
3900 tpl_name_p++;
3901 len--;
3902 }
3903 }
3904
3905 rates = stats_realloc(rates, 0, /* oldsz is unused in kernel. */
3906 (nrates + 1) * sizeof(*rates), M_WAITOK);
3907 rates[nrates].tpl_slot_id =
3908 stats_tpl_fetch_allocid(len ? tpl_name_p : NULL, tpl_hash);
3909 if (rates[nrates].tpl_slot_id < 0) {
3910 err = -rates[nrates].tpl_slot_id;
3911 break;
3912 }
3913 rates[nrates].tpl_sample_pct = pct;
3914 nrates++;
3915 new_rates_usr_str += off;
3916 if (*new_rates_usr_str != ',')
3917 break; /* End-of-input or malformed. */
3918 new_rates_usr_str++; /* Move past comma to next pair. */
3919 }
3920
3921 if (!err) {
3922 if ((new_rates_usr_str - buf) < newlen) {
3923 /* Entire input has not been consumed. */
3924 err = EINVAL;
3925 } else {
3926 /*
3927 * Give subsystem the new rates. They'll return the
3928 * appropriate rates pointer for us to garbage collect.
3929 */
3930 err = subsys_cb(TPL_SR_PUT, &rates, &nrates,
3931 subsys_ctx);
3932 }
3933 }
3934 stats_free(rates);
3935
3936 done:
3937 free(buf, M_TEMP);
3938 free(subsys_ctx, M_TEMP);
3939 return (err);
3940 }
3941
3942 SYSCTL_NODE(_kern, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
3943 "stats(9) MIB");
3944
3945 SYSCTL_PROC(_kern_stats, OID_AUTO, templates,
3946 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
3947 stats_tpl_list_available, "A",
3948 "list the name/hash of all available stats(9) templates");
3949
3950 #else /* ! _KERNEL */
3951
3952 static void __attribute__ ((constructor))
3953 stats_constructor(void)
3954 {
3955
3956 pthread_rwlock_init(&tpllistlock, NULL);
3957 }
3958
3959 static void __attribute__ ((destructor))
3960 stats_destructor(void)
3961 {
3962
3963 pthread_rwlock_destroy(&tpllistlock);
3964 }
3965
3966 #endif /* _KERNEL */
3967