1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2013 Gleb Smirnoff <glebius@FreeBSD.org>
5 * Copyright (c) 2010 Juniper Networks, Inc.
6 * Copyright (c) 2009 Robert N. M. Watson
7 * Copyright (c) 2009 Bjoern A. Zeeb <bz@FreeBSD.org>
8 * Copyright (c) 2008 Yahoo!, Inc.
9 * All rights reserved.
10 *
11 * Written by: John Baldwin <jhb@FreeBSD.org>
12 *
13 * This software was developed by Robert N. M. Watson under contract
14 * to Juniper Networks, Inc.
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 * 1. Redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution.
24 * 3. Neither the name of the author nor the names of any co-contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 */
40
41 #include <sys/cdefs.h>
42 #include <sys/param.h>
43 #include <sys/pcpu.h>
44 #include <sys/sysctl.h>
45 #include <kvm.h>
46 #include <limits.h>
47 #include <stdlib.h>
48
49 #include "kvm_private.h"
50
51 #ifdef __amd64__
52 #define __OFFSET_BY_PCPU
53 #endif
54
55 static struct nlist kvm_pcpu_nl[] = {
56 { .n_name = "_cpuid_to_pcpu" },
57 { .n_name = "_mp_maxcpus" },
58 { .n_name = "_mp_ncpus" },
59 #ifdef __OFFSET_BY_PCPU
60 { .n_name = "___pcpu" },
61 #endif
62 { .n_name = NULL },
63 };
64 #define NL_CPUID_TO_PCPU 0
65 #define NL_MP_MAXCPUS 1
66 #define NL_MP_NCPUS 2
67 #define NL___PCPU 3
68
69 /*
70 * Kernel per-CPU data state. We cache this stuff on the first
71 * access.
72 *
73 * XXXRW: Possibly, this (and kvmpcpu_nl) should be per-kvm_t, in case the
74 * consumer has multiple handles in flight to differently configured
75 * kernels/crashdumps.
76 */
77 static void **pcpu_data;
78 static int maxcpu;
79 static int mp_ncpus;
80 #ifdef __OFFSET_BY_PCPU
81 static unsigned long __pcpu;
82 #endif
83
84 static int
_kvm_pcpu_init(kvm_t * kd)85 _kvm_pcpu_init(kvm_t *kd)
86 {
87 size_t len;
88 int max;
89 void *data;
90
91 if (kvm_nlist(kd, kvm_pcpu_nl) < 0)
92 return (-1);
93 if (kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value == 0) {
94 _kvm_err(kd, kd->program, "unable to find cpuid_to_pcpu");
95 return (-1);
96 }
97 if (kvm_pcpu_nl[NL_MP_MAXCPUS].n_value == 0) {
98 _kvm_err(kd, kd->program, "unable to find mp_maxcpus");
99 return (-1);
100 }
101 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_MAXCPUS].n_value, &max,
102 sizeof(max)) != sizeof(max)) {
103 _kvm_err(kd, kd->program, "cannot read mp_maxcpus");
104 return (-1);
105 }
106 if (kvm_pcpu_nl[NL_MP_NCPUS].n_value == 0) {
107 _kvm_err(kd, kd->program, "unable to find mp_ncpus");
108 return (-1);
109 }
110 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_NCPUS].n_value, &mp_ncpus,
111 sizeof(mp_ncpus)) != sizeof(mp_ncpus)) {
112 _kvm_err(kd, kd->program, "cannot read mp_ncpus");
113 return (-1);
114 }
115 #ifdef __OFFSET_BY_PCPU
116 if (kvm_pcpu_nl[NL___PCPU].n_value == 0) {
117 _kvm_err(kd, kd->program, "unable to find __pcpu");
118 return (-1);
119 }
120 if (kvm_read(kd, kvm_pcpu_nl[NL___PCPU].n_value, &__pcpu,
121 sizeof(__pcpu)) != sizeof(__pcpu)) {
122 _kvm_err(kd, kd->program, "cannot read __pcpu");
123 return (-1);
124 }
125 #endif
126 len = max * sizeof(void *);
127 data = malloc(len);
128 if (data == NULL) {
129 _kvm_err(kd, kd->program, "out of memory");
130 return (-1);
131 }
132 if (kvm_read(kd, kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value, data, len) !=
133 (ssize_t)len) {
134 _kvm_err(kd, kd->program, "cannot read cpuid_to_pcpu array");
135 free(data);
136 return (-1);
137 }
138 pcpu_data = data;
139 maxcpu = max;
140 return (0);
141 }
142
143 static void
_kvm_pcpu_clear(void)144 _kvm_pcpu_clear(void)
145 {
146
147 maxcpu = 0;
148 free(pcpu_data);
149 pcpu_data = NULL;
150 }
151
152 void *
kvm_getpcpu(kvm_t * kd,int cpu)153 kvm_getpcpu(kvm_t *kd, int cpu)
154 {
155 char *buf;
156
157 if (kd == NULL) {
158 _kvm_pcpu_clear();
159 return (NULL);
160 }
161
162 if (maxcpu == 0)
163 if (_kvm_pcpu_init(kd) < 0)
164 return ((void *)-1);
165
166 if (cpu >= maxcpu || pcpu_data[cpu] == NULL)
167 return (NULL);
168
169 buf = malloc(sizeof(struct pcpu));
170 if (buf == NULL) {
171 _kvm_err(kd, kd->program, "out of memory");
172 return ((void *)-1);
173 }
174 if (kvm_read(kd, (uintptr_t)pcpu_data[cpu], buf,
175 sizeof(struct pcpu)) != sizeof(struct pcpu)) {
176 _kvm_err(kd, kd->program, "unable to read per-CPU data");
177 free(buf);
178 return ((void *)-1);
179 }
180 return (buf);
181 }
182
183 int
kvm_getmaxcpu(kvm_t * kd)184 kvm_getmaxcpu(kvm_t *kd)
185 {
186
187 if (kd == NULL) {
188 _kvm_pcpu_clear();
189 return (0);
190 }
191
192 if (maxcpu == 0)
193 if (_kvm_pcpu_init(kd) < 0)
194 return (-1);
195 return (maxcpu);
196 }
197
198 int
kvm_getncpus(kvm_t * kd)199 kvm_getncpus(kvm_t *kd)
200 {
201
202 if (mp_ncpus == 0)
203 if (_kvm_pcpu_init(kd) < 0)
204 return (-1);
205 return (mp_ncpus);
206 }
207
208 static int
_kvm_dpcpu_setcpu(kvm_t * kd,u_int cpu,int report_error)209 _kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu, int report_error)
210 {
211
212 if (!kd->dpcpu_initialized) {
213 if (report_error)
214 _kvm_err(kd, kd->program, "%s: not initialized",
215 __func__);
216 return (-1);
217 }
218 if (cpu >= kd->dpcpu_maxcpus) {
219 if (report_error)
220 _kvm_err(kd, kd->program, "%s: CPU %u too big",
221 __func__, cpu);
222 return (-1);
223 }
224 if (kd->dpcpu_off[cpu] == 0) {
225 if (report_error)
226 _kvm_err(kd, kd->program, "%s: CPU %u not found",
227 __func__, cpu);
228 return (-1);
229 }
230 kd->dpcpu_curcpu = cpu;
231 kd->dpcpu_curoff = kd->dpcpu_off[cpu];
232 return (0);
233 }
234
235 /*
236 * Set up libkvm to handle dynamic per-CPU memory.
237 */
238 static int
_kvm_dpcpu_init(kvm_t * kd)239 _kvm_dpcpu_init(kvm_t *kd)
240 {
241 struct kvm_nlist nl[] = {
242 #define NLIST_START_SET_PCPU 0
243 { .n_name = "___start_" DPCPU_SETNAME },
244 #define NLIST_STOP_SET_PCPU 1
245 { .n_name = "___stop_" DPCPU_SETNAME },
246 #define NLIST_DPCPU_OFF 2
247 { .n_name = "_dpcpu_off" },
248 #define NLIST_MP_MAXCPUS 3
249 { .n_name = "_mp_maxcpus" },
250 { .n_name = NULL },
251 };
252 uintptr_t *dpcpu_off_buf;
253 size_t len;
254 u_int dpcpu_maxcpus;
255
256 /*
257 * XXX: This only works for native kernels for now.
258 */
259 if (!kvm_native(kd))
260 return (-1);
261
262 /*
263 * Locate and cache locations of important symbols using the internal
264 * version of _kvm_nlist, turning off initialization to avoid
265 * recursion in case of unresolveable symbols.
266 */
267 if (_kvm_nlist(kd, nl, 0) != 0)
268 return (-1);
269 if (kvm_read(kd, nl[NLIST_MP_MAXCPUS].n_value, &dpcpu_maxcpus,
270 sizeof(dpcpu_maxcpus)) != sizeof(dpcpu_maxcpus))
271 return (-1);
272 len = dpcpu_maxcpus * sizeof(*dpcpu_off_buf);
273 dpcpu_off_buf = malloc(len);
274 if (dpcpu_off_buf == NULL)
275 return (-1);
276 if (kvm_read(kd, nl[NLIST_DPCPU_OFF].n_value, dpcpu_off_buf, len) !=
277 (ssize_t)len) {
278 free(dpcpu_off_buf);
279 return (-1);
280 }
281 kd->dpcpu_start = nl[NLIST_START_SET_PCPU].n_value;
282 kd->dpcpu_stop = nl[NLIST_STOP_SET_PCPU].n_value;
283 kd->dpcpu_maxcpus = dpcpu_maxcpus;
284 kd->dpcpu_off = dpcpu_off_buf;
285 kd->dpcpu_initialized = 1;
286 (void)_kvm_dpcpu_setcpu(kd, 0, 0);
287 return (0);
288 }
289
290 /*
291 * Check whether the dpcpu module has been initialized successfully or not,
292 * initialize it if permitted.
293 */
294 int
_kvm_dpcpu_initialized(kvm_t * kd,int intialize)295 _kvm_dpcpu_initialized(kvm_t *kd, int intialize)
296 {
297
298 if (kd->dpcpu_initialized || !intialize)
299 return (kd->dpcpu_initialized);
300
301 (void)_kvm_dpcpu_init(kd);
302
303 return (kd->dpcpu_initialized);
304 }
305
306 /*
307 * Check whether the value is within the dpcpu symbol range and only if so
308 * adjust the offset relative to the current offset.
309 */
310 kvaddr_t
_kvm_dpcpu_validaddr(kvm_t * kd,kvaddr_t value)311 _kvm_dpcpu_validaddr(kvm_t *kd, kvaddr_t value)
312 {
313
314 if (value == 0)
315 return (value);
316
317 if (!kd->dpcpu_initialized)
318 return (value);
319
320 if (value < kd->dpcpu_start || value >= kd->dpcpu_stop)
321 return (value);
322
323 return (kd->dpcpu_curoff + value);
324 }
325
326 int
kvm_dpcpu_setcpu(kvm_t * kd,u_int cpu)327 kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu)
328 {
329 int ret;
330
331 if (!kd->dpcpu_initialized) {
332 ret = _kvm_dpcpu_init(kd);
333 if (ret != 0) {
334 _kvm_err(kd, kd->program, "%s: init failed",
335 __func__);
336 return (ret);
337 }
338 }
339
340 return (_kvm_dpcpu_setcpu(kd, cpu, 1));
341 }
342
343 /*
344 * Obtain a per-CPU copy for given cpu from UMA_ZONE_PCPU allocation.
345 */
346 ssize_t
kvm_read_zpcpu(kvm_t * kd,u_long base,void * buf,size_t size,int cpu)347 kvm_read_zpcpu(kvm_t *kd, u_long base, void *buf, size_t size, int cpu)
348 {
349
350 if (!kvm_native(kd))
351 return (-1);
352 if (mp_ncpus == 0)
353 if (_kvm_pcpu_init(kd) < 0)
354 return (0);
355
356 #ifdef __OFFSET_BY_PCPU
357 base += __pcpu;
358 #endif
359 return (kvm_read(kd, (uintptr_t)(base + sizeof(struct pcpu) * cpu),
360 buf, size));
361 }
362
363 /*
364 * Fetch value of a counter(9).
365 */
366 uint64_t
kvm_counter_u64_fetch(kvm_t * kd,u_long base)367 kvm_counter_u64_fetch(kvm_t *kd, u_long base)
368 {
369 uint64_t r, c;
370
371 if (mp_ncpus == 0)
372 if (_kvm_pcpu_init(kd) < 0)
373 return (0);
374
375 r = 0;
376 for (int i = 0; i < mp_ncpus; i++) {
377 if (kvm_read_zpcpu(kd, base, &c, sizeof(c), i) != sizeof(c))
378 return (0);
379 r += c;
380 }
381
382 return (r);
383 }
384