1 /* 2 * Copyright (c) 2008-2016 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #ifndef _CPU_CPUMASK_H_ 36 #define _CPU_CPUMASK_H_ 37 38 #include <machine/stdint.h> 39 #ifdef _KERNEL 40 #include <cpu/atomic.h> 41 #endif 42 43 /* 44 * cpumask_t - a mask representing a set of cpus and supporting routines. 45 * 46 * WARNING! It is recommended that this mask NOT be made variably-sized 47 * because it affects a huge number of system structures. However, 48 * kernel code (non-module) can be optimized to not operate on the 49 * whole mask. 50 */ 51 52 typedef struct { 53 __uint64_t ary[4]; 54 } __cpumask_t; 55 56 #define CPUMASK_ELEMENTS 4 57 58 #define CPUMASK_INITIALIZER_ALLONES { .ary = { (__uint64_t)-1, \ 59 (__uint64_t)-1, \ 60 (__uint64_t)-1, \ 61 (__uint64_t)-1 } } 62 #define CPUMASK_INITIALIZER_ONLYONE { .ary = { 1, 0, 0, 0 } } 63 64 #define CPUMASK_SIMPLE(cpu) ((__uint64_t)1 << (cpu)) 65 66 #define CPUMASK_ADDR(mask, cpu) (&(mask).ary[((cpu) >> 6) & 3]) 67 68 #define BSRCPUMASK(val) ((val).ary[3] ? 192 + bsrq((val).ary[3]) : \ 69 ((val).ary[2] ? 128 + bsrq((val).ary[2]) : \ 70 ((val).ary[1] ? 64 + bsrq((val).ary[1]) : \ 71 bsrq((val).ary[0])))) 72 73 #define BSFCPUMASK(val) ((val).ary[0] ? bsfq((val).ary[0]) : \ 74 ((val).ary[1] ? 64 + bsfq((val).ary[1]) : \ 75 ((val).ary[2] ? 128 + bsfq((val).ary[2]) : \ 76 192 + bsfq((val).ary[3])))) 77 78 #define CPUMASK_CMPMASKEQ(val1, val2) ((val1).ary[0] == (val2).ary[0] && \ 79 (val1).ary[1] == (val2).ary[1] && \ 80 (val1).ary[2] == (val2).ary[2] && \ 81 (val1).ary[3] == (val2).ary[3]) 82 83 #define CPUMASK_CMPMASKNEQ(val1, val2) ((val1).ary[0] != (val2).ary[0] || \ 84 (val1).ary[1] != (val2).ary[1] || \ 85 (val1).ary[2] != (val2).ary[2] || \ 86 (val1).ary[3] != (val2).ary[3]) 87 88 #define CPUMASK_ISUP(val) ((val).ary[0] == 1 && \ 89 (val).ary[1] == 0 && \ 90 (val).ary[2] == 0 && \ 91 (val).ary[3] == 0) 92 93 #define CPUMASK_TESTZERO(val) ((val).ary[0] == 0 && \ 94 (val).ary[1] == 0 && \ 95 (val).ary[2] == 0 && \ 96 (val).ary[3] == 0) 97 98 #define CPUMASK_TESTNZERO(val) ((val).ary[0] != 0 || \ 99 (val).ary[1] != 0 || \ 100 (val).ary[2] != 0 || \ 101 (val).ary[3] != 0) 102 103 #define CPUMASK_TESTBIT(val, i) ((val).ary[((i) >> 6) & 3] & \ 104 CPUMASK_SIMPLE((i) & 63)) 105 106 #define CPUMASK_TESTMASK(val1, val2) (((val1).ary[0] & (val2.ary[0])) || \ 107 ((val1).ary[1] & (val2.ary[1])) || \ 108 ((val1).ary[2] & (val2.ary[2])) || \ 109 ((val1).ary[3] & (val2.ary[3]))) 110 111 #define CPUMASK_LOWMASK(val) ((val).ary[0]) 112 113 #define CPUMASK_ORBIT(mask, i) ((mask).ary[((i) >> 6) & 3] |= \ 114 CPUMASK_SIMPLE((i) & 63)) 115 116 #define CPUMASK_ANDBIT(mask, i) ((mask).ary[((i) >> 6) & 3] &= \ 117 CPUMASK_SIMPLE((i) & 63)) 118 119 #define CPUMASK_NANDBIT(mask, i) ((mask).ary[((i) >> 6) & 3] &= \ 120 ~CPUMASK_SIMPLE((i) & 63)) 121 122 #define CPUMASK_ASSZERO(mask) do { \ 123 (mask).ary[0] = 0; \ 124 (mask).ary[1] = 0; \ 125 (mask).ary[2] = 0; \ 126 (mask).ary[3] = 0; \ 127 } while(0) 128 129 #define CPUMASK_ASSALLONES(mask) do { \ 130 (mask).ary[0] = (__uint64_t)-1; \ 131 (mask).ary[1] = (__uint64_t)-1; \ 132 (mask).ary[2] = (__uint64_t)-1; \ 133 (mask).ary[3] = (__uint64_t)-1; \ 134 } while(0) 135 136 #define CPUMASK_ASSBIT(mask, i) do { \ 137 CPUMASK_ASSZERO(mask); \ 138 CPUMASK_ORBIT(mask, i); \ 139 } while(0) 140 141 #define CPUMASK_ASSBMASK(mask, i) do { \ 142 if (i < 64) { \ 143 (mask).ary[0] = CPUMASK_SIMPLE(i) - 1; \ 144 (mask).ary[1] = 0; \ 145 (mask).ary[2] = 0; \ 146 (mask).ary[3] = 0; \ 147 } else if (i < 128) { \ 148 (mask).ary[0] = (__uint64_t)-1; \ 149 (mask).ary[1] = CPUMASK_SIMPLE((i) - 64) - 1; \ 150 (mask).ary[2] = 0; \ 151 (mask).ary[3] = 0; \ 152 } else if (i < 192) { \ 153 (mask).ary[0] = (__uint64_t)-1; \ 154 (mask).ary[1] = (__uint64_t)-1; \ 155 (mask).ary[2] = CPUMASK_SIMPLE((i) - 128) - 1; \ 156 (mask).ary[3] = 0; \ 157 } else { \ 158 (mask).ary[0] = (__uint64_t)-1; \ 159 (mask).ary[1] = (__uint64_t)-1; \ 160 (mask).ary[2] = (__uint64_t)-1; \ 161 (mask).ary[3] = CPUMASK_SIMPLE((i) - 192) - 1; \ 162 } \ 163 } while(0) 164 165 #define CPUMASK_ASSNBMASK(mask, i) do { \ 166 if (i < 64) { \ 167 (mask).ary[0] = ~(CPUMASK_SIMPLE(i) - 1); \ 168 (mask).ary[1] = (__uint64_t)-1; \ 169 (mask).ary[2] = (__uint64_t)-1; \ 170 (mask).ary[3] = (__uint64_t)-1; \ 171 } else if (i < 128) { \ 172 (mask).ary[0] = 0; \ 173 (mask).ary[1] = ~(CPUMASK_SIMPLE((i) - 64) - 1);\ 174 (mask).ary[2] = (__uint64_t)-1; \ 175 (mask).ary[3] = (__uint64_t)-1; \ 176 } else if (i < 192) { \ 177 (mask).ary[0] = 0; \ 178 (mask).ary[1] = 0; \ 179 (mask).ary[2] = ~(CPUMASK_SIMPLE((i) - 128) - 1);\ 180 (mask).ary[3] = (__uint64_t)-1; \ 181 } else { \ 182 (mask).ary[0] = 0; \ 183 (mask).ary[1] = 0; \ 184 (mask).ary[2] = 0; \ 185 (mask).ary[3] = ~(CPUMASK_SIMPLE((i) - 192) - 1);\ 186 } \ 187 } while(0) 188 189 #define CPUMASK_ANDMASK(mask, val) do { \ 190 (mask).ary[0] &= (val).ary[0]; \ 191 (mask).ary[1] &= (val).ary[1]; \ 192 (mask).ary[2] &= (val).ary[2]; \ 193 (mask).ary[3] &= (val).ary[3]; \ 194 } while(0) 195 196 #define CPUMASK_NANDMASK(mask, val) do { \ 197 (mask).ary[0] &= ~(val).ary[0]; \ 198 (mask).ary[1] &= ~(val).ary[1]; \ 199 (mask).ary[2] &= ~(val).ary[2]; \ 200 (mask).ary[3] &= ~(val).ary[3]; \ 201 } while(0) 202 203 #define CPUMASK_ORMASK(mask, val) do { \ 204 (mask).ary[0] |= (val).ary[0]; \ 205 (mask).ary[1] |= (val).ary[1]; \ 206 (mask).ary[2] |= (val).ary[2]; \ 207 (mask).ary[3] |= (val).ary[3]; \ 208 } while(0) 209 210 #define CPUMASK_XORMASK(mask, val) do { \ 211 (mask).ary[0] ^= (val).ary[0]; \ 212 (mask).ary[1] ^= (val).ary[1]; \ 213 (mask).ary[2] ^= (val).ary[2]; \ 214 (mask).ary[3] ^= (val).ary[3]; \ 215 } while(0) 216 217 #define CPUMASK_INVMASK(mask) do { \ 218 (mask).ary[0] ^= -1L; \ 219 (mask).ary[1] ^= -1L; \ 220 (mask).ary[2] ^= -1L; \ 221 (mask).ary[3] ^= -1L; \ 222 } while(0) 223 224 #ifndef _KERNEL 225 #define __CPU_SETSIZE ((int)(sizeof(cpumask_t) * 8)) 226 227 #define __CPU_COUNT(set) ( \ 228 __builtin_popcountl((set)->ary[0]) + \ 229 __builtin_popcountl((set)->ary[1]) + \ 230 __builtin_popcountl((set)->ary[2]) + \ 231 __builtin_popcountl((set)->ary[3])) 232 233 #define __CPU_CLR(cpu, set) CPUMASK_NANDBIT(*set, cpu) 234 #define __CPU_EQUAL(set1, set2) CPUMASK_CMPMASKEQ(*set1, *set2) 235 #define __CPU_ISSET(cpu, set) CPUMASK_TESTBIT(*set, cpu) 236 #define __CPU_SET(cpu, set) CPUMASK_ORBIT(*set, cpu) 237 #define __CPU_ZERO(set) CPUMASK_ASSZERO(*set) 238 239 #define __CPU_AND(dst, set1, set2) \ 240 do { \ 241 if (dst == set1) { \ 242 CPUMASK_ANDMASK(*dst, *set2); \ 243 } else { \ 244 *dst = *set2; \ 245 CPUMASK_ANDMASK(*dst, *set1); \ 246 } \ 247 } while (0) 248 249 #define __CPU_OR(dst, set1, set2) \ 250 do { \ 251 if (dst == set1) { \ 252 CPUMASK_ORMASK(*dst, *set2); \ 253 } else { \ 254 *dst = *set2; \ 255 CPUMASK_ORMASK(*dst, *set1); \ 256 } \ 257 } while (0) 258 259 #define __CPU_XOR(dst, set1, set2) \ 260 do { \ 261 if (dst == set1) { \ 262 CPUMASK_XORMASK(*dst, *set2); \ 263 } else { \ 264 *dst = *set2; \ 265 CPUMASK_XORMASK(*dst, *set1); \ 266 } \ 267 } while (0) 268 #endif 269 270 #ifdef _KERNEL 271 #define ATOMIC_CPUMASK_ORBIT(mask, i) \ 272 atomic_set_cpumask(&(mask).ary[((i) >> 6) & 3], \ 273 CPUMASK_SIMPLE((i) & 63)) 274 275 #define ATOMIC_CPUMASK_NANDBIT(mask, i) \ 276 atomic_clear_cpumask(&(mask).ary[((i) >> 6) & 3], \ 277 CPUMASK_SIMPLE((i) & 63)) 278 279 #define ATOMIC_CPUMASK_TESTANDSET(mask, i) \ 280 atomic_testandset_long(&(mask).ary[((i) >> 6) & 3], (i)) 281 282 #define ATOMIC_CPUMASK_TESTANDCLR(mask, i) \ 283 atomic_testandclear_long(&(mask).ary[((i) >> 6) & 3], (i)) 284 285 #define ATOMIC_CPUMASK_ORMASK(mask, val) do { \ 286 atomic_set_cpumask(&(mask).ary[0], (val).ary[0]); \ 287 atomic_set_cpumask(&(mask).ary[1], (val).ary[1]); \ 288 atomic_set_cpumask(&(mask).ary[2], (val).ary[2]); \ 289 atomic_set_cpumask(&(mask).ary[3], (val).ary[3]); \ 290 } while(0) 291 292 #define ATOMIC_CPUMASK_NANDMASK(mask, val) do { \ 293 atomic_clear_cpumask(&(mask).ary[0], (val).ary[0]); \ 294 atomic_clear_cpumask(&(mask).ary[1], (val).ary[1]); \ 295 atomic_clear_cpumask(&(mask).ary[2], (val).ary[2]); \ 296 atomic_clear_cpumask(&(mask).ary[3], (val).ary[3]); \ 297 } while(0) 298 299 #define ATOMIC_CPUMASK_COPY(mask, val) do { \ 300 atomic_store_rel_cpumask(&(mask).ary[0], (val).ary[0]);\ 301 atomic_store_rel_cpumask(&(mask).ary[1], (val).ary[1]);\ 302 atomic_store_rel_cpumask(&(mask).ary[2], (val).ary[2]);\ 303 atomic_store_rel_cpumask(&(mask).ary[3], (val).ary[3]);\ 304 } while(0) 305 #endif 306 307 #endif /* !_CPU_CPUMASK_H_ */ 308