1/* $NetBSD: pio_subr.S,v 1.18 2020/07/06 10:31:24 rin Exp $ */ 2 3/* 4 * Copyright (c) 2003 Matt Thomas 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 */ 30 31#ifdef _KERNEL_OPT 32#include "opt_ppcarch.h" 33#endif 34 35/* 36 * Assembly note: 37 * We use rotlw instead of slw because rotlw ignores bit 26 and slw 38 * doesn't. However, this may make the high bits of the offset rotate 39 * in the low bits but if that happens then the offset was too large 40 * to being with. 41 */ 42#ifdef DEBUG 43#define DBGSYNC sync 44#else 45#define DBGSYNC /* nothing */ 46#endif 47 48#undef DBGSYNC 49#if defined(PPC_IBM4XX) && !defined(PPC_IBM440) 50#define DBGSYNC /* nothing; eieio is implemented as sync */ 51#else 52#define DBGSYNC msync 53#endif 54#define eieio mbar 0 55 56/* LINTSTUB: include <sys/param.h> */ 57/* LINTSTUB: include <sys/types.h> */ 58/* LINTSTUB: include <sys/bus.h> */ 59 60/* LINTSTUB: Func: void out8(volatile u_int8_t *a, u_int8_t v) */ 61 62ENTRY(out8) 63 stbx %r4,0,%r3 64 eieio /* memory barrier (reorder protection) */ 65 DBGSYNC /* force exceptions */ 66 blr /* return */ 67 68/* LINTSTUB: Func: void bsw1_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int8_t v) */ 69/* LINTSTUB: Func: void bsw1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int8_t v) */ 70 71ENTRY(bsw1_s) 72 lwz %r0,0(%r3) /* get log2(stride) */ 73 rotlw %r5,%r5,%r0 /* shift offset */ 74ENTRY(bsw1) 75 stbx %r6,%r4,%r5 /* store value */ 76 eieio /* memory barrier (reorder protection) */ 77 DBGSYNC /* force exceptions */ 78 blr /* return */ 79 80/* LINTSTUB: Func: void out16(volatile u_int16_t *a, u_int16_t v) */ 81 82ENTRY(out16) 83 sth %r4,0(%r3) 84 eieio /* memory barrier (reorder protection) */ 85 DBGSYNC /* force exceptions */ 86 blr /* return */ 87 88/* LINTSTUB: Func: void bsw2_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int16_t v) */ 89/* LINTSTUB: Func: void bsw2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int16_t v) */ 90 91ENTRY(bsw2_s) 92 lwz %r0,0(%r3) /* get log2(stride) */ 93 rotlw %r5,%r5,%r0 /* shift offset */ 94ENTRY(bsw2) 95 sthx %r6,%r4,%r5 /* store value */ 96 eieio /* memory barrier (reorder protection) */ 97 DBGSYNC /* force exceptions */ 98 blr /* return */ 99 100/* LINTSTUB: Func: void out32(volatile u_int32_t *a, u_int32_t v) */ 101 102ENTRY(out32) 103 stw %r4,0(%r3) 104 eieio /* memory barrier (reorder protection) */ 105 DBGSYNC /* force exceptions */ 106 blr /* return */ 107 108/* LINTSTUB: Func: void bsw4_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int32_t v) */ 109/* LINTSTUB: Func: void bsw4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int32_t v) */ 110 111ENTRY(bsw4_s) 112 lwz %r0,0(%r3) /* get log2(stride) */ 113 rotlw %r5,%r5,%r0 /* shift offset */ 114ENTRY(bsw4) 115 stwx %r6,%r4,%r5 /* store value */ 116 eieio /* memory barrier (reorder protection) */ 117 DBGSYNC /* force exceptions */ 118 blr /* return */ 119 120/* LINTSTUB: Func: void bsw8_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int64_t v) */ 121/* LINTSTUB: Func: void bsw8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int64_t v) */ 122ENTRY(bsw8_s) 123 lwz %r0,0(%r3) /* get log2(stride) */ 124 rotlw %r5,%r5,%r0 /* shift offset */ 125ENTRY(bsw8) 126#ifdef __ARCH64__ 127 stdx %r6,%r4,%r5 /* store value */ 128#else 129 trap /* die */ 130#endif 131 eieio /* memory barrier (reorder protection) */ 132 DBGSYNC /* force exceptions */ 133 blr /* return */ 134 135/* LINTSTUB: Func: void out16rb(volatile u_int16_t *a, u_int16_t v) */ 136 137ENTRY(out16rb) 138 sthbrx %r4,0,%r3 139 eieio /* memory barrier (reorder protection) */ 140 DBGSYNC /* force exceptions */ 141 blr /* return */ 142 143/* LINTSTUB: Func: void bsw2rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int16_t v) */ 144/* LINTSTUB: Func: void bsw2rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int16_t v) */ 145 146 147ENTRY(bsw2rb_s) 148 lwz %r0,0(%r3) /* get log2(stride) */ 149 rotlw %r5,%r5,%r0 /* shift offset */ 150ENTRY(bsw2rb) 151 sthbrx %r6,%r4,%r5 /* store value */ 152 eieio /* memory barrier (reorder protection) */ 153 DBGSYNC /* force exceptions */ 154 blr /* return */ 155 156/* LINTSTUB: Func: void out32rb(volatile u_int32_t *a, u_int32_t v) */ 157 158ENTRY(out32rb) 159 stwbrx %r4,0,%r3 160 eieio /* memory barrier (reorder protection) */ 161 DBGSYNC /* force exceptions */ 162 blr /* return */ 163 164/* LINTSTUB: Func: void bsw4rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int32_t v) */ 165/* LINTSTUB: Func: void bsw4rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int32_t v) */ 166 167ENTRY(bsw4rb_s) 168 lwz %r0,0(%r3) /* get log2(stride) */ 169 rotlw %r5,%r5,%r0 /* shift offset */ 170ENTRY(bsw4rb) 171 stwbrx %r6,%r4,%r5 /* store value */ 172 eieio /* memory barrier (reorder protection) */ 173 DBGSYNC /* force exceptions */ 174 blr /* return */ 175 176/* LINTSTUB: Func: void bsw8rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int64_t v) */ 177/* LINTSTUB: Func: void bsw8rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int64_t v) */ 178 179ENTRY(bsw8rb_s) 180 lwz %r0,0(%r3) /* get log2(stride) */ 181 rotlw %r5,%r5,%r0 /* shift offset */ 182ENTRY(bsw8rb) 183#ifdef __ARCH64__ 184 stdbrx %r6,%r4,%r5 /* store value */ 185#else 186 trap /* die */ 187#endif 188 eieio /* memory barrier (reorder protection) */ 189 DBGSYNC /* force exceptions */ 190 blr /* return */ 191 192/* LINTSTUB: Func: int in8(const volatile u_int8_t *a) */ 193 194ENTRY(in8) 195 lbz %r3,0(%r3) 196 eieio /* memory barrier (reorder protection) */ 197 DBGSYNC /* force exceptions */ 198 blr /* return */ 199 200/* LINTSTUB: Func: int bsr1_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 201/* LINTSTUB: Func: int bsr1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 202 203ENTRY(bsr1_s) 204 lwz %r0,0(%r3) /* get log2(stride) */ 205 rotlw %r5,%r5,%r0 /* shift offset */ 206ENTRY(bsr1) 207 lbzx %r3,%r4,%r5 /* load value */ 208 eieio /* memory barrier (reorder protection) */ 209 DBGSYNC /* force exceptions */ 210 blr /* return */ 211 212/* LINTSTUB: Func: int in16(const volatile u_int16_t *a) */ 213 214ENTRY(in16) 215 lhz %r3,0(%r3) 216 eieio /* memory barrier (reorder protection) */ 217 DBGSYNC /* force exceptions */ 218 blr /* return */ 219 220/* LINTSTUB: Func: int bsr2_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 221/* LINTSTUB: Func: int bsr2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 222 223ENTRY(bsr2_s) 224 lwz %r0,0(%r3) /* get log2(stride) */ 225 rotlw %r5,%r5,%r0 /* shift offset */ 226ENTRY(bsr2) 227 lhzx %r3,%r4,%r5 /* load value */ 228 eieio /* memory barrier (reorder protection) */ 229 DBGSYNC /* force exceptions */ 230 blr /* return */ 231 232/* LINTSTUB: Func: int in32(const volatile u_int32_t *a) */ 233 234ENTRY(in32) 235 lwz %r3,0(%r3) 236 eieio /* memory barrier (reorder protection) */ 237 DBGSYNC /* force exceptions */ 238 blr /* return */ 239 240/* LINTSTUB: Func: int bsr4_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 241/* LINTSTUB: Func: int bsr4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 242 243ENTRY(bsr4_s) 244 lwz %r0,0(%r3) /* get log2(stride) */ 245 rotlw %r5,%r5,%r0 /* shift offset */ 246ENTRY(bsr4) 247 lwzx %r3,%r4,%r5 /* load value */ 248 eieio /* memory barrier (reorder protection) */ 249 DBGSYNC /* force exceptions */ 250 blr /* return */ 251 252/* LINTSTUB: Func: u_int64_t bsr8_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 253/* LINTSTUB: Func: u_int64_t bsr8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 254 255ENTRY(bsr8_s) 256 lwz %r0,0(%r3) /* get log2(stride) */ 257 rotlw %r5,%r5,%r0 /* shift offset */ 258ENTRY(bsr8) 259#ifdef __ARCH64__ 260 lwdx %r3,%r4,%r5 /* load value */ 261#else 262 trap 263#endif 264 eieio /* memory barrier (reorder protection) */ 265 DBGSYNC /* force exceptions */ 266 blr /* return */ 267 268/* LINTSTUB: Func: int in16rb(const volatile u_int16_t *a) */ 269 270ENTRY(in16rb) 271 lhbrx %r3,0,%r3 272 eieio /* memory barrier (reorder protection) */ 273 DBGSYNC /* force exceptions */ 274 blr /* return */ 275 276/* LINTSTUB: Func: int bsr2rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 277/* LINTSTUB: Func: int bsr2rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 278 279ENTRY(bsr2rb_s) 280 lwz %r0,0(%r3) /* get log2(stride) */ 281 rotlw %r5,%r5,%r0 /* shift offset */ 282ENTRY(bsr2rb) 283 lhbrx %r3,%r4,%r5 /* load value */ 284 eieio /* memory barrier (reorder protection) */ 285 DBGSYNC /* force exceptions */ 286 blr /* return */ 287 288/* LINTSTUB: Func: int in32rb(const volatile u_int32_t *a) */ 289 290ENTRY(in32rb) 291 lwbrx %r3,0,%r3 292 eieio /* memory barrier (reorder protection) */ 293 DBGSYNC /* force exceptions */ 294 blr /* return */ 295 296/* LINTSTUB: Func: int bsr4rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 297/* LINTSTUB: Func: int bsr4rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 298 299ENTRY(bsr4rb_s) 300 lwz %r0,0(%r3) /* get log2(stride) */ 301 rotlw %r5,%r5,%r0 /* shift offset */ 302ENTRY(bsr4rb) 303 lwbrx %r3,%r4,%r5 /* load value */ 304 eieio /* memory barrier (reorder protection) */ 305 DBGSYNC /* force exceptions */ 306 blr /* return */ 307 308/* LINTSTUB: Func: u_int64_t bsr8rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 309/* LINTSTUB: Func: u_int64_t bsr8rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o) */ 310 311ENTRY(bsr8rb_s) 312 lwz %r0,0(%r3) /* get log2(stride) */ 313 rotlw %r5,%r5,%r0 /* shift offset */ 314ENTRY(bsr8rb) 315#ifdef __ARCH64__ 316 ldbrx %r3,%r4,%r5 /* load value */ 317#else 318 trap 319#endif 320 eieio /* memory barrier (reorder protection) */ 321 DBGSYNC /* force exceptions */ 322 blr /* return */ 323 324/* LINTSTUB: Func: void bswm1_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int8_t *addr, bus_size_t len) */ 325/* LINTSTUB: Func: void bswm1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int8_t *addr, bus_size_t len) */ 326/* LINTSTUB: Func: void outs8(volatile u_int8_t *dst, const u_int8_t *src, size_t len) */ 327 328ENTRY(bswm1_s) 329 lwz %r0,0(%r3) /* get log2(stride) */ 330 rotlw %r5,%r5,%r0 /* shift offset */ 331ENTRY(bswm1) 332 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 333 mr %r4,%r6 /* move addr to argument 1 register */ 334 mr %r5,%r7 /* move count to argument 2 register */ 335ENTRY(outs8) 336 cmpwi %r5,0 /* len == 0? */ 337 beqlr- /* return if len == 0 */ 338 addi %r5,%r5,-1 /* len -= 1 */ 339 add %r5,%r5,%r4 /* len += src */ 340 addi %r4,%r4,-1 /* pre-decrement */ 3411: lbzu %r0,1(%r4) /* load and increment */ 342 stb %r0,0(%r3) /* store */ 343 cmplw %r4,%r5 /* at the end? */ 344 bne+ 1b /* nope, do another pass */ 345 eieio /* memory barrier (reorder protection) */ 346 DBGSYNC /* force exceptions */ 347 blr /* return */ 348 349/* LINTSTUB: Func: void bswm2_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int16_t *addr, bus_size_t len) */ 350/* LINTSTUB: Func: void bswm2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int16_t *addr, bus_size_t len) */ 351/* LINTSTUB: Func: void outs16(volatile u_int16_t *dst, const u_int16_t *src, size_t len) */ 352 353ENTRY(bswm2_s) 354 lwz %r0,0(%r3) /* get log2(stride) */ 355 rotlw %r5,%r5,%r0 /* shift offset */ 356ENTRY(bswm2) 357 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 358 mr %r4,%r6 /* move addr to argument 1 register */ 359 mr %r5,%r7 /* move count to argument 2 register */ 360ENTRY(outs16) 361 cmpwi %r5,0 /* len == 0? */ 362 beqlr- /* return if len == 0 */ 363 addi %r5,%r5,-1 /* len -= 1 */ 364 slwi %r5,%r5,1 /* len *= 2 */ 365 add %r5,%r5,%r4 /* len += src */ 366 addi %r4,%r4,-2 /* pre-decrement */ 3671: lhzu %r0,2(%r4) /* load and increment */ 368 sth %r0,0(%r3) /* store */ 369 cmplw %r4,%r5 /* at the end? */ 370 bne+ 1b /* nope, do another pass */ 371 eieio /* memory barrier (reorder protection) */ 372 DBGSYNC /* force exceptions */ 373 blr /* return */ 374 375/* LINTSTUB: Func: void bswm4_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int32_t *addr, bus_size_t len) */ 376/* LINTSTUB: Func: void bswm4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int32_t *addr, bus_size_t len) */ 377/* LINTSTUB: Func: void outs32(volatile u_int32_t *dst, const u_int32_t *src, size_t len) */ 378 379ENTRY(bswm4_s) 380 lwz %r0,0(%r3) /* get log2(stride) */ 381 rotlw %r5,%r5,%r0 /* shift offset */ 382ENTRY(bswm4) 383 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 384 mr %r4,%r6 /* move addr to argument 1 register */ 385 mr %r5,%r7 /* move count to argument 2 register */ 386ENTRY(outs32) 387 cmpwi %r5,0 /* len == 0? */ 388 beqlr- /* return if len == 0 */ 389 addi %r5,%r5,-1 /* len -= 1 */ 390 slwi %r5,%r5,2 /* len *= 4 */ 391 add %r5,%r5,%r4 /* len += src */ 392 addi %r4,%r4,-4 /* pre-decrement */ 3931: lwzu %r0,4(%r4) /* load and increment */ 394 stw %r0,0(%r3) /* store */ 395 cmplw %r4,%r5 /* at the end? */ 396 bne+ 1b /* nope, do another pass */ 397 eieio /* memory barrier (reorder protection) */ 398 DBGSYNC /* force exceptions */ 399 blr /* return */ 400 401/* LINTSTUB: Func: void bswm8_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int64_t *addr, bus_size_t len) */ 402/* LINTSTUB: Func: void bswm8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int64_t *addr, bus_size_t len) */ 403 404#ifdef _LP64 405ENTRY(bswm8_s) 406 ld %r0,0(%r3) /* get log2(stride) */ 407 rotld %r5,%r5,%r0 /* shift offset */ 408ENTRY(bswm8) 409 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 410 mr %r4,%r6 /* move addr to argument 1 register */ 411 mr %r5,%r7 /* move count to argument 2 register */ 412ENTRY(outs64) 413 cmpdi %r5,0 /* len == 0? */ 414 beqlr- /* return if len == 0 */ 415 addi %r5,%r5,-1 /* len -= 1 */ 416 sldi %r5,%r5,2 /* len *= 4 */ 417 add %r5,%r5,%r4 /* len += src */ 418 addi %r4,%r4,-4 /* pre-decrement */ 4191: ldu %r0,4(%r4) /* load and increment */ 420 std %r0,0(%r3) /* store */ 421 cmplw %r4,%r5 /* at the end? */ 422 bne+ 1b /* nope, do another pass */ 423 eieio /* memory barrier (reorder protection) */ 424 DBGSYNC /* force exceptions */ 425 blr /* return */ 426#else 427ENTRY(bswm8_s) 428ENTRY(bswm8) 429 trap /* die */ 430#endif 431 432/* LINTSTUB: Func: void bswm2rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int16_t *addr, bus_size_t len) */ 433/* LINTSTUB: Func: void bswm2rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int16_t *addr, bus_size_t len) */ 434/* LINTSTUB: Func: void outs16rb(volatile u_int16_t *dst, const u_int16_t *src, size_t len) */ 435 436ENTRY(bswm2rb_s) 437 lwz %r0,0(%r3) /* get log2(stride) */ 438 rotlw %r5,%r5,%r0 /* shift offset */ 439ENTRY(bswm2rb) 440 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 441 mr %r4,%r6 /* move addr to argument 1 register */ 442 mr %r5,%r7 /* move count to argument 2 register */ 443ENTRY(outs16rb) 444 cmpwi %r5,0 /* len == 0? */ 445 beqlr- /* return if len == 0 */ 446 addi %r5,%r5,-1 /* len -= 1 */ 447 slwi %r5,%r5,1 /* len *= 2 */ 448 add %r5,%r5,%r4 /* len += src */ 449 addi %r4,%r4,-2 /* pre-decrement */ 4501: lwzu %r0,2(%r4) /* load and increment */ 451 sthbrx %r0,0,%r3 /* store (byte-reversed) */ 452 cmplw %r4,%r5 /* at the end? */ 453 bne+ 1b /* nope, do another pass */ 454 eieio /* memory barrier (reorder protection) */ 455 DBGSYNC /* force exceptions */ 456 blr /* return */ 457 458/* LINTSTUB: Func: void bswm4rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int32_t *addr, bus_size_t len) */ 459/* LINTSTUB: Func: void bswm4rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int32_t *addr, bus_size_t len) */ 460/* LINTSTUB: Func: void outs32rb(volatile u_int32_t *dst, const u_int32_t *src, size_t len) */ 461 462ENTRY(bswm4rb_s) 463 lwz %r0,0(%r3) /* get log2(stride) */ 464 rotlw %r5,%r5,%r0 /* shift offset */ 465ENTRY(bswm4rb) 466 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 467 mr %r4,%r6 /* move addr to argument 1 register */ 468 mr %r5,%r7 /* move count to argument 2 register */ 469ENTRY(outs32rb) 470 cmpwi %r5,0 /* len == 0? */ 471 beqlr- /* return if len == 0 */ 472 addi %r5,%r5,-1 /* len -= 1 */ 473 slwi %r5,%r5,2 /* len *= 4 */ 474 add %r5,%r5,%r4 /* len += src */ 475 addi %r4,%r4,-4 /* pre-decrement */ 4761: lwzu %r0,4(%r4) /* load and increment */ 477 stwbrx %r0,0,%r3 /* store (byte-reversed) */ 478 cmplw %r4,%r5 /* at the end? */ 479 bne+ 1b /* nope, do another pass */ 480 eieio /* memory barrier (reorder protection) */ 481 DBGSYNC /* force exceptions */ 482 blr /* return */ 483 484/* LINTSTUB: Func: void bswm8rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int64_t *addr, bus_size_t len) */ 485/* LINTSTUB: Func: void bswm8rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int64_t *addr, bus_size_t len) */ 486ENTRY(bswm8rb_s) 487ENTRY(bswm8rb) 488 trap 489 490/* LINTSTUB: Func: void bsrm1_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int8_t *addr, bus_size_t len) */ 491/* LINTSTUB: Func: void bsrm1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int8_t *addr, bus_size_t len) */ 492/* LINTSTUB: Func: void ins8(const volatile u_int8_t *src, u_int8_t *dst, size_t len) */ 493 494ENTRY(bsrm1_s) 495 lwz %r0,0(%r3) /* get log2(stride) */ 496 rotlw %r5,%r5,%r0 /* shift offset */ 497ENTRY(bsrm1) 498 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 499 mr %r4,%r6 /* move addr to argument 1 register */ 500 mr %r5,%r7 /* move count to argument 2 register */ 501ENTRY(ins8) 502 cmpwi %r5,0 /* len == 0? */ 503 beqlr- /* return if len == 0 */ 504 addi %r5,%r5,-1 /* len -= 1 */ 505 add %r5,%r5,%r4 /* len += src */ 506 addi %r4,%r4,-1 /* pre-decrement */ 5071: lbz %r0,0(%r3) /* load value */ 508 stbu %r0,1(%r4) /* store and increment */ 509 cmplw %r4,%r5 /* at the end? */ 510 bne+ 1b /* nope, do another pass */ 511 eieio /* memory barrier (reorder protection) */ 512 DBGSYNC /* force exceptions */ 513 blr /* return */ 514 515/* LINTSTUB: Func: void bsrm2_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int16_t *addr, bus_size_t len) */ 516/* LINTSTUB: Func: void bsrm2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int16_t *addr, bus_size_t len) */ 517/* LINTSTUB: Func: void ins16(const volatile u_int16_t *src, u_int16_t *dst, size_t len) */ 518 519ENTRY(bsrm2_s) 520 lwz %r0,0(%r3) /* get log2(stride) */ 521 rotlw %r5,%r5,%r0 /* shift offset */ 522ENTRY(bsrm2) 523 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 524 mr %r4,%r6 /* move addr to argument 1 register */ 525 mr %r5,%r7 /* move count to argument 2 register */ 526ENTRY(ins16) 527 cmpwi %r5,0 /* len == 0? */ 528 beqlr- /* return if len == 0 */ 529 addi %r5,%r5,-1 /* len -= 1 */ 530 slwi %r5,%r5,1 /* len *= 2 */ 531 add %r5,%r5,%r4 /* len += src */ 532 addi %r4,%r4,-2 /* pre-decrement */ 5331: lhz %r0,0(%r3) /* load value */ 534 sthu %r0,2(%r4) /* store and increment */ 535 cmplw %r4,%r5 /* at the end? */ 536 bne+ 1b /* nope, do another pass */ 537 eieio /* memory barrier (reorder protection) */ 538 DBGSYNC /* force exceptions */ 539 blr /* return */ 540 541/* LINTSTUB: Func: void bsrm4_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int32_t *addr, bus_size_t len) */ 542/* LINTSTUB: Func: void bsrm4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int32_t *addr, bus_size_t len) */ 543/* LINTSTUB: Func: void ins32(const volatile u_int32_t *src, u_int32_t *dst, size_t len) */ 544 545ENTRY(bsrm4_s) 546 lwz %r0,0(%r3) /* get log2(stride) */ 547 rotlw %r5,%r5,%r0 /* shift offset */ 548ENTRY(bsrm4) 549 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 550 mr %r4,%r6 /* move addr to argument 1 register */ 551 mr %r5,%r7 /* move count to argument 2 register */ 552ENTRY(ins32) 553 cmpwi %r5,0 /* len == 0? */ 554 beqlr- /* return if len == 0 */ 555 addi %r5,%r5,-1 /* len -= 1 */ 556 slwi %r5,%r5,2 /* len *= 4 */ 557 add %r5,%r5,%r4 /* len += src */ 558 addi %r4,%r4,-4 /* pre-decrement */ 5591: lwz %r0,0(%r3) /* load value */ 560 stwu %r0,4(%r4) /* store and increment */ 561 cmplw %r4,%r5 /* at the end? */ 562 bne+ 1b /* nope, do another pass */ 563 eieio /* memory barrier (reorder protection) */ 564 DBGSYNC /* force exceptions */ 565 blr /* return */ 566 567/* LINTSTUB: Func: void bsrm8_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int64_t *addr, bus_size_t len) */ 568/* LINTSTUB: Func: void bsrm8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int64_t *addr, bus_size_t len) */ 569ENTRY(bsrm8_s) 570ENTRY(bsrm8) 571 trap 572 573/* LINTSTUB: Func: void bsrm2rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int16_t *addr, bus_size_t len) */ 574/* LINTSTUB: Func: void ins16rb(const volatile u_int16_t *src, u_int16_t *dst, size_t len) */ 575 576ENTRY(bsrm2rb_s) 577 lwz %r0,0(%r3) /* get log2(stride) */ 578 rotlw %r5,%r5,%r0 /* shift offset */ 579ENTRY(bsrm2rb) 580 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 581 mr %r4,%r6 /* move addr to argument 1 register */ 582 mr %r5,%r7 /* move count to argument 2 register */ 583ENTRY(ins16rb) 584 cmpwi %r5,0 /* len == 0? */ 585 beqlr- /* return if len == 0 */ 586 addi %r5,%r5,-1 /* len -= 1 */ 587 slwi %r5,%r5,1 /* len *= 2 */ 588 add %r5,%r5,%r4 /* len += src */ 589 addi %r4,%r4,-2 /* pre-decrement */ 5901: lhbrx %r0,0,%r3 /* load value (byte reversed) */ 591 sthu %r0,2(%r4) /* store and increment */ 592 cmplw %r4,%r5 /* at the end? */ 593 bne+ 1b /* nope, do another pass */ 594 eieio /* memory barrier (reorder protection) */ 595 DBGSYNC /* force exceptions */ 596 blr /* return */ 597 598/* LINTSTUB: Func: void bsrm4rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int32_t *addr, bus_size_t len) */ 599/* LINTSTUB: Func: void bsrm4rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int32_t *addr, bus_size_t len) */ 600/* LINTSTUB: Func: void ins32rb(const volatile u_int32_t *src, u_int32_t *dst, size_t len) */ 601ENTRY(bsrm4rb_s) 602 lwz %r0,0(%r3) /* get log2(stride) */ 603 rotlw %r5,%r5,%r0 /* shift offset */ 604ENTRY(bsrm4rb) 605 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 606 mr %r4,%r6 /* move addr to argument 1 register */ 607 mr %r5,%r7 /* move count to argument 2 register */ 608ENTRY(ins32rb) 609 cmpwi %r5,0 /* len == 0? */ 610 beqlr- /* return if len == 0 */ 611 addi %r5,%r5,-1 /* len -= 1 */ 612 slwi %r5,%r5,2 /* len *= 4 */ 613 add %r5,%r5,%r4 /* len += src */ 614 addi %r4,%r4,-4 /* pre-decrement */ 6151: lwbrx %r0,0,%r3 /* load value (byte reversed) */ 616 stwu %r0,4(%r4) /* store and increment */ 617 cmplw %r4,%r5 /* at the end? */ 618 bne+ 1b /* nope, do another pass */ 619 eieio /* memory barrier (reorder protection) */ 620 DBGSYNC /* force exceptions */ 621 blr /* return */ 622 623/* LINTSTUB: Func: void bsrm8rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int64_t *addr, bus_size_t len) */ 624/* LINTSTUB: Func: void bsrm8rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int64_t *addr, bus_size_t len) */ 625ENTRY(bsrm8rb_s) 626ENTRY(bsrm8rb) 627 trap 628 629/* LINTSTUB: Func: void bswr1_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int8_t *a, bus_size_t c); */ 630/* LINTSTUB: Func: void bswr1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int8_t *a, bus_size_t c); */ 631ENTRY(bswr1_s) 632 lwz %r0,0(%r3) /* get log2(stride) */ 633 li %r8,1 /* distance between dst bytes */ 634 rotlw %r5,%r5,%r0 /* shift offset */ 635 rotlw %r8,%r8,%r0 /* shift distance */ 636 b .Lbswr1_enter 637ENTRY(bswr1) 638 li %r8,1 /* distance between dst bytes */ 639.Lbswr1_enter: 640 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 641 cmpwi %r7,0 /* len == 0? */ 642 beqlr- /* return if len == 0 */ 643 addi %r7,%r7,-1 /* len -= 1 */ 644 add %r7,%r7,%r6 /* len += src */ 645 addi %r6,%r6,-1 /* pre-decrement */ 646 sub %r3,%r3,%r8 6471: lbzu %r0,1(%r6) /* load and increment */ 648 stbux %r0,%r3,%r8 /* store and add distance */ 649 cmplw %r6,%r7 /* at the end? */ 650 bne+ 1b /* nope, do another pass */ 651 eieio /* memory barrier (reorder protection) */ 652 DBGSYNC /* force exceptions */ 653 blr /* return */ 654 655/* LINTSTUB: Func: void bswr2_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int16_t *a, bus_size_t c); */ 656/* LINTSTUB: Func: void bswr2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int16_t *a, bus_size_t c); */ 657ENTRY(bswr2_s) 658 lwz %r0,0(%r3) /* get log2(stride) */ 659 li %r8,2 /* distance between dst halfwords */ 660 rotlw %r5,%r5,%r0 /* shift offset */ 661 rotlw %r8,%r8,%r0 /* shift distance */ 662 b .Lbswr2_enter 663ENTRY(bswr2) 664 li %r8,2 /* distance between dst halfwords */ 665.Lbswr2_enter: 666 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 667 cmpwi %r7,0 /* len == 0? */ 668 beqlr- /* return if len == 0 */ 669 addi %r7,%r7,-1 /* len -= 1 */ 670 slwi %r7,%r7,1 /* len *= 2 */ 671 add %r7,%r7,%r6 /* len += src */ 672 addi %r6,%r6,-2 /* pre-decrement */ 673 sub %r3,%r3,%r8 6741: lhzu %r0,2(%r6) /* load and increment */ 675 sthux %r0,%r3,%r8 /* store and add distance */ 676 cmplw %r6,%r7 /* at the end? */ 677 bne+ 1b /* nope, do another pass */ 678 eieio /* memory barrier (reorder protection) */ 679 DBGSYNC /* force exceptions */ 680 blr /* return */ 681 682/* LINTSTUB: Func: void bswr2rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int16_t *a, bus_size_t c); */ 683/* LINTSTUB: Func: void bswr2rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int16_t *a, bus_size_t c); */ 684ENTRY(bswr2rb_s) 685 lwz %r0,0(%r3) /* get log2(stride) */ 686 li %r8,2 /* distance between dst halfwords */ 687 rotlw %r5,%r5,%r0 /* shift offset */ 688 rotlw %r8,%r8,%r0 /* shift distance */ 689 b .Lbswr2rb_enter 690ENTRY(bswr2rb) 691 li %r8,2 /* distance between dst halfwords */ 692.Lbswr2rb_enter: 693 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 694 cmpwi %r7,0 /* len == 0? */ 695 beqlr- /* return if len == 0 */ 696 addi %r7,%r7,-1 /* len -= 1 */ 697 slwi %r7,%r7,1 /* len *= 2 */ 698 add %r7,%r7,%r6 /* len += src */ 699 addi %r6,%r6,-2 /* pre-decrement */ 7001: lhzu %r0,2(%r6) /* load and increment */ 701 sthbrx %r0,0,%r3 /* store (reversed) */ 702 add %r3,%r3,%r8 /* dst += distance */ 703 cmplw %r6,%r7 /* at the end? */ 704 bne+ 1b /* nope, do another pass */ 705 eieio /* memory barrier (reorder protection) */ 706 DBGSYNC /* force exceptions */ 707 blr /* return */ 708 709/* LINTSTUB: Func: void bswr4_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int32_t *a, bus_size_t c); */ 710/* LINTSTUB: Func: void bswr4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int32_t *a, bus_size_t c); */ 711ENTRY(bswr4_s) 712 lwz %r0,0(%r3) /* get log2(stride) */ 713 li %r8,4 /* distance between dst halfwords */ 714 rotlw %r5,%r5,%r0 /* shift offset */ 715 rotlw %r8,%r8,%r0 /* shift distance */ 716 b .Lbswr4_enter 717ENTRY(bswr4) 718 li %r8,4 /* distance between dst halfwords */ 719.Lbswr4_enter: 720 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 721 cmpwi %r7,0 /* len == 0? */ 722 beqlr- /* return if len == 0 */ 723 addi %r7,%r7,-1 /* len -= 1 */ 724 slwi %r7,%r7,2 /* len *= 4 */ 725 add %r7,%r7,%r6 /* len += src */ 726 addi %r6,%r6,-4 /* pre-decrement */ 727 sub %r3,%r3,%r8 7281: lwzu %r0,4(%r6) /* load and increment */ 729 stwux %r0,%r3,%r8 /* store and add distance */ 730 cmplw %r6,%r7 /* at the end? */ 731 bne+ 1b /* nope, do another pass */ 732 eieio /* memory barrier (reorder protection) */ 733 DBGSYNC /* force exceptions */ 734 blr /* return */ 735 736/* LINTSTUB: Func: void bswr4rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int32_t *a, bus_size_t c); */ 737/* LINTSTUB: Func: void bswr4rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int32_t *a, bus_size_t c); */ 738ENTRY(bswr4rb_s) 739 lwz %r0,0(%r3) /* get log2(stride) */ 740 li %r8,4 /* distance between dst halfwords */ 741 rotlw %r5,%r5,%r0 /* shift offset */ 742 rotlw %r8,%r8,%r0 /* shift distance */ 743 b .Lbswr4rb_enter 744ENTRY(bswr4rb) 745 li %r8,4 /* distance between dst halfwords */ 746.Lbswr4rb_enter: 747 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 748 cmpwi %r7,0 /* len == 0? */ 749 beqlr- /* return if len == 0 */ 750 addi %r7,%r7,-1 /* len -= 1 */ 751 slwi %r7,%r7,2 /* len *= 4 */ 752 add %r7,%r7,%r6 /* len += src */ 753 addi %r6,%r6,-4 /* pre-decrement */ 7541: lwzu %r0,4(%r6) /* load and increment */ 755 stwbrx %r0,0,%r3 /* store (reversed) */ 756 add %r3,%r3,%r8 /* dst += distance */ 757 cmplw %r6,%r7 /* at the end? */ 758 bne+ 1b /* nope, do another pass */ 759 eieio /* memory barrier (reorder protection) */ 760 DBGSYNC /* force exceptions */ 761 blr /* return */ 762 763/* LINTSTUB: Func: void bswr8_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int64_t *a, bus_size_t c); */ 764/* LINTSTUB: Func: void bswr8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int64_t *a, bus_size_t c); */ 765/* LINTSTUB: Func: void bswr8rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int64_t *a, bus_size_t c); */ 766/* LINTSTUB: Func: void bswr8rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int64_t *a, bus_size_t c); */ 767ENTRY(bswr8_s) 768ENTRY(bswr8rb_s) 769 lwz %r0,0(%r3) 770 rotlw %r5,%r5,%r0 771ENTRY(bswr8) 772ENTRY(bswr8rb) 773 trap 774 775/* LINTSTUB: Func: void bsrr1_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int8_t *a, bus_size_t c); */ 776/* LINTSTUB: Func: void bsrr1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int8_t *a, bus_size_t c); */ 777ENTRY(bsrr1_s) 778 lwz %r0,0(%r3) /* get log2(stride) */ 779 li %r8,1 /* distance between src bytes */ 780 rotlw %r5,%r5,%r0 /* shift offset */ 781 rotlw %r8,%r8,%r0 /* shift distance */ 782 b .Lbsrr1_enter 783ENTRY(bsrr1) 784 li %r8,1 /* distance between src bytes */ 785.Lbsrr1_enter: 786 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 787 cmpwi %r7,0 /* len == 0? */ 788 beqlr- /* return if len == 0 */ 789 addi %r7,%r7,-1 /* len -= 1 */ 790 add %r7,%r7,%r6 /* len += src */ 791 addi %r6,%r6,-1 /* pre-decrement */ 792 sub %r3,%r3,%r8 7931: lbzux %r0,%r3,%r8 /* load value and add distance */ 794 stbu %r0,1(%r6) /* store and increment */ 795 cmplw %r6,%r7 /* at the end? */ 796 bne+ 1b /* nope, do another pass */ 797 eieio /* memory barrier (reorder protection) */ 798 DBGSYNC /* force exceptions */ 799 blr /* return */ 800 801/* LINTSTUB: Func: void bsrr2_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int16_t *a, bus_size_t c); */ 802/* LINTSTUB: Func: void bsrr2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int16_t *a, bus_size_t c); */ 803ENTRY(bsrr2_s) 804 lwz %r0,0(%r3) /* get log2(stride) */ 805 li %r8,2 /* distance between src halfwords */ 806 rotlw %r5,%r5,%r0 /* shift offset */ 807 rotlw %r8,%r8,%r0 /* shift distance */ 808 b .Lbsrr2_enter 809ENTRY(bsrr2) 810 li %r8,2 /* distance between src halfwords */ 811.Lbsrr2_enter: 812 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 813 cmpwi %r7,0 /* len == 0? */ 814 beqlr- /* return if len == 0 */ 815 addi %r7,%r7,-1 /* len -= 1 */ 816 slwi %r7,%r7,1 /* len *= 2 */ 817 add %r7,%r7,%r6 /* len += src */ 818 addi %r6,%r6,-2 /* pre-decrement */ 819 sub %r3,%r3,%r8 8201: lhzux %r0,%r3,%r8 /* load value and add distance */ 821 sthu %r0,2(%r6) /* store and increment */ 822 cmplw %r6,%r7 /* at the end? */ 823 bne+ 1b /* nope, do another pass */ 824 eieio /* memory barrier (reorder protection) */ 825 DBGSYNC /* force exceptions */ 826 blr /* return */ 827 828/* LINTSTUB: Func: void bsrr2rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int16_t *a, bus_size_t c); */ 829/* LINTSTUB: Func: void bsrr2rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int16_t *a, bus_size_t c); */ 830ENTRY(bsrr2rb_s) 831 lwz %r0,0(%r3) /* get log2(stride) */ 832 li %r8,2 /* distance between source halfwords */ 833 rotlw %r5,%r5,%r0 /* shift offset */ 834 rotlw %r8,%r8,%r0 /* shift distance */ 835 b .Lbsrr2rb_enter 836ENTRY(bsrr2rb) 837 li %r8,2 /* distance between source halfwords */ 838.Lbsrr2rb_enter: 839 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 840 cmpwi %r7,0 /* len == 0? */ 841 beqlr- /* return if len == 0 */ 842 addi %r7,%r7,-1 /* len -= 1 */ 843 slwi %r7,%r7,1 /* len *= 2 */ 844 add %r7,%r7,%r6 /* len += src */ 845 addi %r6,%r6,-2 /* pre-decrement */ 8461: lhbrx %r0,0,%r3 /* load value (reversed) */ 847 add %r3,%r3,%r8 /* src += distance */ 848 sthu %r0,2(%r6) /* store and increment */ 849 cmplw %r6,%r7 /* at the end? */ 850 bne+ 1b /* nope, do another pass */ 851 eieio /* memory barrier (reorder protection) */ 852 DBGSYNC /* force exceptions */ 853 blr /* return */ 854 855/* LINTSTUB: Func: void bsrr4_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int32_t *a, bus_size_t c); */ 856/* LINTSTUB: Func: void bsrr4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int32_t *a, bus_size_t c); */ 857ENTRY(bsrr4_s) 858 lwz %r0,0(%r3) /* get log2(stride) */ 859 li %r8,4 /* distance between src words */ 860 rotlw %r5,%r5,%r0 /* shift offset */ 861 rotlw %r8,%r8,%r0 /* shift distance */ 862 b .Lbsrr4_enter 863ENTRY(bsrr4) 864 li %r8,4 /* distance between src words */ 865.Lbsrr4_enter: 866 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 867 cmpwi %r7,0 /* len == 0? */ 868 beqlr- /* return if len == 0 */ 869 addi %r7,%r7,-1 /* len -= 1 */ 870 slwi %r7,%r7,2 /* len *= 4 */ 871 add %r7,%r7,%r6 /* len += src */ 872 addi %r6,%r6,-4 /* pre-decrement */ 873 sub %r3,%r3,%r8 8741: lwzux %r0,%r3,%r8 /* load value and add distance */ 875 stwu %r0,4(%r6) /* store and increment */ 876 cmplw %r6,%r7 /* at the end? */ 877 bne+ 1b /* nope, do another pass */ 878 eieio /* memory barrier (reorder protection) */ 879 DBGSYNC /* force exceptions */ 880 blr /* return */ 881 882/* LINTSTUB: Func: void bsrr4rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int32_t *a, bus_size_t c); */ 883/* LINTSTUB: Func: void bsrr4rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int32_t *a, bus_size_t c); */ 884ENTRY(bsrr4rb_s) 885 lwz %r0,0(%r3) /* get log2(stride) */ 886 li %r8,4 /* distance between src words */ 887 rotlw %r5,%r5,%r0 /* shift offset */ 888 rotlw %r8,%r8,%r0 /* shift distance */ 889 b .Lbsrr4rb_enter 890ENTRY(bsrr4rb) 891 li %r8,4 /* distance between src words */ 892.Lbsrr4rb_enter: 893 add %r3,%r4,%r5 /* add offset to handle & place in argument 0 */ 894 mr %r4,%r6 /* move addr to argument 1 register */ 895 cmpwi %r7,0 /* len == 0? */ 896 beqlr- /* return if len == 0 */ 897 addi %r7,%r7,-1 /* len -= 1 */ 898 slwi %r7,%r7,2 /* len *= 4 */ 899 add %r7,%r7,%r6 /* len += src */ 900 addi %r6,%r6,-4 /* pre-decrement */ 9011: lwbrx %r0,0,%r3 /* load value (reversed) */ 902 add %r3,%r3,%r8 /* src += distance */ 903 sthu %r0,4(%r6) /* store and increment */ 904 cmplw %r6,%r7 /* at the end? */ 905 bne+ 1b /* nope, do another pass */ 906 eieio /* memory barrier (reorder protection) */ 907 DBGSYNC /* force exceptions */ 908 blr /* return */ 909 910/* LINTSTUB: Func: void bsrr8_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int64_t *a, bus_size_t c); */ 911/* LINTSTUB: Func: void bsrr8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int64_t *a, bus_size_t c); */ 912/* LINTSTUB: Func: void bsrr8rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int64_t *a, bus_size_t c); */ 913/* LINTSTUB: Func: void bsrr8rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int64_t *a, bus_size_t c); */ 914ENTRY(bsrr8_s) 915ENTRY(bsrr8rb_s) 916 lwz %r0,0(%r3) 917 rotlw %r5,%r5,%r0 918ENTRY(bsrr8) 919ENTRY(bsrr8rb) 920 trap 921 922/* LINTSTUB: Func: void bssr1_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int8_t v, bus_size_t c); */ 923/* LINTSTUB: Func: void bssr1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, u_int8_t v, bus_size_t c); */ 924ENTRY(bssr1_s) 925 lwz %r0,0(%r3) /* get log2(stride) */ 926 li %r8,1 /* distance between src bytes */ 927 rotlw %r5,%r5,%r0 /* shift offset */ 928 rotlw %r8,%r8,%r0 /* shift distance */ 929 b .Lbssr1_enter 930ENTRY(bssr1) 931 li %r8,1 /* distance between src bytes */ 932.Lbssr1_enter: 933 cmpwi %r7,0 /* len == 0? */ 934 beqlr- /* return if len == 0 */ 9351: addi %r7,%r7,-1 /* len -= 1 */ 936 stbx %r6,%r4,%r5 /* store value */ 937 add %r5,%r5,%r8 /* add offset */ 938 cmpwi %r7,0 /* len == 0? */ 939 bne+ 1b /* nope, do another pass */ 940 eieio /* memory barrier (reorder protection) */ 941 DBGSYNC /* force exceptions */ 942 blr /* return */ 943 944/* LINTSTUB: Func: void bssr2_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int16_t v, bus_size_t c); */ 945/* LINTSTUB: Func: void bssr2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int16_t v, bus_size_t c); */ 946ENTRY(bssr2_s) 947 lwz %r0,0(%r3) /* get log2(stride) */ 948 li %r8,2 /* distance between src halfwords */ 949 rotlw %r5,%r5,%r0 /* shift offset */ 950 rotlw %r8,%r8,%r0 /* shift distance */ 951 b .Lbssr2_enter 952ENTRY(bssr2) 953 li %r8,2 /* distance between src halfwords */ 954.Lbssr2_enter: 955 cmpwi %r7,0 /* len == 0? */ 956 beqlr- /* return if len == 0 */ 9571: addi %r7,%r7,-1 /* len -= 1 */ 958 sthx %r6,%r4,%r5 /* store value */ 959 add %r5,%r5,%r8 /* add offset */ 960 cmpwi %r7,0 /* len == 0? */ 961 bne+ 1b /* nope, do another pass */ 962 eieio /* memory barrier (reorder protection) */ 963 DBGSYNC /* force exceptions */ 964 blr /* return */ 965 966/* LINTSTUB: Func: void bssr2rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int16_t v, bus_size_t c); */ 967/* LINTSTUB: Func: void bssr2rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int16_t v, bus_size_t c); */ 968ENTRY(bssr2rb_s) 969 lwz %r0,0(%r3) /* get log2(stride) */ 970 li %r8,2 /* distance between src halfwords */ 971 rotlw %r5,%r5,%r0 /* shift offset */ 972 rotlw %r8,%r8,%r0 /* shift distance */ 973 b .Lbssr2rb_enter 974ENTRY(bssr2rb) 975 li %r8,2 /* distance between src halfwords */ 976.Lbssr2rb_enter: 977 cmpwi %r7,0 /* len == 0? */ 978 beqlr- /* return if len == 0 */ 9791: addi %r7,%r7,-1 /* len -= 1 */ 980 sthbrx %r6,%r4,%r5 /* store value */ 981 add %r5,%r5,%r8 /* add offset */ 982 cmpwi %r7,0 /* len == 0? */ 983 bne+ 1b /* nope, do another pass */ 984 eieio /* memory barrier (reorder protection) */ 985 DBGSYNC /* force exceptions */ 986 blr /* return */ 987 988/* LINTSTUB: Func: void bssr4_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int32_t v, bus_size_t c); */ 989/* LINTSTUB: Func: void bssr4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int32_t v, bus_size_t c); */ 990ENTRY(bssr4_s) 991 lwz %r0,0(%r3) /* get log2(stride) */ 992 li %r8,4 /* distance between src words */ 993 rotlw %r5,%r5,%r0 /* shift offset */ 994 rotlw %r8,%r8,%r0 /* shift distance */ 995 b .Lbssr4_enter 996ENTRY(bssr4) 997 li %r8,4 /* distance between src words */ 998.Lbssr4_enter: 999 cmpwi %r7,0 /* len == 0? */ 1000 beqlr- /* return if len == 0 */ 10011: addi %r7,%r7,-1 /* len -= 1 */ 1002 stwx %r6,%r4,%r5 /* store value */ 1003 add %r5,%r5,%r8 /* add offset */ 1004 cmpwi %r7,0 /* len == 0? */ 1005 bne+ 1b /* nope, do another pass */ 1006 eieio /* memory barrier (reorder protection) */ 1007 DBGSYNC /* force exceptions */ 1008 blr /* return */ 1009 1010/* LINTSTUB: Func: void bssr4rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int32_t v, bus_size_t c); */ 1011/* LINTSTUB: Func: void bssr4rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int32_t v, bus_size_t c); */ 1012ENTRY(bssr4rb_s) 1013 lwz %r0,0(%r3) /* get log2(stride) */ 1014 li %r8,4 /* distance between src words */ 1015 rotlw %r5,%r5,%r0 /* shift offset */ 1016 rotlw %r8,%r8,%r0 /* shift distance */ 1017 b .Lbssr4rb_enter 1018ENTRY(bssr4rb) 1019 li %r8,4 /* distance between src words */ 1020.Lbssr4rb_enter: 1021 cmpwi %r7,0 /* len == 0? */ 1022 beqlr- /* return if len == 0 */ 10231: addi %r7,%r7,-1 /* len -= 1 */ 1024 stwbrx %r6,%r4,%r5 /* store value */ 1025 add %r5,%r5,%r8 /* add offset */ 1026 cmpwi %r7,0 /* len == 0? */ 1027 bne+ 1b /* nope, do another pass */ 1028 eieio /* memory barrier (reorder protection) */ 1029 DBGSYNC /* force exceptions */ 1030 blr /* return */ 1031 1032/* LINTSTUB: Func: void bssr8_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int64_t v, bus_size_t c); */ 1033/* LINTSTUB: Func: void bssr8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int64_t v, bus_size_t c); */ 1034/* LINTSTUB: Func: void bssr8rb_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int64_t v, bus_size_t c); */ 1035/* LINTSTUB: Func: void bssr8rb(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, const u_int64_t v, bus_size_t c); */ 1036ENTRY(bssr8_s) 1037ENTRY(bssr8rb_s) 1038 lwz %r0,0(%r3) 1039 rotlw %r5,%r5,%r0 1040ENTRY(bssr8) 1041ENTRY(bssr8rb) 1042 trap 1043 1044/* LINTSTUB: Func: void bscr1_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, bus_space_handle_t h2, bus_size_t o2, bus_size_t len); */ 1045/* LINTSTUB: Func: void bscr1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, bus_space_handle_t h2, bus_size_t o2, bus_size_t len); */ 1046ENTRY(bscr1_s) 1047 lwz %r0,0(%r3) /* get log2(stride) */ 1048 b .Lbscr1_enter 1049ENTRY(bscr1) 1050 li %r0,0 /* non stride */ 1051.Lbscr1_enter: 1052 li %r9,1 /* distance between src bytes */ 1053 rotlw %r9,%r9,%r0 /* shift distance */ 1054 cmpwi %r8,0 /* len == 0? */ 1055 beqlr- /* return if len == 0 */ 1056 rotlw %r5,%r5,%r0 /* shift src offset */ 1057 rotlw %r7,%r7,%r0 /* shift dest offset */ 1058 add %r10,%r4,%r5 /* calculate src end address */ 1059 add %r11,%r6,%r7 /* calculate dest end address */ 1060 mtctr %r8 1061 cmpw %r10,%r11 /* compare end address */ 1062 blt 2f /* jump if h + o < h2 + o2 */ 1063 1064 /* h + o >= h2 + o2 */ 10651: lbzx %r12,%r4,%r5 /* load value */ 1066 stbx %r12,%r6,%r7 /* store value */ 1067 add %r5,%r5,%r9 /* src offset += 1 */ 1068 add %r7,%r7,%r9 /* dest offset += 1 */ 1069 bdnz+ 1b /* jump if len != 0 */ 1070 b .Lbscr1_end /* end */ 1071 1072 /* h + o < h2 + o2 */ 10732: addi %r8,%r8,-1 /* len -= 1 */ 1074 rotlw %r8,%r8,%r0 /* shift len */ 1075 add %r5,%r10,%r8 /* src offset = o + len - 1 */ 1076 add %r7,%r11,%r8 /* dest offset = o2 + len - 1 */ 10773: lbzx %r12,%r4,%r5 /* load value */ 1078 stbx %r12,%r6,%r7 /* store value */ 1079 sub %r5,%r5,%r9 /* src offset -= 1 */ 1080 sub %r7,%r7,%r9 /* dest offset -= 1 */ 1081 bdnz+ 3b /* jump if len != 0 */ 1082.Lbscr1_end: 1083 eieio /* memory barrier (reorder protection) */ 1084 DBGSYNC /* force exceptions */ 1085 blr /* return */ 1086 1087/* LINTSTUB: Func: void bscr2_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, bus_space_handle_t h2, bus_size_t o2, bus_size_t len); */ 1088/* LINTSTUB: Func: void bscr2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, bus_space_handle_t h2, bus_size_t o2, bus_size_t len); */ 1089ENTRY(bscr2_s) 1090 lwz %r0,0(%r3) /* get log2(stride) */ 1091 b .Lbscr2_enter 1092ENTRY(bscr2) 1093 li %r0,0 /* non stride */ 1094.Lbscr2_enter: 1095 li %r9,2 /* distance between src halfwords */ 1096 rotlw %r9,%r9,%r0 /* shift distance */ 1097 cmpwi %r8,0 /* len == 0? */ 1098 beqlr- /* return if len == 0 */ 1099 rotlw %r5,%r5,%r0 /* shift src offset */ 1100 rotlw %r7,%r7,%r0 /* shift dest offset */ 1101 add %r10,%r4,%r5 /* calculate src end address */ 1102 add %r11,%r6,%r7 /* calculate dest end address */ 1103 mtctr %r8 1104 cmpw %r10,%r11 /* compare end address */ 1105 blt 2f /* jump if h + o < h2 + o2 */ 1106 1107 /* h + o >= h2 + o2 */ 11081: lhzx %r12,%r4,%r5 /* load value */ 1109 sthx %r12,%r6,%r7 /* store value */ 1110 add %r5,%r5,%r9 /* src offset += 2 */ 1111 add %r7,%r7,%r9 /* dest offset += 2 */ 1112 bdnz+ 1b /* jump if len != 0 */ 1113 b .Lbscr2_end /* end */ 1114 1115 /* h + o < h2 + o2 */ 11162: addi %r8,%r8,-1 /* len -= 1 */ 1117 rotlw %r8,%r8,%r0 /* shift len */ 1118 add %r5,%r10,%r8 /* src offset = o + len - 1 */ 1119 add %r7,%r11,%r8 /* dest offset = o2 + len - 1 */ 11203: lhzx %r12,%r4,%r5 /* load value */ 1121 sthx %r12,%r6,%r7 /* store value */ 1122 sub %r5,%r5,%r9 /* src offset -= 2 */ 1123 sub %r7,%r7,%r9 /* dest offset -= 2 */ 1124 bdnz+ 3b /* jump if len != 0 */ 1125.Lbscr2_end: 1126 eieio /* memory barrier (reorder protection) */ 1127 DBGSYNC /* force exceptions */ 1128 blr /* return */ 1129 1130/* LINTSTUB: Func: void bscr4_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, bus_space_handle_t h2, bus_size_t o2, bus_size_t len); */ 1131/* LINTSTUB: Func: void bscr4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, bus_space_handle_t h2, bus_size_t o2, bus_size_t len); */ 1132ENTRY(bscr4_s) 1133 lwz %r0,0(%r3) /* get log2(stride) */ 1134 b .Lbscr4_enter 1135ENTRY(bscr4) 1136 li %r0,0 /* non stride */ 1137.Lbscr4_enter: 1138 li %r9,4 /* distance between src words */ 1139 rotlw %r9,%r9,%r0 /* shift distance */ 1140 cmpwi %r8,0 /* len == 0? */ 1141 beqlr- /* return if len == 0 */ 1142 rotlw %r5,%r5,%r0 /* shift src offset */ 1143 rotlw %r7,%r7,%r0 /* shift dest offset */ 1144 add %r10,%r4,%r5 /* calculate src end address */ 1145 add %r11,%r6,%r7 /* calculate dest end address */ 1146 mtctr %r8 1147 cmpw %r10,%r11 /* compare end address */ 1148 blt 2f /* jump if h + o < h2 + o2 */ 1149 1150 /* h + o >= h2 + o2 */ 11511: lwzx %r12,%r4,%r5 /* load value */ 1152 stwx %r12,%r6,%r7 /* store value */ 1153 add %r5,%r5,%r9 /* src offset += 4 */ 1154 add %r7,%r7,%r9 /* dest offset += 4 */ 1155 bdnz+ 1b /* jump if len != 0 */ 1156 b .Lbscr4_end /* end */ 1157 1158 /* h + o < h2 + o2 */ 11592: addi %r8,%r8,-1 /* len -= 1 */ 1160 rotlw %r8,%r8,%r0 /* shift len */ 1161 add %r5,%r10,%r8 /* src offset = o + len - 1 */ 1162 add %r7,%r11,%r8 /* dest offset = o2 + len - 1 */ 11633: lwzx %r12,%r4,%r5 /* load value */ 1164 stwx %r12,%r6,%r7 /* store value */ 1165 sub %r5,%r5,%r9 /* src offset -= 4 */ 1166 sub %r7,%r7,%r9 /* dest offset -= 4 */ 1167 bdnz+ 3b /* jump if len != 0 */ 1168.Lbscr4_end: 1169 eieio /* memory barrier (reorder protection) */ 1170 DBGSYNC /* force exceptions */ 1171 blr /* return */ 1172 1173/* LINTSTUB: Func: void bscr8_s(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, bus_space_handle_t h2, bus_size_t o2, bus_size_t len); */ 1174/* LINTSTUB: Func: void bscr8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, bus_space_handle_t h2, bus_size_t o2, bus_size_t len); */ 1175ENTRY(bscr8_s) 1176ENTRY(bscr8) 1177 trap 1178 1179