1#! /usr/bin/env perl
2# Copyright 2012-2021 The OpenSSL Project Authors. All Rights Reserved.
3#
4# Licensed under the Apache License 2.0 (the "License").  You may not use
5# this file except in compliance with the License.  You can obtain a copy
6# in the file LICENSE in the source distribution or at
7# https://www.openssl.org/source/license.html
8
9
10# ====================================================================
11# Written by David S. Miller and Andy Polyakov
12# The module is licensed under 2-clause BSD license.
13# November 2012. All rights reserved.
14# ====================================================================
15
16######################################################################
17# Montgomery squaring-n-multiplication module for SPARC T4.
18#
19# The module consists of three parts:
20#
21# 1) collection of "single-op" subroutines that perform single
22#    operation, Montgomery squaring or multiplication, on 512-,
23#    1024-, 1536- and 2048-bit operands;
24# 2) collection of "multi-op" subroutines that perform 5 squaring and
25#    1 multiplication operations on operands of above lengths;
26# 3) fall-back and helper VIS3 subroutines.
27#
28# RSA sign is dominated by multi-op subroutine, while RSA verify and
29# DSA - by single-op. Special note about 4096-bit RSA verify result.
30# Operands are too long for dedicated hardware and it's handled by
31# VIS3 code, which is why you don't see any improvement. It's surely
32# possible to improve it [by deploying 'mpmul' instruction], maybe in
33# the future...
34#
35# Performance improvement.
36#
37# 64-bit process, VIS3:
38#                   sign    verify    sign/s verify/s
39# rsa 1024 bits 0.000628s 0.000028s   1592.4  35434.4
40# rsa 2048 bits 0.003282s 0.000106s    304.7   9438.3
41# rsa 4096 bits 0.025866s 0.000340s     38.7   2940.9
42# dsa 1024 bits 0.000301s 0.000332s   3323.7   3013.9
43# dsa 2048 bits 0.001056s 0.001233s    946.9    810.8
44#
45# 64-bit process, this module:
46#                   sign    verify    sign/s verify/s
47# rsa 1024 bits 0.000256s 0.000016s   3904.4  61411.9
48# rsa 2048 bits 0.000946s 0.000029s   1056.8  34292.7
49# rsa 4096 bits 0.005061s 0.000340s    197.6   2940.5
50# dsa 1024 bits 0.000176s 0.000195s   5674.7   5130.5
51# dsa 2048 bits 0.000296s 0.000354s   3383.2   2827.6
52#
53######################################################################
54# 32-bit process, VIS3:
55#                   sign    verify    sign/s verify/s
56# rsa 1024 bits 0.000665s 0.000028s   1504.8  35233.3
57# rsa 2048 bits 0.003349s 0.000106s    298.6   9433.4
58# rsa 4096 bits 0.025959s 0.000341s     38.5   2934.8
59# dsa 1024 bits 0.000320s 0.000341s   3123.3   2929.6
60# dsa 2048 bits 0.001101s 0.001260s    908.2    793.4
61#
62# 32-bit process, this module:
63#                   sign    verify    sign/s verify/s
64# rsa 1024 bits 0.000301s 0.000017s   3317.1  60240.0
65# rsa 2048 bits 0.001034s 0.000030s    966.9  33812.7
66# rsa 4096 bits 0.005244s 0.000341s    190.7   2935.4
67# dsa 1024 bits 0.000201s 0.000205s   4976.1   4879.2
68# dsa 2048 bits 0.000328s 0.000360s   3051.1   2774.2
69#
70# 32-bit code is prone to performance degradation as interrupt rate
71# dispatched to CPU executing the code grows. This is because in
72# standard process of handling interrupt in 32-bit process context
73# upper halves of most integer registers used as input or output are
74# zeroed. This renders result invalid, and operation has to be re-run.
75# If CPU is "bothered" with timer interrupts only, the penalty is
76# hardly measurable. But in order to mitigate this problem for higher
77# interrupt rates contemporary Linux kernel recognizes biased stack
78# even in 32-bit process context and preserves full register contents.
79# See http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=517ffce4e1a03aea979fe3a18a3dd1761a24fafb
80# for details.
81
82$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
83push(@INC,"${dir}","${dir}../../perlasm");
84require "sparcv9_modes.pl";
85
86$output = pop and open STDOUT,">$output";
87
88$code.=<<___;
89#ifndef __ASSEMBLER__
90# define __ASSEMBLER__ 1
91#endif
92#include "crypto/sparc_arch.h"
93
94#ifdef    __arch64__
95.register %g2,#scratch
96.register %g3,#scratch
97#endif
98
99.section  ".text",#alloc,#execinstr
100
101#ifdef    __PIC__
102SPARC_PIC_THUNK(%g1)
103#endif
104___
105
106########################################################################
107# Register layout for mont[mul|sqr] instructions.
108# For details see "Oracle SPARC Architecture 2011" manual at
109# http://www.oracle.com/technetwork/server-storage/sun-sparc-enterprise/documentation/.
110#
111my @R=map("%f".2*$_,(0..11,30,31,12..29));
112my @N=(map("%l$_",(0..7)),map("%o$_",(0..5))); @N=(@N,@N,@N[0..3]);
113my @A=(@N[0..13],@R[14..31]);
114my @B=(map("%i$_",(0..5)),map("%l$_",(0..7))); @B=(@B,@B,map("%o$_",(0..3)));
115
116########################################################################
117# int bn_mul_mont_t4_$NUM(u64 *rp,const u64 *ap,const u64 *bp,
118#                               const u64 *np,const BN_ULONG *n0);
119#
120sub generate_bn_mul_mont_t4() {
121my $NUM=shift;
122my ($rp,$ap,$bp,$np,$sentinel)=map("%g$_",(1..5));
123
124$code.=<<___;
125.globl    bn_mul_mont_t4_$NUM
126.align    32
127bn_mul_mont_t4_$NUM:
128#ifdef    __arch64__
129          mov       0,$sentinel
130          mov       -128,%g4
131#elif defined(SPARCV9_64BIT_STACK)
132          SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5)
133          ld        [%g1+0],%g1         ! OPENSSL_sparcv9_P[0]
134          mov       -2047,%g4
135          and       %g1,SPARCV9_64BIT_STACK,%g1
136          movrz     %g1,0,%g4
137          mov       -1,$sentinel
138          add       %g4,-128,%g4
139#else
140          mov       -1,$sentinel
141          mov       -128,%g4
142#endif
143          sllx      $sentinel,32,$sentinel
144          save      %sp,%g4,%sp
145#ifndef   __arch64__
146          save      %sp,-128,%sp        ! warm it up
147          save      %sp,-128,%sp
148          save      %sp,-128,%sp
149          save      %sp,-128,%sp
150          save      %sp,-128,%sp
151          save      %sp,-128,%sp
152          restore
153          restore
154          restore
155          restore
156          restore
157          restore
158#endif
159          and       %sp,1,%g4
160          or        $sentinel,%fp,%fp
161          or        %g4,$sentinel,$sentinel
162
163          ! copy arguments to global registers
164          mov       %i0,$rp
165          mov       %i1,$ap
166          mov       %i2,$bp
167          mov       %i3,$np
168          ld        [%i4+0],%f1         ! load *n0
169          ld        [%i4+4],%f0
170          fsrc2     %f0,%f60
171___
172
173# load ap[$NUM] ########################################################
174$code.=<<___;
175          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
176___
177for($i=0; $i<14 && $i<$NUM; $i++) {
178my $lo=$i<13?@A[$i+1]:"%o7";
179$code.=<<___;
180          ld        [$ap+$i*8+0],$lo
181          ld        [$ap+$i*8+4],@A[$i]
182          sllx      @A[$i],32,@A[$i]
183          or        $lo,@A[$i],@A[$i]
184___
185}
186for(; $i<$NUM; $i++) {
187my ($hi,$lo)=("%f".2*($i%4),"%f".(2*($i%4)+1));
188$code.=<<___;
189          ld        [$ap+$i*8+0],$lo
190          ld        [$ap+$i*8+4],$hi
191          fsrc2     $hi,@A[$i]
192___
193}
194# load np[$NUM] ########################################################
195$code.=<<___;
196          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
197___
198for($i=0; $i<14 && $i<$NUM; $i++) {
199my $lo=$i<13?@N[$i+1]:"%o7";
200$code.=<<___;
201          ld        [$np+$i*8+0],$lo
202          ld        [$np+$i*8+4],@N[$i]
203          sllx      @N[$i],32,@N[$i]
204          or        $lo,@N[$i],@N[$i]
205___
206}
207$code.=<<___;
208          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
209___
210for(; $i<28 && $i<$NUM; $i++) {
211my $lo=$i<27?@N[$i+1]:"%o7";
212$code.=<<___;
213          ld        [$np+$i*8+0],$lo
214          ld        [$np+$i*8+4],@N[$i]
215          sllx      @N[$i],32,@N[$i]
216          or        $lo,@N[$i],@N[$i]
217___
218}
219$code.=<<___;
220          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
221___
222for(; $i<$NUM; $i++) {
223my $lo=($i<$NUM-1)?@N[$i+1]:"%o7";
224$code.=<<___;
225          ld        [$np+$i*8+0],$lo
226          ld        [$np+$i*8+4],@N[$i]
227          sllx      @N[$i],32,@N[$i]
228          or        $lo,@N[$i],@N[$i]
229___
230}
231$code.=<<___;
232          cmp       $ap,$bp
233          be        SIZE_T_CC,.Lmsquare_$NUM
234          nop
235___
236
237# load bp[$NUM] ########################################################
238$code.=<<___;
239          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
240___
241for($i=0; $i<14 && $i<$NUM; $i++) {
242my $lo=$i<13?@B[$i+1]:"%o7";
243$code.=<<___;
244          ld        [$bp+$i*8+0],$lo
245          ld        [$bp+$i*8+4],@B[$i]
246          sllx      @B[$i],32,@B[$i]
247          or        $lo,@B[$i],@B[$i]
248___
249}
250$code.=<<___;
251          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
252___
253for(; $i<$NUM; $i++) {
254my $lo=($i<$NUM-1)?@B[$i+1]:"%o7";
255$code.=<<___;
256          ld        [$bp+$i*8+0],$lo
257          ld        [$bp+$i*8+4],@B[$i]
258          sllx      @B[$i],32,@B[$i]
259          or        $lo,@B[$i],@B[$i]
260___
261}
262# magic ################################################################
263$code.=<<___;
264          .word     0x81b02920+$NUM-1   ! montmul $NUM-1
265.Lmresume_$NUM:
266          fbu,pn    %fcc3,.Lmabort_$NUM
267#ifndef   __arch64__
268          and       %fp,$sentinel,$sentinel
269          brz,pn    $sentinel,.Lmabort_$NUM
270#endif
271          nop
272#ifdef    __arch64__
273          restore
274          restore
275          restore
276          restore
277          restore
278#else
279          restore;            and       %fp,$sentinel,$sentinel
280          restore;            and       %fp,$sentinel,$sentinel
281          restore;            and       %fp,$sentinel,$sentinel
282          restore;            and       %fp,$sentinel,$sentinel
283           brz,pn   $sentinel,.Lmabort1_$NUM
284          restore
285#endif
286___
287
288# save tp[$NUM] ########################################################
289for($i=0; $i<14 && $i<$NUM; $i++) {
290$code.=<<___;
291          movxtod   @A[$i],@R[$i]
292___
293}
294$code.=<<___;
295#ifdef    __arch64__
296          restore
297#else
298           and      %fp,$sentinel,$sentinel
299          restore
300           and      $sentinel,1,%o7
301           and      %fp,$sentinel,$sentinel
302           srl      %fp,0,%fp           ! just in case?
303           or       %o7,$sentinel,$sentinel
304          brz,a,pn $sentinel,.Lmdone_$NUM
305          mov       0,%i0               ! return failure
306#endif
307___
308for($i=0; $i<12 && $i<$NUM; $i++) {
309@R[$i] =~ /%f([0-9]+)/;
310my $lo = "%f".($1+1);
311$code.=<<___;
312          st        $lo,[$rp+$i*8+0]
313          st        @R[$i],[$rp+$i*8+4]
314___
315}
316for(; $i<$NUM; $i++) {
317my ($hi,$lo)=("%f".2*($i%4),"%f".(2*($i%4)+1));
318$code.=<<___;
319          fsrc2     @R[$i],$hi
320          st        $lo,[$rp+$i*8+0]
321          st        $hi,[$rp+$i*8+4]
322___
323}
324$code.=<<___;
325          mov       1,%i0               ! return success
326.Lmdone_$NUM:
327          ret
328          restore
329
330.Lmabort_$NUM:
331          restore
332          restore
333          restore
334          restore
335          restore
336.Lmabort1_$NUM:
337          restore
338
339          mov       0,%i0               ! return failure
340          ret
341          restore
342
343.align    32
344.Lmsquare_$NUM:
345          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
346          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
347          .word   0x81b02940+$NUM-1     ! montsqr $NUM-1
348          ba        .Lmresume_$NUM
349          nop
350.type     bn_mul_mont_t4_$NUM, #function
351.size     bn_mul_mont_t4_$NUM, .-bn_mul_mont_t4_$NUM
352___
353}
354
355for ($i=8;$i<=32;$i+=8) {
356          &generate_bn_mul_mont_t4($i);
357}
358
359########################################################################
360#
361sub load_ccr {
362my ($ptbl,$pwr,$ccr,$skip_wr)=@_;
363$code.=<<___;
364          srl       $pwr,     2,        %o4
365          and       $pwr,     3,        %o5
366          and       %o4,      7,        %o4
367          sll       %o5,      3,        %o5       ! offset within first cache line
368          add       %o5,      $ptbl,    $ptbl     ! of the pwrtbl
369          or        %g0,      1,        %o5
370          sll       %o5,      %o4,      $ccr
371___
372$code.=<<___        if (!$skip_wr);
373          wr        $ccr,     %g0,      %ccr
374___
375}
376sub load_b_pair {
377my ($pwrtbl,$B0,$B1)=@_;
378
379$code.=<<___;
380          ldx       [$pwrtbl+0*32],     $B0
381          ldx       [$pwrtbl+8*32],     $B1
382          ldx       [$pwrtbl+1*32],     %o4
383          ldx       [$pwrtbl+9*32],     %o5
384          movvs     %icc,     %o4,      $B0
385          ldx       [$pwrtbl+2*32],     %o4
386          movvs     %icc,     %o5,      $B1
387          ldx       [$pwrtbl+10*32],%o5
388          move      %icc,     %o4,      $B0
389          ldx       [$pwrtbl+3*32],     %o4
390          move      %icc,     %o5,      $B1
391          ldx       [$pwrtbl+11*32],%o5
392          movneg    %icc,     %o4,      $B0
393          ldx       [$pwrtbl+4*32],     %o4
394          movneg    %icc,     %o5,      $B1
395          ldx       [$pwrtbl+12*32],%o5
396          movcs     %xcc,     %o4,      $B0
397          ldx       [$pwrtbl+5*32],%o4
398          movcs     %xcc,     %o5,      $B1
399          ldx       [$pwrtbl+13*32],%o5
400          movvs     %xcc,     %o4,      $B0
401          ldx       [$pwrtbl+6*32],     %o4
402          movvs     %xcc,     %o5,      $B1
403          ldx       [$pwrtbl+14*32],%o5
404          move      %xcc,     %o4,      $B0
405          ldx       [$pwrtbl+7*32],     %o4
406          move      %xcc,     %o5,      $B1
407          ldx       [$pwrtbl+15*32],%o5
408          movneg    %xcc,     %o4,      $B0
409          add       $pwrtbl,16*32,      $pwrtbl
410          movneg    %xcc,     %o5,      $B1
411___
412}
413sub load_b {
414my ($pwrtbl,$Bi)=@_;
415
416$code.=<<___;
417          ldx       [$pwrtbl+0*32],     $Bi
418          ldx       [$pwrtbl+1*32],     %o4
419          ldx       [$pwrtbl+2*32],     %o5
420          movvs     %icc,     %o4,      $Bi
421          ldx       [$pwrtbl+3*32],     %o4
422          move      %icc,     %o5,      $Bi
423          ldx       [$pwrtbl+4*32],     %o5
424          movneg    %icc,     %o4,      $Bi
425          ldx       [$pwrtbl+5*32],     %o4
426          movcs     %xcc,     %o5,      $Bi
427          ldx       [$pwrtbl+6*32],     %o5
428          movvs     %xcc,     %o4,      $Bi
429          ldx       [$pwrtbl+7*32],     %o4
430          move      %xcc,     %o5,      $Bi
431          add       $pwrtbl,8*32,       $pwrtbl
432          movneg    %xcc,     %o4,      $Bi
433___
434}
435
436########################################################################
437# int bn_pwr5_mont_t4_$NUM(u64 *tp,const u64 *np,const BN_ULONG *n0,
438#                                const u64 *pwrtbl,int pwr,int stride);
439#
440sub generate_bn_pwr5_mont_t4() {
441my $NUM=shift;
442my ($tp,$np,$pwrtbl,$pwr,$sentinel)=map("%g$_",(1..5));
443
444$code.=<<___;
445.globl    bn_pwr5_mont_t4_$NUM
446.align    32
447bn_pwr5_mont_t4_$NUM:
448#ifdef    __arch64__
449          mov       0,$sentinel
450          mov       -128,%g4
451#elif defined(SPARCV9_64BIT_STACK)
452          SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5)
453          ld        [%g1+0],%g1         ! OPENSSL_sparcv9_P[0]
454          mov       -2047,%g4
455          and       %g1,SPARCV9_64BIT_STACK,%g1
456          movrz     %g1,0,%g4
457          mov       -1,$sentinel
458          add       %g4,-128,%g4
459#else
460          mov       -1,$sentinel
461          mov       -128,%g4
462#endif
463          sllx      $sentinel,32,$sentinel
464          save      %sp,%g4,%sp
465#ifndef   __arch64__
466          save      %sp,-128,%sp        ! warm it up
467          save      %sp,-128,%sp
468          save      %sp,-128,%sp
469          save      %sp,-128,%sp
470          save      %sp,-128,%sp
471          save      %sp,-128,%sp
472          restore
473          restore
474          restore
475          restore
476          restore
477          restore
478#endif
479          and       %sp,1,%g4
480          or        $sentinel,%fp,%fp
481          or        %g4,$sentinel,$sentinel
482
483          ! copy arguments to global registers
484          mov       %i0,$tp
485          mov       %i1,$np
486          ld        [%i2+0],%f1         ! load *n0
487          ld        [%i2+4],%f0
488          mov       %i3,$pwrtbl
489          srl       %i4,%g0,%i4         ! pack last arguments
490          sllx      %i5,32,$pwr
491          or        %i4,$pwr,$pwr
492          fsrc2     %f0,%f60
493___
494
495# load tp[$NUM] ########################################################
496$code.=<<___;
497          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
498___
499for($i=0; $i<14 && $i<$NUM; $i++) {
500$code.=<<___;
501          ldx       [$tp+$i*8],@A[$i]
502___
503}
504for(; $i<$NUM; $i++) {
505$code.=<<___;
506          ldd       [$tp+$i*8],@A[$i]
507___
508}
509# load np[$NUM] ########################################################
510$code.=<<___;
511          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
512___
513for($i=0; $i<14 && $i<$NUM; $i++) {
514$code.=<<___;
515          ldx       [$np+$i*8],@N[$i]
516___
517}
518$code.=<<___;
519          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
520___
521for(; $i<28 && $i<$NUM; $i++) {
522$code.=<<___;
523          ldx       [$np+$i*8],@N[$i]
524___
525}
526$code.=<<___;
527          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
528___
529for(; $i<$NUM; $i++) {
530$code.=<<___;
531          ldx       [$np+$i*8],@N[$i]
532___
533}
534# load pwrtbl[pwr] ########################################################
535$code.=<<___;
536          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
537
538          srlx      $pwr,     32,       %o4                 ! unpack $pwr
539          srl       $pwr,     %g0,      %o5
540          sub       %o4,      5,        %o4
541          mov       $pwrtbl,  %o7
542          sllx      %o4,      32,       $pwr                ! re-pack $pwr
543          or        %o5,      $pwr,     $pwr
544          srl       %o5,      %o4,      %o5
545___
546          &load_ccr("%o7","%o5","%o4");
547$code.=<<___;
548          b         .Lstride_$NUM
549          nop
550.align    16
551.Lstride_$NUM:
552___
553for($i=0; $i<14 && $i<$NUM; $i+=2) {
554          &load_b_pair("%o7",@B[$i],@B[$i+1]);
555}
556$code.=<<___;
557          save      %sp,-128,%sp;                 or        $sentinel,%fp,%fp
558___
559for(; $i<$NUM; $i+=2) {
560          &load_b_pair("%i7",@B[$i],@B[$i+1]);
561}
562$code.=<<___;
563          srax      $pwr,     32,       %o4                 ! unpack $pwr
564          srl       $pwr,     %g0,      %o5
565          sub       %o4,      5,        %o4
566          mov       $pwrtbl,  %i7
567          sllx      %o4,      32,       $pwr                ! re-pack $pwr
568          or        %o5,      $pwr,     $pwr
569          srl       %o5,      %o4,      %o5
570___
571          &load_ccr("%i7","%o5","%o4",1);
572
573# magic ################################################################
574for($i=0; $i<5; $i++) {
575$code.=<<___;
576          .word     0x81b02940+$NUM-1   ! montsqr $NUM-1
577          fbu,pn    %fcc3,.Labort_$NUM
578#ifndef   __arch64__
579          and       %fp,$sentinel,$sentinel
580          brz,pn    $sentinel,.Labort_$NUM
581#endif
582          nop
583___
584}
585$code.=<<___;
586          wr        %o4,      %g0,      %ccr
587          .word     0x81b02920+$NUM-1   ! montmul $NUM-1
588          fbu,pn    %fcc3,.Labort_$NUM
589#ifndef   __arch64__
590          and       %fp,$sentinel,$sentinel
591          brz,pn    $sentinel,.Labort_$NUM
592#endif
593
594          srax      $pwr,     32,       %o4
595#ifdef    __arch64__
596          brgez     %o4,.Lstride_$NUM
597          restore
598          restore
599          restore
600          restore
601          restore
602#else
603          brgez     %o4,.Lstride_$NUM
604          restore;            and       %fp,$sentinel,$sentinel
605          restore;            and       %fp,$sentinel,$sentinel
606          restore;            and       %fp,$sentinel,$sentinel
607          restore;            and       %fp,$sentinel,$sentinel
608           brz,pn   $sentinel,.Labort1_$NUM
609          restore
610#endif
611___
612
613# save tp[$NUM] ########################################################
614for($i=0; $i<14 && $i<$NUM; $i++) {
615$code.=<<___;
616          movxtod   @A[$i],@R[$i]
617___
618}
619$code.=<<___;
620#ifdef    __arch64__
621          restore
622#else
623           and      %fp,$sentinel,$sentinel
624          restore
625           and      $sentinel,1,%o7
626           and      %fp,$sentinel,$sentinel
627           srl      %fp,0,%fp           ! just in case?
628           or       %o7,$sentinel,$sentinel
629          brz,a,pn $sentinel,.Ldone_$NUM
630          mov       0,%i0               ! return failure
631#endif
632___
633for($i=0; $i<$NUM; $i++) {
634$code.=<<___;
635          std       @R[$i],[$tp+$i*8]
636___
637}
638$code.=<<___;
639          mov       1,%i0               ! return success
640.Ldone_$NUM:
641          ret
642          restore
643
644.Labort_$NUM:
645          restore
646          restore
647          restore
648          restore
649          restore
650.Labort1_$NUM:
651          restore
652
653          mov       0,%i0               ! return failure
654          ret
655          restore
656.type     bn_pwr5_mont_t4_$NUM, #function
657.size     bn_pwr5_mont_t4_$NUM, .-bn_pwr5_mont_t4_$NUM
658___
659}
660
661for ($i=8;$i<=32;$i+=8) {
662          &generate_bn_pwr5_mont_t4($i);
663}
664
665{
666########################################################################
667# Fall-back subroutines
668#
669# copy of bn_mul_mont_vis3 adjusted for vectors of 64-bit values
670#
671($n0,$m0,$m1,$lo0,$hi0, $lo1,$hi1,$aj,$alo,$nj,$nlo,$tj)=
672          (map("%g$_",(1..5)),map("%o$_",(0..5,7)));
673
674# int bn_mul_mont(
675$rp="%o0";          # u64 *rp,
676$ap="%o1";          # const u64 *ap,
677$bp="%o2";          # const u64 *bp,
678$np="%o3";          # const u64 *np,
679$n0p="%o4";         # const BN_ULONG *n0,
680$num="%o5";         # int num);         # caller ensures that num is >=3
681$code.=<<___;
682.globl    bn_mul_mont_t4
683.align    32
684bn_mul_mont_t4:
685          add       %sp,      STACK_BIAS,         %g4       ! real top of stack
686          sll       $num,     3,        $num                ! size in bytes
687          add       $num,     63,       %g1
688          andn      %g1,      63,       %g1                 ! buffer size rounded up to 64 bytes
689          sub       %g4,      %g1,      %g1
690          andn      %g1,      63,       %g1                 ! align at 64 byte
691          sub       %g1,      STACK_FRAME,        %g1       ! new top of stack
692          sub       %g1,      %g4,      %g1
693
694          save      %sp,      %g1,      %sp
695___
696#         +-------------------------------+<----- %sp
697#         .                                       .
698#         +-------------------------------+<----- aligned at 64 bytes
699#         | __int64 tmp[0]              |
700#         +-------------------------------+
701#         .                                       .
702#         .                                       .
703#         +-------------------------------+<----- aligned at 64 bytes
704#         .                                       .
705($rp,$ap,$bp,$np,$n0p,$num)=map("%i$_",(0..5));
706($t0,$t1,$t2,$t3,$cnt,$tp,$bufsz)=map("%l$_",(0..7));
707($ovf,$i)=($t0,$t1);
708$code.=<<___;
709          ld        [$n0p+0], $t0       ! pull n0[0..1] value
710          ld        [$n0p+4], $t1
711          add       %sp, STACK_BIAS+STACK_FRAME, $tp
712          ldx       [$bp+0],  $m0       ! m0=bp[0]
713          sllx      $t1,      32,       $n0
714          add       $bp,      8,        $bp
715          or        $t0,      $n0,      $n0
716
717          ldx       [$ap+0],  $aj       ! ap[0]
718
719          mulx      $aj,      $m0,      $lo0      ! ap[0]*bp[0]
720          umulxhi   $aj,      $m0,      $hi0
721
722          ldx       [$ap+8],  $aj       ! ap[1]
723          add       $ap,      16,       $ap
724          ldx       [$np+0],  $nj       ! np[0]
725
726          mulx      $lo0,     $n0,      $m1       ! "tp[0]"*n0
727
728          mulx      $aj,      $m0,      $alo      ! ap[1]*bp[0]
729          umulxhi   $aj,      $m0,      $aj       ! ahi=aj
730
731          mulx      $nj,      $m1,      $lo1      ! np[0]*m1
732          umulxhi   $nj,      $m1,      $hi1
733
734          ldx       [$np+8],  $nj       ! np[1]
735
736          addcc     $lo0,     $lo1,     $lo1
737          add       $np,      16,       $np
738          addxc     %g0,      $hi1,     $hi1
739
740          mulx      $nj,      $m1,      $nlo      ! np[1]*m1
741          umulxhi   $nj,      $m1,      $nj       ! nhi=nj
742
743          ba        .L1st
744          sub       $num,     24,       $cnt      ! cnt=num-3
745
746.align    16
747.L1st:
748          addcc     $alo,     $hi0,     $lo0
749          addxc     $aj,      %g0,      $hi0
750
751          ldx       [$ap+0],  $aj       ! ap[j]
752          addcc     $nlo,     $hi1,     $lo1
753          add       $ap,      8,        $ap
754          addxc     $nj,      %g0,      $hi1      ! nhi=nj
755
756          ldx       [$np+0],  $nj       ! np[j]
757          mulx      $aj,      $m0,      $alo      ! ap[j]*bp[0]
758          add       $np,      8,        $np
759          umulxhi   $aj,      $m0,      $aj       ! ahi=aj
760
761          mulx      $nj,      $m1,      $nlo      ! np[j]*m1
762          addcc     $lo0,     $lo1,     $lo1      ! np[j]*m1+ap[j]*bp[0]
763          umulxhi   $nj,      $m1,      $nj       ! nhi=nj
764          addxc     %g0,      $hi1,     $hi1
765          stxa      $lo1,     [$tp]0xe2 ! tp[j-1]
766          add       $tp,      8,        $tp       ! tp++
767
768          brnz,pt   $cnt,     .L1st
769          sub       $cnt,     8,        $cnt      ! j--
770!.L1st
771          addcc     $alo,     $hi0,     $lo0
772          addxc     $aj,      %g0,      $hi0      ! ahi=aj
773
774          addcc     $nlo,     $hi1,     $lo1
775          addxc     $nj,      %g0,      $hi1
776          addcc     $lo0,     $lo1,     $lo1      ! np[j]*m1+ap[j]*bp[0]
777          addxc     %g0,      $hi1,     $hi1
778          stxa      $lo1,     [$tp]0xe2 ! tp[j-1]
779          add       $tp,      8,        $tp
780
781          addcc     $hi0,     $hi1,     $hi1
782          addxc     %g0,      %g0,      $ovf      ! upmost overflow bit
783          stxa      $hi1,     [$tp]0xe2
784          add       $tp,      8,        $tp
785
786          ba        .Louter
787          sub       $num,     16,       $i        ! i=num-2
788
789.align    16
790.Louter:
791          ldx       [$bp+0],  $m0       ! m0=bp[i]
792          add       $bp,      8,        $bp
793
794          sub       $ap,      $num,     $ap       ! rewind
795          sub       $np,      $num,     $np
796          sub       $tp,      $num,     $tp
797
798          ldx       [$ap+0],  $aj       ! ap[0]
799          ldx       [$np+0],  $nj       ! np[0]
800
801          mulx      $aj,      $m0,      $lo0      ! ap[0]*bp[i]
802          ldx       [$tp],              $tj       ! tp[0]
803          umulxhi   $aj,      $m0,      $hi0
804          ldx       [$ap+8],  $aj       ! ap[1]
805          addcc     $lo0,     $tj,      $lo0      ! ap[0]*bp[i]+tp[0]
806          mulx      $aj,      $m0,      $alo      ! ap[1]*bp[i]
807          addxc     %g0,      $hi0,     $hi0
808          mulx      $lo0,     $n0,      $m1       ! tp[0]*n0
809          umulxhi   $aj,      $m0,      $aj       ! ahi=aj
810          mulx      $nj,      $m1,      $lo1      ! np[0]*m1
811          add       $ap,      16,       $ap
812          umulxhi   $nj,      $m1,      $hi1
813          ldx       [$np+8],  $nj       ! np[1]
814          add       $np,      16,       $np
815          addcc     $lo1,     $lo0,     $lo1
816          mulx      $nj,      $m1,      $nlo      ! np[1]*m1
817          addxc     %g0,      $hi1,     $hi1
818          umulxhi   $nj,      $m1,      $nj       ! nhi=nj
819
820          ba        .Linner
821          sub       $num,     24,       $cnt      ! cnt=num-3
822.align    16
823.Linner:
824          addcc     $alo,     $hi0,     $lo0
825          ldx       [$tp+8],  $tj       ! tp[j]
826          addxc     $aj,      %g0,      $hi0      ! ahi=aj
827          ldx       [$ap+0],  $aj       ! ap[j]
828          add       $ap,      8,        $ap
829          addcc     $nlo,     $hi1,     $lo1
830          mulx      $aj,      $m0,      $alo      ! ap[j]*bp[i]
831          addxc     $nj,      %g0,      $hi1      ! nhi=nj
832          ldx       [$np+0],  $nj       ! np[j]
833          add       $np,      8,        $np
834          umulxhi   $aj,      $m0,      $aj       ! ahi=aj
835          addcc     $lo0,     $tj,      $lo0      ! ap[j]*bp[i]+tp[j]
836          mulx      $nj,      $m1,      $nlo      ! np[j]*m1
837          addxc     %g0,      $hi0,     $hi0
838          umulxhi   $nj,      $m1,      $nj       ! nhi=nj
839          addcc     $lo1,     $lo0,     $lo1      ! np[j]*m1+ap[j]*bp[i]+tp[j]
840          addxc     %g0,      $hi1,     $hi1
841          stx       $lo1,     [$tp]               ! tp[j-1]
842          add       $tp,      8,        $tp
843          brnz,pt   $cnt,     .Linner
844          sub       $cnt,     8,        $cnt
845!.Linner
846          ldx       [$tp+8],  $tj       ! tp[j]
847          addcc     $alo,     $hi0,     $lo0
848          addxc     $aj,      %g0,      $hi0      ! ahi=aj
849          addcc     $lo0,     $tj,      $lo0      ! ap[j]*bp[i]+tp[j]
850          addxc     %g0,      $hi0,     $hi0
851
852          addcc     $nlo,     $hi1,     $lo1
853          addxc     $nj,      %g0,      $hi1      ! nhi=nj
854          addcc     $lo1,     $lo0,     $lo1      ! np[j]*m1+ap[j]*bp[i]+tp[j]
855          addxc     %g0,      $hi1,     $hi1
856          stx       $lo1,     [$tp]               ! tp[j-1]
857
858          subcc     %g0,      $ovf,     %g0       ! move upmost overflow to CCR.xcc
859          addxccc   $hi1,     $hi0,     $hi1
860          addxc     %g0,      %g0,      $ovf
861          stx       $hi1,     [$tp+8]
862          add       $tp,      16,       $tp
863
864          brnz,pt   $i,       .Louter
865          sub       $i,       8,        $i
866
867          sub       $ap,      $num,     $ap       ! rewind
868          sub       $np,      $num,     $np
869          sub       $tp,      $num,     $tp
870          ba        .Lsub
871          subcc     $num,     8,        $cnt      ! cnt=num-1 and clear CCR.xcc
872
873.align    16
874.Lsub:
875          ldx       [$tp],              $tj
876          add       $tp,      8,        $tp
877          ldx       [$np+0],  $nj
878          add       $np,      8,        $np
879          subccc    $tj,      $nj,      $t2       ! tp[j]-np[j]
880          srlx      $tj,      32,       $tj
881          srlx      $nj,      32,       $nj
882          subccc    $tj,      $nj,      $t3
883          add       $rp,      8,        $rp
884          st        $t2,      [$rp-4]             ! reverse order
885          st        $t3,      [$rp-8]
886          brnz,pt   $cnt,     .Lsub
887          sub       $cnt,     8,        $cnt
888
889          sub       $np,      $num,     $np       ! rewind
890          sub       $tp,      $num,     $tp
891          sub       $rp,      $num,     $rp
892
893          subccc    $ovf,     %g0,      $ovf      ! handle upmost overflow bit
894          ba        .Lcopy
895          sub       $num,     8,        $cnt
896
897.align    16
898.Lcopy:                                           ! conditional copy
899          ldx       [$tp],              $tj
900          ldx       [$rp+0],  $t2
901          stx       %g0,      [$tp]               ! zap
902          add       $tp,      8,        $tp
903          movcs     %icc,     $tj,      $t2
904          stx       $t2,      [$rp+0]
905          add       $rp,      8,        $rp
906          brnz      $cnt,     .Lcopy
907          sub       $cnt,     8,        $cnt
908
909          mov       1,        %o0
910          ret
911          restore
912.type     bn_mul_mont_t4, #function
913.size     bn_mul_mont_t4, .-bn_mul_mont_t4
914___
915
916# int bn_mul_mont_gather5(
917$rp="%o0";          # u64 *rp,
918$ap="%o1";          # const u64 *ap,
919$bp="%o2";          # const u64 *pwrtbl,
920$np="%o3";          # const u64 *np,
921$n0p="%o4";         # const BN_ULONG *n0,
922$num="%o5";         # int num,          # caller ensures that num is >=3
923                    # int power);
924$code.=<<___;
925.globl    bn_mul_mont_gather5_t4
926.align    32
927bn_mul_mont_gather5_t4:
928          add       %sp,      STACK_BIAS,         %g4       ! real top of stack
929          sll       $num,     3,        $num                ! size in bytes
930          add       $num,     63,       %g1
931          andn      %g1,      63,       %g1                 ! buffer size rounded up to 64 bytes
932          sub       %g4,      %g1,      %g1
933          andn      %g1,      63,       %g1                 ! align at 64 byte
934          sub       %g1,      STACK_FRAME,        %g1       ! new top of stack
935          sub       %g1,      %g4,      %g1
936          LDPTR     [%sp+STACK_7thARG], %g4       ! load power, 7th argument
937
938          save      %sp,      %g1,      %sp
939___
940#         +-------------------------------+<----- %sp
941#         .                                       .
942#         +-------------------------------+<----- aligned at 64 bytes
943#         | __int64 tmp[0]              |
944#         +-------------------------------+
945#         .                                       .
946#         .                                       .
947#         +-------------------------------+<----- aligned at 64 bytes
948#         .                                       .
949($rp,$ap,$bp,$np,$n0p,$num)=map("%i$_",(0..5));
950($t0,$t1,$t2,$t3,$cnt,$tp,$bufsz,$ccr)=map("%l$_",(0..7));
951($ovf,$i)=($t0,$t1);
952          &load_ccr($bp,"%g4",$ccr);
953          &load_b($bp,$m0,"%o7");                 # m0=bp[0]
954
955$code.=<<___;
956          ld        [$n0p+0], $t0       ! pull n0[0..1] value
957          ld        [$n0p+4], $t1
958          add       %sp, STACK_BIAS+STACK_FRAME, $tp
959          sllx      $t1,      32,       $n0
960          or        $t0,      $n0,      $n0
961
962          ldx       [$ap+0],  $aj       ! ap[0]
963
964          mulx      $aj,      $m0,      $lo0      ! ap[0]*bp[0]
965          umulxhi   $aj,      $m0,      $hi0
966
967          ldx       [$ap+8],  $aj       ! ap[1]
968          add       $ap,      16,       $ap
969          ldx       [$np+0],  $nj       ! np[0]
970
971          mulx      $lo0,     $n0,      $m1       ! "tp[0]"*n0
972
973          mulx      $aj,      $m0,      $alo      ! ap[1]*bp[0]
974          umulxhi   $aj,      $m0,      $aj       ! ahi=aj
975
976          mulx      $nj,      $m1,      $lo1      ! np[0]*m1
977          umulxhi   $nj,      $m1,      $hi1
978
979          ldx       [$np+8],  $nj       ! np[1]
980
981          addcc     $lo0,     $lo1,     $lo1
982          add       $np,      16,       $np
983          addxc     %g0,      $hi1,     $hi1
984
985          mulx      $nj,      $m1,      $nlo      ! np[1]*m1
986          umulxhi   $nj,      $m1,      $nj       ! nhi=nj
987
988          ba        .L1st_g5
989          sub       $num,     24,       $cnt      ! cnt=num-3
990
991.align    16
992.L1st_g5:
993          addcc     $alo,     $hi0,     $lo0
994          addxc     $aj,      %g0,      $hi0
995
996          ldx       [$ap+0],  $aj       ! ap[j]
997          addcc     $nlo,     $hi1,     $lo1
998          add       $ap,      8,        $ap
999          addxc     $nj,      %g0,      $hi1      ! nhi=nj
1000
1001          ldx       [$np+0],  $nj       ! np[j]
1002          mulx      $aj,      $m0,      $alo      ! ap[j]*bp[0]
1003          add       $np,      8,        $np
1004          umulxhi   $aj,      $m0,      $aj       ! ahi=aj
1005
1006          mulx      $nj,      $m1,      $nlo      ! np[j]*m1
1007          addcc     $lo0,     $lo1,     $lo1      ! np[j]*m1+ap[j]*bp[0]
1008          umulxhi   $nj,      $m1,      $nj       ! nhi=nj
1009          addxc     %g0,      $hi1,     $hi1
1010          stxa      $lo1,     [$tp]0xe2 ! tp[j-1]
1011          add       $tp,      8,        $tp       ! tp++
1012
1013          brnz,pt   $cnt,     .L1st_g5
1014          sub       $cnt,     8,        $cnt      ! j--
1015!.L1st_g5
1016          addcc     $alo,     $hi0,     $lo0
1017          addxc     $aj,      %g0,      $hi0      ! ahi=aj
1018
1019          addcc     $nlo,     $hi1,     $lo1
1020          addxc     $nj,      %g0,      $hi1
1021          addcc     $lo0,     $lo1,     $lo1      ! np[j]*m1+ap[j]*bp[0]
1022          addxc     %g0,      $hi1,     $hi1
1023          stxa      $lo1,     [$tp]0xe2 ! tp[j-1]
1024          add       $tp,      8,        $tp
1025
1026          addcc     $hi0,     $hi1,     $hi1
1027          addxc     %g0,      %g0,      $ovf      ! upmost overflow bit
1028          stxa      $hi1,     [$tp]0xe2
1029          add       $tp,      8,        $tp
1030
1031          ba        .Louter_g5
1032          sub       $num,     16,       $i        ! i=num-2
1033
1034.align    16
1035.Louter_g5:
1036          wr        $ccr,     %g0,      %ccr
1037___
1038          &load_b($bp,$m0);             # m0=bp[i]
1039$code.=<<___;
1040          sub       $ap,      $num,     $ap       ! rewind
1041          sub       $np,      $num,     $np
1042          sub       $tp,      $num,     $tp
1043
1044          ldx       [$ap+0],  $aj       ! ap[0]
1045          ldx       [$np+0],  $nj       ! np[0]
1046
1047          mulx      $aj,      $m0,      $lo0      ! ap[0]*bp[i]
1048          ldx       [$tp],              $tj       ! tp[0]
1049          umulxhi   $aj,      $m0,      $hi0
1050          ldx       [$ap+8],  $aj       ! ap[1]
1051          addcc     $lo0,     $tj,      $lo0      ! ap[0]*bp[i]+tp[0]
1052          mulx      $aj,      $m0,      $alo      ! ap[1]*bp[i]
1053          addxc     %g0,      $hi0,     $hi0
1054          mulx      $lo0,     $n0,      $m1       ! tp[0]*n0
1055          umulxhi   $aj,      $m0,      $aj       ! ahi=aj
1056          mulx      $nj,      $m1,      $lo1      ! np[0]*m1
1057          add       $ap,      16,       $ap
1058          umulxhi   $nj,      $m1,      $hi1
1059          ldx       [$np+8],  $nj       ! np[1]
1060          add       $np,      16,       $np
1061          addcc     $lo1,     $lo0,     $lo1
1062          mulx      $nj,      $m1,      $nlo      ! np[1]*m1
1063          addxc     %g0,      $hi1,     $hi1
1064          umulxhi   $nj,      $m1,      $nj       ! nhi=nj
1065
1066          ba        .Linner_g5
1067          sub       $num,     24,       $cnt      ! cnt=num-3
1068.align    16
1069.Linner_g5:
1070          addcc     $alo,     $hi0,     $lo0
1071          ldx       [$tp+8],  $tj       ! tp[j]
1072          addxc     $aj,      %g0,      $hi0      ! ahi=aj
1073          ldx       [$ap+0],  $aj       ! ap[j]
1074          add       $ap,      8,        $ap
1075          addcc     $nlo,     $hi1,     $lo1
1076          mulx      $aj,      $m0,      $alo      ! ap[j]*bp[i]
1077          addxc     $nj,      %g0,      $hi1      ! nhi=nj
1078          ldx       [$np+0],  $nj       ! np[j]
1079          add       $np,      8,        $np
1080          umulxhi   $aj,      $m0,      $aj       ! ahi=aj
1081          addcc     $lo0,     $tj,      $lo0      ! ap[j]*bp[i]+tp[j]
1082          mulx      $nj,      $m1,      $nlo      ! np[j]*m1
1083          addxc     %g0,      $hi0,     $hi0
1084          umulxhi   $nj,      $m1,      $nj       ! nhi=nj
1085          addcc     $lo1,     $lo0,     $lo1      ! np[j]*m1+ap[j]*bp[i]+tp[j]
1086          addxc     %g0,      $hi1,     $hi1
1087          stx       $lo1,     [$tp]               ! tp[j-1]
1088          add       $tp,      8,        $tp
1089          brnz,pt   $cnt,     .Linner_g5
1090          sub       $cnt,     8,        $cnt
1091!.Linner_g5
1092          ldx       [$tp+8],  $tj       ! tp[j]
1093          addcc     $alo,     $hi0,     $lo0
1094          addxc     $aj,      %g0,      $hi0      ! ahi=aj
1095          addcc     $lo0,     $tj,      $lo0      ! ap[j]*bp[i]+tp[j]
1096          addxc     %g0,      $hi0,     $hi0
1097
1098          addcc     $nlo,     $hi1,     $lo1
1099          addxc     $nj,      %g0,      $hi1      ! nhi=nj
1100          addcc     $lo1,     $lo0,     $lo1      ! np[j]*m1+ap[j]*bp[i]+tp[j]
1101          addxc     %g0,      $hi1,     $hi1
1102          stx       $lo1,     [$tp]               ! tp[j-1]
1103
1104          subcc     %g0,      $ovf,     %g0       ! move upmost overflow to CCR.xcc
1105          addxccc   $hi1,     $hi0,     $hi1
1106          addxc     %g0,      %g0,      $ovf
1107          stx       $hi1,     [$tp+8]
1108          add       $tp,      16,       $tp
1109
1110          brnz,pt   $i,       .Louter_g5
1111          sub       $i,       8,        $i
1112
1113          sub       $ap,      $num,     $ap       ! rewind
1114          sub       $np,      $num,     $np
1115          sub       $tp,      $num,     $tp
1116          ba        .Lsub_g5
1117          subcc     $num,     8,        $cnt      ! cnt=num-1 and clear CCR.xcc
1118
1119.align    16
1120.Lsub_g5:
1121          ldx       [$tp],              $tj
1122          add       $tp,      8,        $tp
1123          ldx       [$np+0],  $nj
1124          add       $np,      8,        $np
1125          subccc    $tj,      $nj,      $t2       ! tp[j]-np[j]
1126          srlx      $tj,      32,       $tj
1127          srlx      $nj,      32,       $nj
1128          subccc    $tj,      $nj,      $t3
1129          add       $rp,      8,        $rp
1130          st        $t2,      [$rp-4]             ! reverse order
1131          st        $t3,      [$rp-8]
1132          brnz,pt   $cnt,     .Lsub_g5
1133          sub       $cnt,     8,        $cnt
1134
1135          sub       $np,      $num,     $np       ! rewind
1136          sub       $tp,      $num,     $tp
1137          sub       $rp,      $num,     $rp
1138
1139          subccc    $ovf,     %g0,      $ovf      ! handle upmost overflow bit
1140          ba        .Lcopy_g5
1141          sub       $num,     8,        $cnt
1142
1143.align    16
1144.Lcopy_g5:                                        ! conditional copy
1145          ldx       [$tp],              $tj
1146          ldx       [$rp+0],  $t2
1147          stx       %g0,      [$tp]               ! zap
1148          add       $tp,      8,        $tp
1149          movcs     %icc,     $tj,      $t2
1150          stx       $t2,      [$rp+0]
1151          add       $rp,      8,        $rp
1152          brnz      $cnt,     .Lcopy_g5
1153          sub       $cnt,     8,        $cnt
1154
1155          mov       1,        %o0
1156          ret
1157          restore
1158.type     bn_mul_mont_gather5_t4, #function
1159.size     bn_mul_mont_gather5_t4, .-bn_mul_mont_gather5_t4
1160___
1161}
1162
1163$code.=<<___;
1164.globl    bn_flip_t4
1165.align    32
1166bn_flip_t4:
1167.Loop_flip:
1168          ld        [%o1+0],  %o4
1169          sub       %o2,      1,        %o2
1170          ld        [%o1+4],  %o5
1171          add       %o1,      8,        %o1
1172          st        %o5,      [%o0+0]
1173          st        %o4,      [%o0+4]
1174          brnz      %o2,      .Loop_flip
1175          add       %o0,      8,        %o0
1176          retl
1177          nop
1178.type     bn_flip_t4, #function
1179.size     bn_flip_t4, .-bn_flip_t4
1180
1181.globl    bn_flip_n_scatter5_t4
1182.align    32
1183bn_flip_n_scatter5_t4:
1184          sll       %o3,      3,        %o3
1185          srl       %o1,      1,        %o1
1186          add       %o3,      %o2,      %o2       ! &pwrtbl[pwr]
1187          sub       %o1,      1,        %o1
1188.Loop_flip_n_scatter5:
1189          ld        [%o0+0],  %o4       ! inp[i]
1190          ld        [%o0+4],  %o5
1191          add       %o0,      8,        %o0
1192          sllx      %o5,      32,       %o5
1193          or        %o4,      %o5,      %o5
1194          stx       %o5,      [%o2]
1195          add       %o2,      32*8,     %o2
1196          brnz      %o1,      .Loop_flip_n_scatter5
1197          sub       %o1,      1,        %o1
1198          retl
1199          nop
1200.type     bn_flip_n_scatter5_t4, #function
1201.size     bn_flip_n_scatter5_t4, .-bn_flip_n_scatter5_t4
1202
1203.globl    bn_gather5_t4
1204.align    32
1205bn_gather5_t4:
1206___
1207          &load_ccr("%o2","%o3","%g1");
1208$code.=<<___;
1209          sub       %o1,      1,        %o1
1210.Loop_gather5:
1211___
1212          &load_b("%o2","%g1");
1213$code.=<<___;
1214          stx       %g1,      [%o0]
1215          add       %o0,      8,        %o0
1216          brnz      %o1,      .Loop_gather5
1217          sub       %o1,      1,        %o1
1218
1219          retl
1220          nop
1221.type     bn_gather5_t4, #function
1222.size     bn_gather5_t4, .-bn_gather5_t4
1223
1224.asciz    "Montgomery Multiplication for SPARC T4, David S. Miller, Andy Polyakov"
1225.align    4
1226___
1227
1228&emit_assembler();
1229
1230close STDOUT or die "error closing STDOUT: $!";
1231