gonum.org/v1/gonum@v0.14.0/internal/asm/c64/dotcunitary_amd64.s (about)

     1  // Copyright ©2017 The Gonum Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // +build !noasm,!gccgo,!safe
     6  
     7  #include "textflag.h"
     8  
     9  #define MOVSLDUP_XPTR_IDX_8__X3    LONG $0x1C120FF3; BYTE $0xC6 // MOVSLDUP (SI)(AX*8), X3
    10  #define MOVSLDUP_16_XPTR_IDX_8__X5    LONG $0x6C120FF3; WORD $0x10C6 // MOVSLDUP 16(SI)(AX*8), X5
    11  #define MOVSLDUP_32_XPTR_IDX_8__X7    LONG $0x7C120FF3; WORD $0x20C6 // MOVSLDUP 32(SI)(AX*8), X7
    12  #define MOVSLDUP_48_XPTR_IDX_8__X9    LONG $0x120F44F3; WORD $0xC64C; BYTE $0x30 // MOVSLDUP 48(SI)(AX*8), X9
    13  
    14  #define MOVSHDUP_XPTR_IDX_8__X2    LONG $0x14160FF3; BYTE $0xC6 // MOVSHDUP (SI)(AX*8), X2
    15  #define MOVSHDUP_16_XPTR_IDX_8__X4    LONG $0x64160FF3; WORD $0x10C6 // MOVSHDUP 16(SI)(AX*8), X4
    16  #define MOVSHDUP_32_XPTR_IDX_8__X6    LONG $0x74160FF3; WORD $0x20C6 // MOVSHDUP 32(SI)(AX*8), X6
    17  #define MOVSHDUP_48_XPTR_IDX_8__X8    LONG $0x160F44F3; WORD $0xC644; BYTE $0x30 // MOVSHDUP 48(SI)(AX*8), X8
    18  
    19  #define MOVSHDUP_X3_X2    LONG $0xD3160FF3 // MOVSHDUP X3, X2
    20  #define MOVSLDUP_X3_X3    LONG $0xDB120FF3 // MOVSLDUP X3, X3
    21  
    22  #define ADDSUBPS_X2_X3    LONG $0xDAD00FF2 // ADDSUBPS X2, X3
    23  #define ADDSUBPS_X4_X5    LONG $0xECD00FF2 // ADDSUBPS X4, X5
    24  #define ADDSUBPS_X6_X7    LONG $0xFED00FF2 // ADDSUBPS X6, X7
    25  #define ADDSUBPS_X8_X9    LONG $0xD00F45F2; BYTE $0xC8 // ADDSUBPS X8, X9
    26  
    27  #define X_PTR SI
    28  #define Y_PTR DI
    29  #define LEN CX
    30  #define TAIL BX
    31  #define SUM X0
    32  #define P_SUM X1
    33  #define IDX AX
    34  #define I_IDX DX
    35  #define NEG1 X15
    36  #define P_NEG1 X14
    37  
    38  // func DotcUnitary(x, y []complex64) (sum complex64)
    39  TEXT ·DotcUnitary(SB), NOSPLIT, $0
    40  	MOVQ    x_base+0(FP), X_PTR  // X_PTR = &x
    41  	MOVQ    y_base+24(FP), Y_PTR // Y_PTR = &y
    42  	PXOR    SUM, SUM             // SUM = 0
    43  	PXOR    P_SUM, P_SUM         // P_SUM = 0
    44  	MOVQ    x_len+8(FP), LEN     // LEN = min( len(x), len(y) )
    45  	CMPQ    y_len+32(FP), LEN
    46  	CMOVQLE y_len+32(FP), LEN
    47  	CMPQ    LEN, $0              // if LEN == 0 { return }
    48  	JE      dotc_end
    49  	XORQ    IDX, IDX             // i = 0
    50  	MOVSS   $(-1.0), NEG1
    51  	SHUFPS  $0, NEG1, NEG1       // { -1, -1, -1, -1 }
    52  
    53  	MOVQ X_PTR, DX
    54  	ANDQ $15, DX      // DX = &x & 15
    55  	JZ   dotc_aligned // if DX == 0 { goto dotc_aligned }
    56  
    57  	MOVSD  (X_PTR)(IDX*8), X3  // X_i     = { imag(x[i]), real(x[i]) }
    58  	MOVSHDUP_X3_X2             // X_(i-1) = { imag(x[i]), imag(x[i]) }
    59  	MOVSLDUP_X3_X3             // X_i     = { real(x[i]), real(x[i]) }
    60  	MOVSD  (Y_PTR)(IDX*8), X10 // X_j     = { imag(y[i]), real(y[i]) }
    61  	MULPS  NEG1, X2            // X_(i-1) = { -imag(x[i]), imag(x[i]) }
    62  	MULPS  X10, X3             // X_i     = { imag(y[i]) * real(x[i]), real(y[i]) * real(x[i]) }
    63  	SHUFPS $0x1, X10, X10      // X_j     = { real(y[i]), imag(y[i]) }
    64  	MULPS  X10, X2             // X_(i-1) = { real(y[i]) * imag(x[i]), imag(y[i]) * imag(x[i]) }
    65  
    66  	// X_i = {
    67  	//	imag(result[i]):  imag(y[i])*real(x[i]) + real(y[i])*imag(x[i]),
    68  	//	real(result[i]):  real(y[i])*real(x[i]) - imag(y[i])*imag(x[i]) }
    69  	ADDSUBPS_X2_X3
    70  
    71  	MOVAPS X3, SUM  // SUM = X_i
    72  	INCQ   IDX      // IDX++
    73  	DECQ   LEN      // LEN--
    74  	JZ     dotc_ret // if LEN == 0 { goto dotc_ret }
    75  
    76  dotc_aligned:
    77  	MOVQ   LEN, TAIL
    78  	ANDQ   $7, TAIL     // TAIL = LEN % 8
    79  	SHRQ   $3, LEN      // LEN = floor( LEN / 8 )
    80  	JZ     dotc_tail    // if LEN == 0 { return }
    81  	MOVUPS NEG1, P_NEG1 // Copy NEG1 for pipelining
    82  
    83  dotc_loop: // do {
    84  	MOVSLDUP_XPTR_IDX_8__X3    // X_i = { real(x[i]), real(x[i]), real(x[i+1]), real(x[i+1]) }
    85  	MOVSLDUP_16_XPTR_IDX_8__X5
    86  	MOVSLDUP_32_XPTR_IDX_8__X7
    87  	MOVSLDUP_48_XPTR_IDX_8__X9
    88  
    89  	MOVSHDUP_XPTR_IDX_8__X2    // X_(i-1) = { imag(x[i]), imag(x[i]), imag(x[i+1]), imag(x[i+1]) }
    90  	MOVSHDUP_16_XPTR_IDX_8__X4
    91  	MOVSHDUP_32_XPTR_IDX_8__X6
    92  	MOVSHDUP_48_XPTR_IDX_8__X8
    93  
    94  	// X_j = { imag(y[i]), real(y[i]), imag(y[i+1]), real(y[i+1]) }
    95  	MOVUPS (Y_PTR)(IDX*8), X10
    96  	MOVUPS 16(Y_PTR)(IDX*8), X11
    97  	MOVUPS 32(Y_PTR)(IDX*8), X12
    98  	MOVUPS 48(Y_PTR)(IDX*8), X13
    99  
   100  	// X_(i-1) = { -imag(x[i]), -imag(x[i]), -imag(x[i]+1), -imag(x[i]+1) }
   101  	MULPS NEG1, X2
   102  	MULPS P_NEG1, X4
   103  	MULPS NEG1, X6
   104  	MULPS P_NEG1, X8
   105  
   106  	// X_i     = {  imag(y[i])   * real(x[i]),   real(y[i])   * real(x[i]),
   107  	// 		imag(y[i+1]) * real(x[i+1]), real(y[i+1]) * real(x[i+1])  }
   108  	MULPS X10, X3
   109  	MULPS X11, X5
   110  	MULPS X12, X7
   111  	MULPS X13, X9
   112  
   113  	// X_j = { real(y[i]), imag(y[i]), real(y[i+1]), imag(y[i+1]) }
   114  	SHUFPS $0xB1, X10, X10
   115  	SHUFPS $0xB1, X11, X11
   116  	SHUFPS $0xB1, X12, X12
   117  	SHUFPS $0xB1, X13, X13
   118  
   119  	// X_(i-1) = {  real(y[i])   * imag(x[i]),   imag(y[i])   * imag(x[i]),
   120  	//		real(y[i+1]) * imag(x[i+1]), imag(y[i+1]) * imag(x[i+1])  }
   121  	MULPS X10, X2
   122  	MULPS X11, X4
   123  	MULPS X12, X6
   124  	MULPS X13, X8
   125  
   126  	// X_i = {
   127  	//	imag(result[i]):   imag(y[i])   * real(x[i])   + real(y[i])   * imag(x[i]),
   128  	//	real(result[i]):   real(y[i])   * real(x[i])   - imag(y[i])   * imag(x[i]),
   129  	//	imag(result[i+1]): imag(y[i+1]) * real(x[i+1]) + real(y[i+1]) * imag(x[i+1]),
   130  	//	real(result[i+1]): real(y[i+1]) * real(x[i+1]) - imag(y[i+1]) * imag(x[i+1]),
   131  	//  }
   132  	ADDSUBPS_X2_X3
   133  	ADDSUBPS_X4_X5
   134  	ADDSUBPS_X6_X7
   135  	ADDSUBPS_X8_X9
   136  
   137  	// SUM += X_i
   138  	ADDPS X3, SUM
   139  	ADDPS X5, P_SUM
   140  	ADDPS X7, SUM
   141  	ADDPS X9, P_SUM
   142  
   143  	ADDQ $8, IDX   // IDX += 8
   144  	DECQ LEN
   145  	JNZ  dotc_loop // } while --LEN > 0
   146  
   147  	ADDPS SUM, P_SUM // P_SUM = { P_SUM[1] + SUM[1], P_SUM[0] + SUM[0] }
   148  	XORPS SUM, SUM   // SUM = 0
   149  
   150  	CMPQ TAIL, $0 // if TAIL == 0 { return }
   151  	JE   dotc_end
   152  
   153  dotc_tail:
   154  	MOVQ TAIL, LEN
   155  	SHRQ $1, LEN       // LEN = floor( LEN / 2 )
   156  	JZ   dotc_tail_one // if LEN == 0 { goto dotc_tail_one }
   157  
   158  dotc_tail_two: // do {
   159  	MOVSLDUP_XPTR_IDX_8__X3    // X_i = { real(x[i]), real(x[i]), real(x[i+1]), real(x[i+1]) }
   160  	MOVSHDUP_XPTR_IDX_8__X2    // X_(i-1) = { imag(x[i]), imag(x[i]), imag(x[i]+1), imag(x[i]+1) }
   161  	MOVUPS (Y_PTR)(IDX*8), X10 // X_j = { imag(y[i]), real(y[i]) }
   162  	MULPS  NEG1, X2            // X_(i-1) = { -imag(x[i]), imag(x[i]) }
   163  	MULPS  X10, X3             // X_i = { imag(y[i]) * real(x[i]), real(y[i]) * real(x[i]) }
   164  	SHUFPS $0xB1, X10, X10     // X_j = { real(y[i]), imag(y[i]) }
   165  	MULPS  X10, X2             // X_(i-1) = { real(y[i]) * imag(x[i]), imag(y[i]) * imag(x[i]) }
   166  
   167  	// X_i = {
   168  	//	imag(result[i]):  imag(y[i])*real(x[i]) + real(y[i])*imag(x[i]),
   169  	//	real(result[i]):  real(y[i])*real(x[i]) - imag(y[i])*imag(x[i]) }
   170  	ADDSUBPS_X2_X3
   171  
   172  	ADDPS X3, SUM // SUM += X_i
   173  
   174  	ADDQ $2, IDX       // IDX += 2
   175  	DECQ LEN
   176  	JNZ  dotc_tail_two // } while --LEN > 0
   177  
   178  	ADDPS SUM, P_SUM // P_SUM = { P_SUM[1] + SUM[1], P_SUM[0] + SUM[0] }
   179  	XORPS SUM, SUM   // SUM = 0
   180  
   181  	ANDQ $1, TAIL
   182  	JZ   dotc_end
   183  
   184  dotc_tail_one:
   185  	MOVSD  (X_PTR)(IDX*8), X3  // X_i = { imag(x[i]), real(x[i]) }
   186  	MOVSHDUP_X3_X2             // X_(i-1) = { imag(x[i]), imag(x[i]) }
   187  	MOVSLDUP_X3_X3             // X_i = { real(x[i]), real(x[i]) }
   188  	MOVSD  (Y_PTR)(IDX*8), X10 // X_j = { imag(y[i]), real(y[i]) }
   189  	MULPS  NEG1, X2            // X_(i-1) = { -imag(x[i]), imag(x[i]) }
   190  	MULPS  X10, X3             // X_i = { imag(y[i]) * real(x[i]), real(y[i]) * real(x[i]) }
   191  	SHUFPS $0x1, X10, X10      // X_j = { real(y[i]), imag(y[i]) }
   192  	MULPS  X10, X2             // X_(i-1) = { real(y[i]) * imag(x[i]), imag(y[i]) * imag(x[i]) }
   193  
   194  	// X_i = {
   195  	//	imag(result[i]):  imag(y[i])*real(x[i]) + real(y[i])*imag(x[i]),
   196  	//	real(result[i]):  real(y[i])*real(x[i]) - imag(y[i])*imag(x[i]) }
   197  	ADDSUBPS_X2_X3
   198  
   199  	ADDPS X3, SUM // SUM += X_i
   200  
   201  dotc_end:
   202  	ADDPS   P_SUM, SUM   // SUM = { P_SUM[0] + SUM[0] }
   203  	MOVHLPS P_SUM, P_SUM // P_SUM = { P_SUM[1], P_SUM[1] }
   204  	ADDPS   P_SUM, SUM   // SUM = { P_SUM[1] + SUM[0] }
   205  
   206  dotc_ret:
   207  	MOVSD SUM, sum+48(FP) // return SUM
   208  	RET