github.com/jingcheng-WU/gonum@v0.9.1-0.20210323123734-f1a2a11a8f7b/internal/asm/c128/dotcinc_amd64.s (about) 1 // Copyright ©2016 The Gonum Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // +build !noasm,!gccgo,!safe 6 7 #include "textflag.h" 8 9 #define MOVDDUP_XPTR__X3 LONG $0x1E120FF2 // MOVDDUP (SI), X3 10 #define MOVDDUP_XPTR_INCX__X5 LONG $0x120F42F2; WORD $0x062C // MOVDDUP (SI)(R8*1), X5 11 #define MOVDDUP_XPTR_INCX_2__X7 LONG $0x120F42F2; WORD $0x463C // MOVDDUP (SI)(R8*2), X7 12 #define MOVDDUP_XPTR_INCx3X__X9 LONG $0x120F46F2; WORD $0x0E0C // MOVDDUP (SI)(R9*1), X9 13 14 #define MOVDDUP_8_XPTR__X2 LONG $0x56120FF2; BYTE $0x08 // MOVDDUP 8(SI), X2 15 #define MOVDDUP_8_XPTR_INCX__X4 LONG $0x120F42F2; WORD $0x0664; BYTE $0x08 // MOVDDUP 8(SI)(R8*1), X4 16 #define MOVDDUP_8_XPTR_INCX_2__X6 LONG $0x120F42F2; WORD $0x4674; BYTE $0x08 // MOVDDUP 8(SI)(R8*2), X6 17 #define MOVDDUP_8_XPTR_INCx3X__X8 LONG $0x120F46F2; WORD $0x0E44; BYTE $0x08 // MOVDDUP 8(SI)(R9*1), X8 18 19 #define ADDSUBPD_X2_X3 LONG $0xDAD00F66 // ADDSUBPD X2, X3 20 #define ADDSUBPD_X4_X5 LONG $0xECD00F66 // ADDSUBPD X4, X5 21 #define ADDSUBPD_X6_X7 LONG $0xFED00F66 // ADDSUBPD X6, X7 22 #define ADDSUBPD_X8_X9 LONG $0xD00F4566; BYTE $0xC8 // ADDSUBPD X8, X9 23 24 #define X_PTR SI 25 #define Y_PTR DI 26 #define LEN CX 27 #define TAIL BX 28 #define SUM X0 29 #define P_SUM X1 30 #define INC_X R8 31 #define INCx3_X R9 32 #define INC_Y R10 33 #define INCx3_Y R11 34 #define NEG1 X15 35 #define P_NEG1 X14 36 37 // func DotcInc(x, y []complex128, n, incX, incY, ix, iy uintptr) (sum complex128) 38 TEXT ·DotcInc(SB), NOSPLIT, $0 39 MOVQ x_base+0(FP), X_PTR // X_PTR = &x 40 MOVQ y_base+24(FP), Y_PTR // Y_PTR = &y 41 MOVQ n+48(FP), LEN // LEN = n 42 PXOR SUM, SUM // SUM = 0 43 CMPQ LEN, $0 // if LEN == 0 { return } 44 JE dot_end 45 PXOR P_SUM, P_SUM // P_SUM = 0 46 MOVQ ix+72(FP), INC_X // INC_X = ix * sizeof(complex128) 47 SHLQ $4, INC_X 48 MOVQ iy+80(FP), INC_Y // INC_Y = iy * sizeof(complex128) 49 SHLQ $4, INC_Y 50 LEAQ (X_PTR)(INC_X*1), X_PTR // X_PTR = &(X_PTR[ix]) 51 LEAQ (Y_PTR)(INC_Y*1), Y_PTR // Y_PTR = &(Y_PTR[iy]) 52 MOVQ incX+56(FP), INC_X // INC_X = incX 53 SHLQ $4, INC_X // INC_X *= sizeof(complex128) 54 MOVQ incY+64(FP), INC_Y // INC_Y = incY 55 SHLQ $4, INC_Y // INC_Y *= sizeof(complex128) 56 MOVSD $(-1.0), NEG1 57 SHUFPD $0, NEG1, NEG1 // { -1, -1 } 58 MOVQ LEN, TAIL 59 ANDQ $3, TAIL // TAIL = n % 4 60 SHRQ $2, LEN // LEN = floor( n / 4 ) 61 JZ dot_tail // if n <= 4 { goto dot_tail } 62 MOVAPS NEG1, P_NEG1 // Copy NEG1 to P_NEG1 for pipelining 63 LEAQ (INC_X)(INC_X*2), INCx3_X // INCx3_X = 3 * incX * sizeof(complex128) 64 LEAQ (INC_Y)(INC_Y*2), INCx3_Y // INCx3_Y = 3 * incY * sizeof(complex128) 65 66 dot_loop: // do { 67 MOVDDUP_XPTR__X3 // X_(i+1) = { real(x[i], real(x[i]) } 68 MOVDDUP_XPTR_INCX__X5 69 MOVDDUP_XPTR_INCX_2__X7 70 MOVDDUP_XPTR_INCx3X__X9 71 72 MOVDDUP_8_XPTR__X2 // X_i = { imag(x[i]), imag(x[i]) } 73 MOVDDUP_8_XPTR_INCX__X4 74 MOVDDUP_8_XPTR_INCX_2__X6 75 MOVDDUP_8_XPTR_INCx3X__X8 76 77 // X_i = { -imag(x[i]), -imag(x[i]) } 78 MULPD NEG1, X2 79 MULPD P_NEG1, X4 80 MULPD NEG1, X6 81 MULPD P_NEG1, X8 82 83 // X_j = { imag(y[i]), real(y[i]) } 84 MOVUPS (Y_PTR), X10 85 MOVUPS (Y_PTR)(INC_Y*1), X11 86 MOVUPS (Y_PTR)(INC_Y*2), X12 87 MOVUPS (Y_PTR)(INCx3_Y*1), X13 88 89 // X_(i+1) = { imag(a) * real(x[i]), real(a) * real(x[i]) } 90 MULPD X10, X3 91 MULPD X11, X5 92 MULPD X12, X7 93 MULPD X13, X9 94 95 // X_j = { real(y[i]), imag(y[i]) } 96 SHUFPD $0x1, X10, X10 97 SHUFPD $0x1, X11, X11 98 SHUFPD $0x1, X12, X12 99 SHUFPD $0x1, X13, X13 100 101 // X_i = { real(a) * imag(x[i]), imag(a) * imag(x[i]) } 102 MULPD X10, X2 103 MULPD X11, X4 104 MULPD X12, X6 105 MULPD X13, X8 106 107 // X_(i+1) = { 108 // imag(result[i]): imag(a)*real(x[i]) + real(a)*imag(x[i]), 109 // real(result[i]): real(a)*real(x[i]) - imag(a)*imag(x[i]) 110 // } 111 ADDSUBPD_X2_X3 112 ADDSUBPD_X4_X5 113 ADDSUBPD_X6_X7 114 ADDSUBPD_X8_X9 115 116 // psum += result[i] 117 ADDPD X3, SUM 118 ADDPD X5, P_SUM 119 ADDPD X7, SUM 120 ADDPD X9, P_SUM 121 122 LEAQ (X_PTR)(INC_X*4), X_PTR // X_PTR = &(X_PTR[incX*4]) 123 LEAQ (Y_PTR)(INC_Y*4), Y_PTR // Y_PTR = &(Y_PTR[incY*4]) 124 125 DECQ LEN 126 JNZ dot_loop // } while --LEN > 0 127 ADDPD P_SUM, SUM // sum += psum 128 CMPQ TAIL, $0 // if TAIL == 0 { return } 129 JE dot_end 130 131 dot_tail: // do { 132 MOVDDUP_XPTR__X3 // X_(i+1) = { real(x[i], real(x[i]) } 133 MOVDDUP_8_XPTR__X2 // X_i = { imag(x[i]), imag(x[i]) } 134 MULPD NEG1, X2 // X_i = { -imag(x[i]) , -imag(x[i]) } 135 MOVUPS (Y_PTR), X10 // X_j = { imag(y[i]) , real(y[i]) } 136 MULPD X10, X3 // X_(i+1) = { imag(a) * real(x[i]), real(a) * real(x[i]) } 137 SHUFPD $0x1, X10, X10 // X_j = { real(y[i]) , imag(y[i]) } 138 MULPD X10, X2 // X_i = { real(a) * imag(x[i]), imag(a) * imag(x[i]) } 139 140 // X_(i+1) = { 141 // imag(result[i]): imag(a)*real(x[i]) + real(a)*imag(x[i]), 142 // real(result[i]): real(a)*real(x[i]) - imag(a)*imag(x[i]) 143 // } 144 ADDSUBPD_X2_X3 145 ADDPD X3, SUM // sum += result[i] 146 ADDQ INC_X, X_PTR // X_PTR += incX 147 ADDQ INC_Y, Y_PTR // Y_PTR += incY 148 DECQ TAIL 149 JNZ dot_tail // } while --TAIL > 0 150 151 dot_end: 152 MOVUPS SUM, sum+88(FP) 153 RET