github.com/sbinet/go@v0.0.0-20160827155028-54d7de7dd62b/src/crypto/aes/asm_amd64.s (about)

     1  // Copyright 2012 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  #include "textflag.h"
     6  
     7  // func hasAsm() bool
     8  // returns whether AES-NI is supported
     9  TEXT ·hasAsm(SB),NOSPLIT,$0
    10  	XORQ AX, AX
    11  	INCL AX
    12  	CPUID
    13  	SHRQ $25, CX
    14  	ANDQ $1, CX
    15  	MOVB CX, ret+0(FP)
    16  	RET
    17  
    18  // func encryptBlockAsm(nr int, xk *uint32, dst, src *byte)
    19  TEXT ·encryptBlockAsm(SB),NOSPLIT,$0
    20  	MOVQ nr+0(FP), CX
    21  	MOVQ xk+8(FP), AX
    22  	MOVQ dst+16(FP), DX
    23  	MOVQ src+24(FP), BX
    24  	MOVUPS 0(AX), X1
    25  	MOVUPS 0(BX), X0
    26  	ADDQ $16, AX
    27  	PXOR X1, X0
    28  	SUBQ $12, CX
    29  	JE Lenc196
    30  	JB Lenc128
    31  Lenc256:
    32  	MOVUPS 0(AX), X1
    33  	AESENC X1, X0
    34  	MOVUPS 16(AX), X1
    35  	AESENC X1, X0
    36  	ADDQ $32, AX
    37  Lenc196:
    38  	MOVUPS 0(AX), X1
    39  	AESENC X1, X0
    40  	MOVUPS 16(AX), X1
    41  	AESENC X1, X0
    42  	ADDQ $32, AX
    43  Lenc128:
    44  	MOVUPS 0(AX), X1
    45  	AESENC X1, X0
    46  	MOVUPS 16(AX), X1
    47  	AESENC X1, X0
    48  	MOVUPS 32(AX), X1
    49  	AESENC X1, X0
    50  	MOVUPS 48(AX), X1
    51  	AESENC X1, X0
    52  	MOVUPS 64(AX), X1
    53  	AESENC X1, X0
    54  	MOVUPS 80(AX), X1
    55  	AESENC X1, X0
    56  	MOVUPS 96(AX), X1
    57  	AESENC X1, X0
    58  	MOVUPS 112(AX), X1
    59  	AESENC X1, X0
    60  	MOVUPS 128(AX), X1
    61  	AESENC X1, X0
    62  	MOVUPS 144(AX), X1
    63  	AESENCLAST X1, X0
    64  	MOVUPS X0, 0(DX)
    65  	RET
    66  
    67  // func decryptBlockAsm(nr int, xk *uint32, dst, src *byte)
    68  TEXT ·decryptBlockAsm(SB),NOSPLIT,$0
    69  	MOVQ nr+0(FP), CX
    70  	MOVQ xk+8(FP), AX
    71  	MOVQ dst+16(FP), DX
    72  	MOVQ src+24(FP), BX
    73  	MOVUPS 0(AX), X1
    74  	MOVUPS 0(BX), X0
    75  	ADDQ $16, AX
    76  	PXOR X1, X0
    77  	SUBQ $12, CX
    78  	JE Ldec196
    79  	JB Ldec128
    80  Ldec256:
    81  	MOVUPS 0(AX), X1
    82  	AESDEC X1, X0
    83  	MOVUPS 16(AX), X1
    84  	AESDEC X1, X0
    85  	ADDQ $32, AX
    86  Ldec196:
    87  	MOVUPS 0(AX), X1
    88  	AESDEC X1, X0
    89  	MOVUPS 16(AX), X1
    90  	AESDEC X1, X0
    91  	ADDQ $32, AX
    92  Ldec128:
    93  	MOVUPS 0(AX), X1
    94  	AESDEC X1, X0
    95  	MOVUPS 16(AX), X1
    96  	AESDEC X1, X0
    97  	MOVUPS 32(AX), X1
    98  	AESDEC X1, X0
    99  	MOVUPS 48(AX), X1
   100  	AESDEC X1, X0
   101  	MOVUPS 64(AX), X1
   102  	AESDEC X1, X0
   103  	MOVUPS 80(AX), X1
   104  	AESDEC X1, X0
   105  	MOVUPS 96(AX), X1
   106  	AESDEC X1, X0
   107  	MOVUPS 112(AX), X1
   108  	AESDEC X1, X0
   109  	MOVUPS 128(AX), X1
   110  	AESDEC X1, X0
   111  	MOVUPS 144(AX), X1
   112  	AESDECLAST X1, X0
   113  	MOVUPS X0, 0(DX)
   114  	RET
   115  
   116  // func expandKeyAsm(nr int, key *byte, enc, dec *uint32) {
   117  // Note that round keys are stored in uint128 format, not uint32
   118  TEXT ·expandKeyAsm(SB),NOSPLIT,$0
   119  	MOVQ nr+0(FP), CX
   120  	MOVQ key+8(FP), AX
   121  	MOVQ enc+16(FP), BX
   122  	MOVQ dec+24(FP), DX
   123  	MOVUPS (AX), X0
   124  	// enc
   125  	MOVUPS X0, (BX)
   126  	ADDQ $16, BX
   127  	PXOR X4, X4 // _expand_key_* expect X4 to be zero
   128  	CMPL CX, $12
   129  	JE Lexp_enc196
   130  	JB Lexp_enc128
   131  Lexp_enc256:
   132  	MOVUPS 16(AX), X2
   133  	MOVUPS X2, (BX)
   134  	ADDQ $16, BX
   135  	AESKEYGENASSIST $0x01, X2, X1
   136  	CALL _expand_key_256a<>(SB)
   137  	AESKEYGENASSIST $0x01, X0, X1
   138  	CALL _expand_key_256b<>(SB)
   139  	AESKEYGENASSIST $0x02, X2, X1
   140  	CALL _expand_key_256a<>(SB)
   141  	AESKEYGENASSIST $0x02, X0, X1
   142  	CALL _expand_key_256b<>(SB)
   143  	AESKEYGENASSIST $0x04, X2, X1
   144  	CALL _expand_key_256a<>(SB)
   145  	AESKEYGENASSIST $0x04, X0, X1
   146  	CALL _expand_key_256b<>(SB)
   147  	AESKEYGENASSIST $0x08, X2, X1
   148  	CALL _expand_key_256a<>(SB)
   149  	AESKEYGENASSIST $0x08, X0, X1
   150  	CALL _expand_key_256b<>(SB)
   151  	AESKEYGENASSIST $0x10, X2, X1
   152  	CALL _expand_key_256a<>(SB)
   153  	AESKEYGENASSIST $0x10, X0, X1
   154  	CALL _expand_key_256b<>(SB)
   155  	AESKEYGENASSIST $0x20, X2, X1
   156  	CALL _expand_key_256a<>(SB)
   157  	AESKEYGENASSIST $0x20, X0, X1
   158  	CALL _expand_key_256b<>(SB)
   159  	AESKEYGENASSIST $0x40, X2, X1
   160  	CALL _expand_key_256a<>(SB)
   161  	JMP Lexp_dec
   162  Lexp_enc196:
   163  	MOVQ 16(AX), X2
   164  	AESKEYGENASSIST $0x01, X2, X1
   165  	CALL _expand_key_192a<>(SB)
   166  	AESKEYGENASSIST $0x02, X2, X1
   167  	CALL _expand_key_192b<>(SB)
   168  	AESKEYGENASSIST $0x04, X2, X1
   169  	CALL _expand_key_192a<>(SB)
   170  	AESKEYGENASSIST $0x08, X2, X1
   171  	CALL _expand_key_192b<>(SB)
   172  	AESKEYGENASSIST $0x10, X2, X1
   173  	CALL _expand_key_192a<>(SB)
   174  	AESKEYGENASSIST $0x20, X2, X1
   175  	CALL _expand_key_192b<>(SB)
   176  	AESKEYGENASSIST $0x40, X2, X1
   177  	CALL _expand_key_192a<>(SB)
   178  	AESKEYGENASSIST $0x80, X2, X1
   179  	CALL _expand_key_192b<>(SB)
   180  	JMP Lexp_dec
   181  Lexp_enc128:
   182  	AESKEYGENASSIST $0x01, X0, X1
   183  	CALL _expand_key_128<>(SB)
   184  	AESKEYGENASSIST $0x02, X0, X1
   185  	CALL _expand_key_128<>(SB)
   186  	AESKEYGENASSIST $0x04, X0, X1
   187  	CALL _expand_key_128<>(SB)
   188  	AESKEYGENASSIST $0x08, X0, X1
   189  	CALL _expand_key_128<>(SB)
   190  	AESKEYGENASSIST $0x10, X0, X1
   191  	CALL _expand_key_128<>(SB)
   192  	AESKEYGENASSIST $0x20, X0, X1
   193  	CALL _expand_key_128<>(SB)
   194  	AESKEYGENASSIST $0x40, X0, X1
   195  	CALL _expand_key_128<>(SB)
   196  	AESKEYGENASSIST $0x80, X0, X1
   197  	CALL _expand_key_128<>(SB)
   198  	AESKEYGENASSIST $0x1b, X0, X1
   199  	CALL _expand_key_128<>(SB)
   200  	AESKEYGENASSIST $0x36, X0, X1
   201  	CALL _expand_key_128<>(SB)
   202  Lexp_dec:
   203  	// dec
   204  	SUBQ $16, BX
   205  	MOVUPS (BX), X1
   206  	MOVUPS X1, (DX)
   207  	DECQ CX
   208  Lexp_dec_loop:
   209  	MOVUPS -16(BX), X1
   210  	AESIMC X1, X0
   211  	MOVUPS X0, 16(DX)
   212  	SUBQ $16, BX
   213  	ADDQ $16, DX
   214  	DECQ CX
   215  	JNZ Lexp_dec_loop
   216  	MOVUPS -16(BX), X0
   217  	MOVUPS X0, 16(DX)
   218  	RET
   219  
   220  TEXT _expand_key_128<>(SB),NOSPLIT,$0
   221  	PSHUFD $0xff, X1, X1
   222  	SHUFPS $0x10, X0, X4
   223  	PXOR X4, X0
   224  	SHUFPS $0x8c, X0, X4
   225  	PXOR X4, X0
   226  	PXOR X1, X0
   227  	MOVUPS X0, (BX)
   228  	ADDQ $16, BX
   229  	RET
   230  
   231  TEXT _expand_key_192a<>(SB),NOSPLIT,$0
   232  	PSHUFD $0x55, X1, X1
   233  	SHUFPS $0x10, X0, X4
   234  	PXOR X4, X0
   235  	SHUFPS $0x8c, X0, X4
   236  	PXOR X4, X0
   237  	PXOR X1, X0
   238  
   239  	MOVAPS X2, X5
   240  	MOVAPS X2, X6
   241  	PSLLDQ $0x4, X5
   242  	PSHUFD $0xff, X0, X3
   243  	PXOR X3, X2
   244  	PXOR X5, X2
   245  
   246  	MOVAPS X0, X1
   247  	SHUFPS $0x44, X0, X6
   248  	MOVUPS X6, (BX)
   249  	SHUFPS $0x4e, X2, X1
   250  	MOVUPS X1, 16(BX)
   251  	ADDQ $32, BX
   252  	RET
   253  
   254  TEXT _expand_key_192b<>(SB),NOSPLIT,$0
   255  	PSHUFD $0x55, X1, X1
   256  	SHUFPS $0x10, X0, X4
   257  	PXOR X4, X0
   258  	SHUFPS $0x8c, X0, X4
   259  	PXOR X4, X0
   260  	PXOR X1, X0
   261  
   262  	MOVAPS X2, X5
   263  	PSLLDQ $0x4, X5
   264  	PSHUFD $0xff, X0, X3
   265  	PXOR X3, X2
   266  	PXOR X5, X2
   267  
   268  	MOVUPS X0, (BX)
   269  	ADDQ $16, BX
   270  	RET
   271  
   272  TEXT _expand_key_256a<>(SB),NOSPLIT,$0
   273  	JMP _expand_key_128<>(SB)
   274  
   275  TEXT _expand_key_256b<>(SB),NOSPLIT,$0
   276  	PSHUFD $0xaa, X1, X1
   277  	SHUFPS $0x10, X2, X4
   278  	PXOR X4, X2
   279  	SHUFPS $0x8c, X2, X4
   280  	PXOR X4, X2
   281  	PXOR X1, X2
   282  
   283  	MOVUPS X2, (BX)
   284  	ADDQ $16, BX
   285  	RET