github.com/emmansun/gmsm@v0.29.1/sm4/cipher_asm.go (about)

     1  //go:build (amd64 || arm64 || ppc64 || ppc64le) && !purego
     2  
     3  package sm4
     4  
     5  import (
     6  	"crypto/cipher"
     7  	"os"
     8  
     9  	"github.com/emmansun/gmsm/internal/alias"
    10  	"github.com/emmansun/gmsm/internal/cpuid"
    11  	"golang.org/x/sys/cpu"
    12  )
    13  
    14  var supportSM4 = cpu.ARM64.HasSM4 && os.Getenv("DISABLE_SM4NI") != "1"
    15  var supportsAES = cpuid.HasAES
    16  var supportsGFMUL = cpuid.HasGFMUL
    17  var useAVX2 = cpu.X86.HasAVX2
    18  var useAVX = cpu.X86.HasAVX
    19  var useAESNI4SingleBlock = os.Getenv("FORCE_SM4BLOCK_AESNI") == "1"
    20  
    21  const (
    22  	INST_AES int = iota
    23  	INST_SM4
    24  )
    25  
    26  //go:noescape
    27  func encryptBlocksAsm(xk *uint32, dst, src []byte, inst int)
    28  
    29  //go:noescape
    30  func encryptBlockAsm(xk *uint32, dst, src *byte, inst int)
    31  
    32  //go:noescape
    33  func expandKeyAsm(key *byte, ck, enc, dec *uint32, inst int)
    34  
    35  type sm4CipherAsm struct {
    36  	sm4Cipher
    37  	batchBlocks int
    38  	blocksSize  int
    39  }
    40  
    41  // sm4CipherGCM implements crypto/cipher.gcmAble so that crypto/cipher.NewGCM
    42  // will use the optimised implementation in this file when possible. Instances
    43  // of this type only exist when hasGCMAsm and hasAES returns true.
    44  type sm4CipherGCM struct {
    45  	sm4CipherAsm
    46  }
    47  
    48  func newCipher(key []byte) (cipher.Block, error) {
    49  	if supportSM4 {
    50  		return newCipherNI(key)
    51  	}
    52  
    53  	if !supportsAES {
    54  		return newCipherGeneric(key)
    55  	}
    56  
    57  	blocks := 4
    58  	if useAVX2 {
    59  		blocks = 8
    60  	}
    61  	c := &sm4CipherGCM{sm4CipherAsm{sm4Cipher{}, blocks, blocks * BlockSize}}
    62  	expandKeyAsm(&key[0], &ck[0], &c.enc[0], &c.dec[0], INST_AES)
    63  	if supportsGFMUL {
    64  		return c, nil
    65  	}
    66  	return &c.sm4CipherAsm, nil
    67  }
    68  
    69  func (c *sm4CipherAsm) Concurrency() int { return c.batchBlocks }
    70  
    71  func (c *sm4CipherAsm) Encrypt(dst, src []byte) {
    72  	if len(src) < BlockSize {
    73  		panic("sm4: input not full block")
    74  	}
    75  	if len(dst) < BlockSize {
    76  		panic("sm4: output not full block")
    77  	}
    78  	if alias.InexactOverlap(dst[:BlockSize], src[:BlockSize]) {
    79  		panic("sm4: invalid buffer overlap")
    80  	}
    81  	c.encrypt(dst, src)
    82  }
    83  
    84  func (c *sm4CipherAsm) encrypt(dst, src []byte) {
    85  	if useAESNI4SingleBlock {
    86  		encryptBlockAsm(&c.enc[0], &dst[0], &src[0], INST_AES)
    87  	} else {
    88  		encryptBlockGo(&c.enc, dst, src)
    89  	}
    90  }
    91  
    92  func (c *sm4CipherAsm) Decrypt(dst, src []byte) {
    93  	if len(src) < BlockSize {
    94  		panic("sm4: input not full block")
    95  	}
    96  	if len(dst) < BlockSize {
    97  		panic("sm4: output not full block")
    98  	}
    99  	if alias.InexactOverlap(dst[:BlockSize], src[:BlockSize]) {
   100  		panic("sm4: invalid buffer overlap")
   101  	}
   102  	if useAESNI4SingleBlock {
   103  		encryptBlockAsm(&c.dec[0], &dst[0], &src[0], INST_AES)
   104  	} else {
   105  		encryptBlockGo(&c.dec, dst, src)
   106  	}
   107  }
   108  
   109  func (c *sm4CipherAsm) EncryptBlocks(dst, src []byte) {
   110  	if len(src) < c.blocksSize {
   111  		panic("sm4: input not full blocks")
   112  	}
   113  	if len(dst) < c.blocksSize {
   114  		panic("sm4: output not full blocks")
   115  	}
   116  	if alias.InexactOverlap(dst[:c.blocksSize], src[:c.blocksSize]) {
   117  		panic("sm4: invalid buffer overlap")
   118  	}
   119  	encryptBlocksAsm(&c.enc[0], dst, src, INST_AES)
   120  }
   121  
   122  func (c *sm4CipherAsm) DecryptBlocks(dst, src []byte) {
   123  	if len(src) < c.blocksSize {
   124  		panic("sm4: input not full blocks")
   125  	}
   126  	if len(dst) < c.blocksSize {
   127  		panic("sm4: output not full blocks")
   128  	}
   129  	if alias.InexactOverlap(dst[:c.blocksSize], src[:c.blocksSize]) {
   130  		panic("sm4: invalid buffer overlap")
   131  	}
   132  	encryptBlocksAsm(&c.dec[0], dst, src, INST_AES)
   133  }
   134  
   135  // expandKey is used by BenchmarkExpand to ensure that the asm implementation
   136  // of key expansion is used for the benchmark when it is available.
   137  func expandKey(key []byte, enc, dec []uint32) {
   138  	if supportSM4 {
   139  		expandKeyAsm(&key[0], &ck[0], &enc[0], &dec[0], INST_SM4)
   140  	} else if supportsAES && useAESNI4SingleBlock {
   141  		expandKeyAsm(&key[0], &ck[0], &enc[0], &dec[0], INST_AES)
   142  	} else {
   143  		expandKeyGo(key, (*[rounds]uint32)(enc), (*[rounds]uint32)(dec))
   144  	}
   145  }