github.com/sagernet/gvisor@v0.0.0-20240428053021-e691de28565f/pkg/safecopy/memcpy_amd64.s (about)

     1  // Copyright © 1994-1999 Lucent Technologies Inc. All rights reserved.
     2  // Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com).  All rights reserved.
     3  // Portions Copyright 2009 The Go Authors. All rights reserved.
     4  //
     5  // Permission is hereby granted, free of charge, to any person obtaining a copy
     6  // of this software and associated documentation files (the "Software"), to deal
     7  // in the Software without restriction, including without limitation the rights
     8  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     9  // copies of the Software, and to permit persons to whom the Software is
    10  // furnished to do so, subject to the following conditions:
    11  //
    12  // The above copyright notice and this permission notice shall be included in
    13  // all copies or substantial portions of the Software.
    14  //
    15  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    16  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    17  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
    18  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    19  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    20  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    21  // THE SOFTWARE.
    22  
    23  #include "textflag.h"
    24  
    25  // handleMemcpyFault returns (the value stored in AX, the value stored in DI).
    26  // Control is transferred to it when memcpy below receives SIGSEGV or SIGBUS,
    27  // with the faulting address stored in AX and the signal number stored in DI.
    28  //
    29  // It must have the same frame configuration as memcpy so that it can undo any
    30  // potential call frame set up by the assembler.
    31  TEXT handleMemcpyFault(SB), NOSPLIT|NOFRAME, $0-36
    32  	MOVQ	AX, addr+24(FP)
    33  	MOVL	DI, sig+32(FP)
    34  	RET
    35  
    36  // memcpy copies data from src to dst. If a SIGSEGV or SIGBUS signal is received
    37  // during the copy, it returns the address that caused the fault and the number
    38  // of the signal that was received. Otherwise, it returns an unspecified address
    39  // and a signal number of 0.
    40  //
    41  // Data is copied in order, such that if a fault happens at address p, it is
    42  // safe to assume that all data before p-maxRegisterSize has already been
    43  // successfully copied.
    44  //
    45  // The code is derived from the forward copying part of runtime.memmove.
    46  //
    47  // func memcpy(dst, src unsafe.Pointer, n uintptr) (fault unsafe.Pointer, sig int32)
    48  TEXT ·memcpy(SB), NOSPLIT|NOFRAME, $0-36
    49  	// Store 0 as the returned signal number. If we run to completion,
    50  	// this is the value the caller will see; if a signal is received,
    51  	// handleMemcpyFault will store a different value in this address.
    52  	MOVL	$0, sig+32(FP)
    53  
    54  	MOVQ	dst+0(FP), DI
    55  	MOVQ	src+8(FP), SI
    56  	MOVQ	n+16(FP), BX
    57  
    58  tail:
    59  	// BSR+branch table make almost all memmove/memclr benchmarks worse. Not
    60  	// worth doing.
    61  	TESTQ	BX, BX
    62  	JEQ	move_0
    63  	CMPQ	BX, $2
    64  	JBE	move_1or2
    65  	CMPQ	BX, $4
    66  	JBE	move_3or4
    67  	CMPQ	BX, $8
    68  	JB	move_5through7
    69  	JE	move_8
    70  	CMPQ	BX, $16
    71  	JBE	move_9through16
    72  	CMPQ	BX, $32
    73  	JBE	move_17through32
    74  	CMPQ	BX, $64
    75  	JBE	move_33through64
    76  	CMPQ	BX, $128
    77  	JBE	move_65through128
    78  	CMPQ	BX, $256
    79  	JBE	move_129through256
    80  
    81  move_257plus:
    82  	SUBQ	$256, BX
    83  	MOVOU	(SI), X0
    84  	MOVOU	X0, (DI)
    85  	MOVOU	16(SI), X1
    86  	MOVOU	X1, 16(DI)
    87  	MOVOU	32(SI), X2
    88  	MOVOU	X2, 32(DI)
    89  	MOVOU	48(SI), X3
    90  	MOVOU	X3, 48(DI)
    91  	MOVOU	64(SI), X4
    92  	MOVOU	X4, 64(DI)
    93  	MOVOU	80(SI), X5
    94  	MOVOU	X5, 80(DI)
    95  	MOVOU	96(SI), X6
    96  	MOVOU	X6, 96(DI)
    97  	MOVOU	112(SI), X7
    98  	MOVOU	X7, 112(DI)
    99  	MOVOU	128(SI), X8
   100  	MOVOU	X8, 128(DI)
   101  	MOVOU	144(SI), X9
   102  	MOVOU	X9, 144(DI)
   103  	MOVOU	160(SI), X10
   104  	MOVOU	X10, 160(DI)
   105  	MOVOU	176(SI), X11
   106  	MOVOU	X11, 176(DI)
   107  	MOVOU	192(SI), X12
   108  	MOVOU	X12, 192(DI)
   109  	MOVOU	208(SI), X13
   110  	MOVOU	X13, 208(DI)
   111  	MOVOU	224(SI), X14
   112  	MOVOU	X14, 224(DI)
   113  	MOVOU	240(SI), X15
   114  	MOVOU	X15, 240(DI)
   115  	CMPQ	BX, $256
   116  	LEAQ	256(SI), SI
   117  	LEAQ	256(DI), DI
   118  	JGE	move_257plus
   119  	JMP	tail
   120  
   121  move_1or2:
   122  	MOVB	(SI), AX
   123  	MOVB	AX, (DI)
   124  	MOVB	-1(SI)(BX*1), CX
   125  	MOVB	CX, -1(DI)(BX*1)
   126  	RET
   127  move_0:
   128  	RET
   129  move_3or4:
   130  	MOVW	(SI), AX
   131  	MOVW	AX, (DI)
   132  	MOVW	-2(SI)(BX*1), CX
   133  	MOVW	CX, -2(DI)(BX*1)
   134  	RET
   135  move_5through7:
   136  	MOVL	(SI), AX
   137  	MOVL	AX, (DI)
   138  	MOVL	-4(SI)(BX*1), CX
   139  	MOVL	CX, -4(DI)(BX*1)
   140  	RET
   141  move_8:
   142  	// We need a separate case for 8 to make sure we write pointers atomically.
   143  	MOVQ	(SI), AX
   144  	MOVQ	AX, (DI)
   145  	RET
   146  move_9through16:
   147  	MOVQ	(SI), AX
   148  	MOVQ	AX, (DI)
   149  	MOVQ	-8(SI)(BX*1), CX
   150  	MOVQ	CX, -8(DI)(BX*1)
   151  	RET
   152  move_17through32:
   153  	MOVOU	(SI), X0
   154  	MOVOU	X0, (DI)
   155  	MOVOU	-16(SI)(BX*1), X1
   156  	MOVOU	X1, -16(DI)(BX*1)
   157  	RET
   158  move_33through64:
   159  	MOVOU	(SI), X0
   160  	MOVOU	X0, (DI)
   161  	MOVOU	16(SI), X1
   162  	MOVOU	X1, 16(DI)
   163  	MOVOU	-32(SI)(BX*1), X2
   164  	MOVOU	X2, -32(DI)(BX*1)
   165  	MOVOU	-16(SI)(BX*1), X3
   166  	MOVOU	X3, -16(DI)(BX*1)
   167  	RET
   168  move_65through128:
   169  	MOVOU	(SI), X0
   170  	MOVOU	X0, (DI)
   171  	MOVOU	16(SI), X1
   172  	MOVOU	X1, 16(DI)
   173  	MOVOU	32(SI), X2
   174  	MOVOU	X2, 32(DI)
   175  	MOVOU	48(SI), X3
   176  	MOVOU	X3, 48(DI)
   177  	MOVOU	-64(SI)(BX*1), X4
   178  	MOVOU	X4, -64(DI)(BX*1)
   179  	MOVOU	-48(SI)(BX*1), X5
   180  	MOVOU	X5, -48(DI)(BX*1)
   181  	MOVOU	-32(SI)(BX*1), X6
   182  	MOVOU	X6, -32(DI)(BX*1)
   183  	MOVOU	-16(SI)(BX*1), X7
   184  	MOVOU	X7, -16(DI)(BX*1)
   185  	RET
   186  move_129through256:
   187  	MOVOU	(SI), X0
   188  	MOVOU	X0, (DI)
   189  	MOVOU	16(SI), X1
   190  	MOVOU	X1, 16(DI)
   191  	MOVOU	32(SI), X2
   192  	MOVOU	X2, 32(DI)
   193  	MOVOU	48(SI), X3
   194  	MOVOU	X3, 48(DI)
   195  	MOVOU	64(SI), X4
   196  	MOVOU	X4, 64(DI)
   197  	MOVOU	80(SI), X5
   198  	MOVOU	X5, 80(DI)
   199  	MOVOU	96(SI), X6
   200  	MOVOU	X6, 96(DI)
   201  	MOVOU	112(SI), X7
   202  	MOVOU	X7, 112(DI)
   203  	MOVOU	-128(SI)(BX*1), X8
   204  	MOVOU	X8, -128(DI)(BX*1)
   205  	MOVOU	-112(SI)(BX*1), X9
   206  	MOVOU	X9, -112(DI)(BX*1)
   207  	MOVOU	-96(SI)(BX*1), X10
   208  	MOVOU	X10, -96(DI)(BX*1)
   209  	MOVOU	-80(SI)(BX*1), X11
   210  	MOVOU	X11, -80(DI)(BX*1)
   211  	MOVOU	-64(SI)(BX*1), X12
   212  	MOVOU	X12, -64(DI)(BX*1)
   213  	MOVOU	-48(SI)(BX*1), X13
   214  	MOVOU	X13, -48(DI)(BX*1)
   215  	MOVOU	-32(SI)(BX*1), X14
   216  	MOVOU	X14, -32(DI)(BX*1)
   217  	MOVOU	-16(SI)(BX*1), X15
   218  	MOVOU	X15, -16(DI)(BX*1)
   219  	RET
   220  
   221  // func addrOfMemcpy() uintptr
   222  TEXT ·addrOfMemcpy(SB), $0-8
   223  	MOVQ	$·memcpy(SB), AX
   224  	MOVQ	AX, ret+0(FP)
   225  	RET