github.com/consensys/gnark-crypto@v0.14.0/ecc/bls12-378/fp/element_ops_purego.go (about) 1 //go:build !amd64 || purego 2 // +build !amd64 purego 3 4 // Copyright 2020 ConsenSys Software Inc. 5 // 6 // Licensed under the Apache License, Version 2.0 (the "License"); 7 // you may not use this file except in compliance with the License. 8 // You may obtain a copy of the License at 9 // 10 // http://www.apache.org/licenses/LICENSE-2.0 11 // 12 // Unless required by applicable law or agreed to in writing, software 13 // distributed under the License is distributed on an "AS IS" BASIS, 14 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 // See the License for the specific language governing permissions and 16 // limitations under the License. 17 18 // Code generated by consensys/gnark-crypto DO NOT EDIT 19 20 package fp 21 22 import "math/bits" 23 24 // MulBy3 x *= 3 (mod q) 25 func MulBy3(x *Element) { 26 _x := *x 27 x.Double(x).Add(x, &_x) 28 } 29 30 // MulBy5 x *= 5 (mod q) 31 func MulBy5(x *Element) { 32 _x := *x 33 x.Double(x).Double(x).Add(x, &_x) 34 } 35 36 // MulBy13 x *= 13 (mod q) 37 func MulBy13(x *Element) { 38 var y = Element{ 39 8212494240417053874, 40 5029498262967025157, 41 9404736542133420963, 42 13073247822498485877, 43 1581382318314538223, 44 87125160541517067, 45 } 46 x.Mul(x, &y) 47 } 48 49 // Butterfly sets 50 // 51 // a = a + b (mod q) 52 // b = a - b (mod q) 53 func Butterfly(a, b *Element) { 54 _butterflyGeneric(a, b) 55 } 56 57 func fromMont(z *Element) { 58 _fromMontGeneric(z) 59 } 60 61 func reduce(z *Element) { 62 _reduceGeneric(z) 63 } 64 65 // Mul z = x * y (mod q) 66 // 67 // x and y must be less than q 68 func (z *Element) Mul(x, y *Element) *Element { 69 70 // Implements CIOS multiplication -- section 2.3.2 of Tolga Acar's thesis 71 // https://www.microsoft.com/en-us/research/wp-content/uploads/1998/06/97Acar.pdf 72 // 73 // The algorithm: 74 // 75 // for i=0 to N-1 76 // C := 0 77 // for j=0 to N-1 78 // (C,t[j]) := t[j] + x[j]*y[i] + C 79 // (t[N+1],t[N]) := t[N] + C 80 // 81 // C := 0 82 // m := t[0]*q'[0] mod D 83 // (C,_) := t[0] + m*q[0] 84 // for j=1 to N-1 85 // (C,t[j-1]) := t[j] + m*q[j] + C 86 // 87 // (C,t[N-1]) := t[N] + C 88 // t[N] := t[N+1] + C 89 // 90 // → N is the number of machine words needed to store the modulus q 91 // → D is the word size. For example, on a 64-bit architecture D is 2 64 92 // → x[i], y[i], q[i] is the ith word of the numbers x,y,q 93 // → q'[0] is the lowest word of the number -q⁻¹ mod r. This quantity is pre-computed, as it does not depend on the inputs. 94 // → t is a temporary array of size N+2 95 // → C, S are machine words. A pair (C,S) refers to (hi-bits, lo-bits) of a two-word number 96 // 97 // As described here https://hackmd.io/@gnark/modular_multiplication we can get rid of one carry chain and simplify: 98 // (also described in https://eprint.iacr.org/2022/1400.pdf annex) 99 // 100 // for i=0 to N-1 101 // (A,t[0]) := t[0] + x[0]*y[i] 102 // m := t[0]*q'[0] mod W 103 // C,_ := t[0] + m*q[0] 104 // for j=1 to N-1 105 // (A,t[j]) := t[j] + x[j]*y[i] + A 106 // (C,t[j-1]) := t[j] + m*q[j] + C 107 // 108 // t[N-1] = C + A 109 // 110 // This optimization saves 5N + 2 additions in the algorithm, and can be used whenever the highest bit 111 // of the modulus is zero (and not all of the remaining bits are set). 112 113 var t0, t1, t2, t3, t4, t5 uint64 114 var u0, u1, u2, u3, u4, u5 uint64 115 { 116 var c0, c1, c2 uint64 117 v := x[0] 118 u0, t0 = bits.Mul64(v, y[0]) 119 u1, t1 = bits.Mul64(v, y[1]) 120 u2, t2 = bits.Mul64(v, y[2]) 121 u3, t3 = bits.Mul64(v, y[3]) 122 u4, t4 = bits.Mul64(v, y[4]) 123 u5, t5 = bits.Mul64(v, y[5]) 124 t1, c0 = bits.Add64(u0, t1, 0) 125 t2, c0 = bits.Add64(u1, t2, c0) 126 t3, c0 = bits.Add64(u2, t3, c0) 127 t4, c0 = bits.Add64(u3, t4, c0) 128 t5, c0 = bits.Add64(u4, t5, c0) 129 c2, _ = bits.Add64(u5, 0, c0) 130 131 m := qInvNeg * t0 132 133 u0, c1 = bits.Mul64(m, q0) 134 _, c0 = bits.Add64(t0, c1, 0) 135 u1, c1 = bits.Mul64(m, q1) 136 t0, c0 = bits.Add64(t1, c1, c0) 137 u2, c1 = bits.Mul64(m, q2) 138 t1, c0 = bits.Add64(t2, c1, c0) 139 u3, c1 = bits.Mul64(m, q3) 140 t2, c0 = bits.Add64(t3, c1, c0) 141 u4, c1 = bits.Mul64(m, q4) 142 t3, c0 = bits.Add64(t4, c1, c0) 143 u5, c1 = bits.Mul64(m, q5) 144 145 t4, c0 = bits.Add64(0, c1, c0) 146 u5, _ = bits.Add64(u5, 0, c0) 147 t0, c0 = bits.Add64(u0, t0, 0) 148 t1, c0 = bits.Add64(u1, t1, c0) 149 t2, c0 = bits.Add64(u2, t2, c0) 150 t3, c0 = bits.Add64(u3, t3, c0) 151 t4, c0 = bits.Add64(u4, t4, c0) 152 c2, _ = bits.Add64(c2, 0, c0) 153 t4, c0 = bits.Add64(t5, t4, 0) 154 t5, _ = bits.Add64(u5, c2, c0) 155 156 } 157 { 158 var c0, c1, c2 uint64 159 v := x[1] 160 u0, c1 = bits.Mul64(v, y[0]) 161 t0, c0 = bits.Add64(c1, t0, 0) 162 u1, c1 = bits.Mul64(v, y[1]) 163 t1, c0 = bits.Add64(c1, t1, c0) 164 u2, c1 = bits.Mul64(v, y[2]) 165 t2, c0 = bits.Add64(c1, t2, c0) 166 u3, c1 = bits.Mul64(v, y[3]) 167 t3, c0 = bits.Add64(c1, t3, c0) 168 u4, c1 = bits.Mul64(v, y[4]) 169 t4, c0 = bits.Add64(c1, t4, c0) 170 u5, c1 = bits.Mul64(v, y[5]) 171 t5, c0 = bits.Add64(c1, t5, c0) 172 173 c2, _ = bits.Add64(0, 0, c0) 174 t1, c0 = bits.Add64(u0, t1, 0) 175 t2, c0 = bits.Add64(u1, t2, c0) 176 t3, c0 = bits.Add64(u2, t3, c0) 177 t4, c0 = bits.Add64(u3, t4, c0) 178 t5, c0 = bits.Add64(u4, t5, c0) 179 c2, _ = bits.Add64(u5, c2, c0) 180 181 m := qInvNeg * t0 182 183 u0, c1 = bits.Mul64(m, q0) 184 _, c0 = bits.Add64(t0, c1, 0) 185 u1, c1 = bits.Mul64(m, q1) 186 t0, c0 = bits.Add64(t1, c1, c0) 187 u2, c1 = bits.Mul64(m, q2) 188 t1, c0 = bits.Add64(t2, c1, c0) 189 u3, c1 = bits.Mul64(m, q3) 190 t2, c0 = bits.Add64(t3, c1, c0) 191 u4, c1 = bits.Mul64(m, q4) 192 t3, c0 = bits.Add64(t4, c1, c0) 193 u5, c1 = bits.Mul64(m, q5) 194 195 t4, c0 = bits.Add64(0, c1, c0) 196 u5, _ = bits.Add64(u5, 0, c0) 197 t0, c0 = bits.Add64(u0, t0, 0) 198 t1, c0 = bits.Add64(u1, t1, c0) 199 t2, c0 = bits.Add64(u2, t2, c0) 200 t3, c0 = bits.Add64(u3, t3, c0) 201 t4, c0 = bits.Add64(u4, t4, c0) 202 c2, _ = bits.Add64(c2, 0, c0) 203 t4, c0 = bits.Add64(t5, t4, 0) 204 t5, _ = bits.Add64(u5, c2, c0) 205 206 } 207 { 208 var c0, c1, c2 uint64 209 v := x[2] 210 u0, c1 = bits.Mul64(v, y[0]) 211 t0, c0 = bits.Add64(c1, t0, 0) 212 u1, c1 = bits.Mul64(v, y[1]) 213 t1, c0 = bits.Add64(c1, t1, c0) 214 u2, c1 = bits.Mul64(v, y[2]) 215 t2, c0 = bits.Add64(c1, t2, c0) 216 u3, c1 = bits.Mul64(v, y[3]) 217 t3, c0 = bits.Add64(c1, t3, c0) 218 u4, c1 = bits.Mul64(v, y[4]) 219 t4, c0 = bits.Add64(c1, t4, c0) 220 u5, c1 = bits.Mul64(v, y[5]) 221 t5, c0 = bits.Add64(c1, t5, c0) 222 223 c2, _ = bits.Add64(0, 0, c0) 224 t1, c0 = bits.Add64(u0, t1, 0) 225 t2, c0 = bits.Add64(u1, t2, c0) 226 t3, c0 = bits.Add64(u2, t3, c0) 227 t4, c0 = bits.Add64(u3, t4, c0) 228 t5, c0 = bits.Add64(u4, t5, c0) 229 c2, _ = bits.Add64(u5, c2, c0) 230 231 m := qInvNeg * t0 232 233 u0, c1 = bits.Mul64(m, q0) 234 _, c0 = bits.Add64(t0, c1, 0) 235 u1, c1 = bits.Mul64(m, q1) 236 t0, c0 = bits.Add64(t1, c1, c0) 237 u2, c1 = bits.Mul64(m, q2) 238 t1, c0 = bits.Add64(t2, c1, c0) 239 u3, c1 = bits.Mul64(m, q3) 240 t2, c0 = bits.Add64(t3, c1, c0) 241 u4, c1 = bits.Mul64(m, q4) 242 t3, c0 = bits.Add64(t4, c1, c0) 243 u5, c1 = bits.Mul64(m, q5) 244 245 t4, c0 = bits.Add64(0, c1, c0) 246 u5, _ = bits.Add64(u5, 0, c0) 247 t0, c0 = bits.Add64(u0, t0, 0) 248 t1, c0 = bits.Add64(u1, t1, c0) 249 t2, c0 = bits.Add64(u2, t2, c0) 250 t3, c0 = bits.Add64(u3, t3, c0) 251 t4, c0 = bits.Add64(u4, t4, c0) 252 c2, _ = bits.Add64(c2, 0, c0) 253 t4, c0 = bits.Add64(t5, t4, 0) 254 t5, _ = bits.Add64(u5, c2, c0) 255 256 } 257 { 258 var c0, c1, c2 uint64 259 v := x[3] 260 u0, c1 = bits.Mul64(v, y[0]) 261 t0, c0 = bits.Add64(c1, t0, 0) 262 u1, c1 = bits.Mul64(v, y[1]) 263 t1, c0 = bits.Add64(c1, t1, c0) 264 u2, c1 = bits.Mul64(v, y[2]) 265 t2, c0 = bits.Add64(c1, t2, c0) 266 u3, c1 = bits.Mul64(v, y[3]) 267 t3, c0 = bits.Add64(c1, t3, c0) 268 u4, c1 = bits.Mul64(v, y[4]) 269 t4, c0 = bits.Add64(c1, t4, c0) 270 u5, c1 = bits.Mul64(v, y[5]) 271 t5, c0 = bits.Add64(c1, t5, c0) 272 273 c2, _ = bits.Add64(0, 0, c0) 274 t1, c0 = bits.Add64(u0, t1, 0) 275 t2, c0 = bits.Add64(u1, t2, c0) 276 t3, c0 = bits.Add64(u2, t3, c0) 277 t4, c0 = bits.Add64(u3, t4, c0) 278 t5, c0 = bits.Add64(u4, t5, c0) 279 c2, _ = bits.Add64(u5, c2, c0) 280 281 m := qInvNeg * t0 282 283 u0, c1 = bits.Mul64(m, q0) 284 _, c0 = bits.Add64(t0, c1, 0) 285 u1, c1 = bits.Mul64(m, q1) 286 t0, c0 = bits.Add64(t1, c1, c0) 287 u2, c1 = bits.Mul64(m, q2) 288 t1, c0 = bits.Add64(t2, c1, c0) 289 u3, c1 = bits.Mul64(m, q3) 290 t2, c0 = bits.Add64(t3, c1, c0) 291 u4, c1 = bits.Mul64(m, q4) 292 t3, c0 = bits.Add64(t4, c1, c0) 293 u5, c1 = bits.Mul64(m, q5) 294 295 t4, c0 = bits.Add64(0, c1, c0) 296 u5, _ = bits.Add64(u5, 0, c0) 297 t0, c0 = bits.Add64(u0, t0, 0) 298 t1, c0 = bits.Add64(u1, t1, c0) 299 t2, c0 = bits.Add64(u2, t2, c0) 300 t3, c0 = bits.Add64(u3, t3, c0) 301 t4, c0 = bits.Add64(u4, t4, c0) 302 c2, _ = bits.Add64(c2, 0, c0) 303 t4, c0 = bits.Add64(t5, t4, 0) 304 t5, _ = bits.Add64(u5, c2, c0) 305 306 } 307 { 308 var c0, c1, c2 uint64 309 v := x[4] 310 u0, c1 = bits.Mul64(v, y[0]) 311 t0, c0 = bits.Add64(c1, t0, 0) 312 u1, c1 = bits.Mul64(v, y[1]) 313 t1, c0 = bits.Add64(c1, t1, c0) 314 u2, c1 = bits.Mul64(v, y[2]) 315 t2, c0 = bits.Add64(c1, t2, c0) 316 u3, c1 = bits.Mul64(v, y[3]) 317 t3, c0 = bits.Add64(c1, t3, c0) 318 u4, c1 = bits.Mul64(v, y[4]) 319 t4, c0 = bits.Add64(c1, t4, c0) 320 u5, c1 = bits.Mul64(v, y[5]) 321 t5, c0 = bits.Add64(c1, t5, c0) 322 323 c2, _ = bits.Add64(0, 0, c0) 324 t1, c0 = bits.Add64(u0, t1, 0) 325 t2, c0 = bits.Add64(u1, t2, c0) 326 t3, c0 = bits.Add64(u2, t3, c0) 327 t4, c0 = bits.Add64(u3, t4, c0) 328 t5, c0 = bits.Add64(u4, t5, c0) 329 c2, _ = bits.Add64(u5, c2, c0) 330 331 m := qInvNeg * t0 332 333 u0, c1 = bits.Mul64(m, q0) 334 _, c0 = bits.Add64(t0, c1, 0) 335 u1, c1 = bits.Mul64(m, q1) 336 t0, c0 = bits.Add64(t1, c1, c0) 337 u2, c1 = bits.Mul64(m, q2) 338 t1, c0 = bits.Add64(t2, c1, c0) 339 u3, c1 = bits.Mul64(m, q3) 340 t2, c0 = bits.Add64(t3, c1, c0) 341 u4, c1 = bits.Mul64(m, q4) 342 t3, c0 = bits.Add64(t4, c1, c0) 343 u5, c1 = bits.Mul64(m, q5) 344 345 t4, c0 = bits.Add64(0, c1, c0) 346 u5, _ = bits.Add64(u5, 0, c0) 347 t0, c0 = bits.Add64(u0, t0, 0) 348 t1, c0 = bits.Add64(u1, t1, c0) 349 t2, c0 = bits.Add64(u2, t2, c0) 350 t3, c0 = bits.Add64(u3, t3, c0) 351 t4, c0 = bits.Add64(u4, t4, c0) 352 c2, _ = bits.Add64(c2, 0, c0) 353 t4, c0 = bits.Add64(t5, t4, 0) 354 t5, _ = bits.Add64(u5, c2, c0) 355 356 } 357 { 358 var c0, c1, c2 uint64 359 v := x[5] 360 u0, c1 = bits.Mul64(v, y[0]) 361 t0, c0 = bits.Add64(c1, t0, 0) 362 u1, c1 = bits.Mul64(v, y[1]) 363 t1, c0 = bits.Add64(c1, t1, c0) 364 u2, c1 = bits.Mul64(v, y[2]) 365 t2, c0 = bits.Add64(c1, t2, c0) 366 u3, c1 = bits.Mul64(v, y[3]) 367 t3, c0 = bits.Add64(c1, t3, c0) 368 u4, c1 = bits.Mul64(v, y[4]) 369 t4, c0 = bits.Add64(c1, t4, c0) 370 u5, c1 = bits.Mul64(v, y[5]) 371 t5, c0 = bits.Add64(c1, t5, c0) 372 373 c2, _ = bits.Add64(0, 0, c0) 374 t1, c0 = bits.Add64(u0, t1, 0) 375 t2, c0 = bits.Add64(u1, t2, c0) 376 t3, c0 = bits.Add64(u2, t3, c0) 377 t4, c0 = bits.Add64(u3, t4, c0) 378 t5, c0 = bits.Add64(u4, t5, c0) 379 c2, _ = bits.Add64(u5, c2, c0) 380 381 m := qInvNeg * t0 382 383 u0, c1 = bits.Mul64(m, q0) 384 _, c0 = bits.Add64(t0, c1, 0) 385 u1, c1 = bits.Mul64(m, q1) 386 t0, c0 = bits.Add64(t1, c1, c0) 387 u2, c1 = bits.Mul64(m, q2) 388 t1, c0 = bits.Add64(t2, c1, c0) 389 u3, c1 = bits.Mul64(m, q3) 390 t2, c0 = bits.Add64(t3, c1, c0) 391 u4, c1 = bits.Mul64(m, q4) 392 t3, c0 = bits.Add64(t4, c1, c0) 393 u5, c1 = bits.Mul64(m, q5) 394 395 t4, c0 = bits.Add64(0, c1, c0) 396 u5, _ = bits.Add64(u5, 0, c0) 397 t0, c0 = bits.Add64(u0, t0, 0) 398 t1, c0 = bits.Add64(u1, t1, c0) 399 t2, c0 = bits.Add64(u2, t2, c0) 400 t3, c0 = bits.Add64(u3, t3, c0) 401 t4, c0 = bits.Add64(u4, t4, c0) 402 c2, _ = bits.Add64(c2, 0, c0) 403 t4, c0 = bits.Add64(t5, t4, 0) 404 t5, _ = bits.Add64(u5, c2, c0) 405 406 } 407 z[0] = t0 408 z[1] = t1 409 z[2] = t2 410 z[3] = t3 411 z[4] = t4 412 z[5] = t5 413 414 // if z ⩾ q → z -= q 415 if !z.smallerThanModulus() { 416 var b uint64 417 z[0], b = bits.Sub64(z[0], q0, 0) 418 z[1], b = bits.Sub64(z[1], q1, b) 419 z[2], b = bits.Sub64(z[2], q2, b) 420 z[3], b = bits.Sub64(z[3], q3, b) 421 z[4], b = bits.Sub64(z[4], q4, b) 422 z[5], _ = bits.Sub64(z[5], q5, b) 423 } 424 return z 425 } 426 427 // Square z = x * x (mod q) 428 // 429 // x must be less than q 430 func (z *Element) Square(x *Element) *Element { 431 // see Mul for algorithm documentation 432 433 var t0, t1, t2, t3, t4, t5 uint64 434 var u0, u1, u2, u3, u4, u5 uint64 435 { 436 var c0, c1, c2 uint64 437 v := x[0] 438 u0, t0 = bits.Mul64(v, x[0]) 439 u1, t1 = bits.Mul64(v, x[1]) 440 u2, t2 = bits.Mul64(v, x[2]) 441 u3, t3 = bits.Mul64(v, x[3]) 442 u4, t4 = bits.Mul64(v, x[4]) 443 u5, t5 = bits.Mul64(v, x[5]) 444 t1, c0 = bits.Add64(u0, t1, 0) 445 t2, c0 = bits.Add64(u1, t2, c0) 446 t3, c0 = bits.Add64(u2, t3, c0) 447 t4, c0 = bits.Add64(u3, t4, c0) 448 t5, c0 = bits.Add64(u4, t5, c0) 449 c2, _ = bits.Add64(u5, 0, c0) 450 451 m := qInvNeg * t0 452 453 u0, c1 = bits.Mul64(m, q0) 454 _, c0 = bits.Add64(t0, c1, 0) 455 u1, c1 = bits.Mul64(m, q1) 456 t0, c0 = bits.Add64(t1, c1, c0) 457 u2, c1 = bits.Mul64(m, q2) 458 t1, c0 = bits.Add64(t2, c1, c0) 459 u3, c1 = bits.Mul64(m, q3) 460 t2, c0 = bits.Add64(t3, c1, c0) 461 u4, c1 = bits.Mul64(m, q4) 462 t3, c0 = bits.Add64(t4, c1, c0) 463 u5, c1 = bits.Mul64(m, q5) 464 465 t4, c0 = bits.Add64(0, c1, c0) 466 u5, _ = bits.Add64(u5, 0, c0) 467 t0, c0 = bits.Add64(u0, t0, 0) 468 t1, c0 = bits.Add64(u1, t1, c0) 469 t2, c0 = bits.Add64(u2, t2, c0) 470 t3, c0 = bits.Add64(u3, t3, c0) 471 t4, c0 = bits.Add64(u4, t4, c0) 472 c2, _ = bits.Add64(c2, 0, c0) 473 t4, c0 = bits.Add64(t5, t4, 0) 474 t5, _ = bits.Add64(u5, c2, c0) 475 476 } 477 { 478 var c0, c1, c2 uint64 479 v := x[1] 480 u0, c1 = bits.Mul64(v, x[0]) 481 t0, c0 = bits.Add64(c1, t0, 0) 482 u1, c1 = bits.Mul64(v, x[1]) 483 t1, c0 = bits.Add64(c1, t1, c0) 484 u2, c1 = bits.Mul64(v, x[2]) 485 t2, c0 = bits.Add64(c1, t2, c0) 486 u3, c1 = bits.Mul64(v, x[3]) 487 t3, c0 = bits.Add64(c1, t3, c0) 488 u4, c1 = bits.Mul64(v, x[4]) 489 t4, c0 = bits.Add64(c1, t4, c0) 490 u5, c1 = bits.Mul64(v, x[5]) 491 t5, c0 = bits.Add64(c1, t5, c0) 492 493 c2, _ = bits.Add64(0, 0, c0) 494 t1, c0 = bits.Add64(u0, t1, 0) 495 t2, c0 = bits.Add64(u1, t2, c0) 496 t3, c0 = bits.Add64(u2, t3, c0) 497 t4, c0 = bits.Add64(u3, t4, c0) 498 t5, c0 = bits.Add64(u4, t5, c0) 499 c2, _ = bits.Add64(u5, c2, c0) 500 501 m := qInvNeg * t0 502 503 u0, c1 = bits.Mul64(m, q0) 504 _, c0 = bits.Add64(t0, c1, 0) 505 u1, c1 = bits.Mul64(m, q1) 506 t0, c0 = bits.Add64(t1, c1, c0) 507 u2, c1 = bits.Mul64(m, q2) 508 t1, c0 = bits.Add64(t2, c1, c0) 509 u3, c1 = bits.Mul64(m, q3) 510 t2, c0 = bits.Add64(t3, c1, c0) 511 u4, c1 = bits.Mul64(m, q4) 512 t3, c0 = bits.Add64(t4, c1, c0) 513 u5, c1 = bits.Mul64(m, q5) 514 515 t4, c0 = bits.Add64(0, c1, c0) 516 u5, _ = bits.Add64(u5, 0, c0) 517 t0, c0 = bits.Add64(u0, t0, 0) 518 t1, c0 = bits.Add64(u1, t1, c0) 519 t2, c0 = bits.Add64(u2, t2, c0) 520 t3, c0 = bits.Add64(u3, t3, c0) 521 t4, c0 = bits.Add64(u4, t4, c0) 522 c2, _ = bits.Add64(c2, 0, c0) 523 t4, c0 = bits.Add64(t5, t4, 0) 524 t5, _ = bits.Add64(u5, c2, c0) 525 526 } 527 { 528 var c0, c1, c2 uint64 529 v := x[2] 530 u0, c1 = bits.Mul64(v, x[0]) 531 t0, c0 = bits.Add64(c1, t0, 0) 532 u1, c1 = bits.Mul64(v, x[1]) 533 t1, c0 = bits.Add64(c1, t1, c0) 534 u2, c1 = bits.Mul64(v, x[2]) 535 t2, c0 = bits.Add64(c1, t2, c0) 536 u3, c1 = bits.Mul64(v, x[3]) 537 t3, c0 = bits.Add64(c1, t3, c0) 538 u4, c1 = bits.Mul64(v, x[4]) 539 t4, c0 = bits.Add64(c1, t4, c0) 540 u5, c1 = bits.Mul64(v, x[5]) 541 t5, c0 = bits.Add64(c1, t5, c0) 542 543 c2, _ = bits.Add64(0, 0, c0) 544 t1, c0 = bits.Add64(u0, t1, 0) 545 t2, c0 = bits.Add64(u1, t2, c0) 546 t3, c0 = bits.Add64(u2, t3, c0) 547 t4, c0 = bits.Add64(u3, t4, c0) 548 t5, c0 = bits.Add64(u4, t5, c0) 549 c2, _ = bits.Add64(u5, c2, c0) 550 551 m := qInvNeg * t0 552 553 u0, c1 = bits.Mul64(m, q0) 554 _, c0 = bits.Add64(t0, c1, 0) 555 u1, c1 = bits.Mul64(m, q1) 556 t0, c0 = bits.Add64(t1, c1, c0) 557 u2, c1 = bits.Mul64(m, q2) 558 t1, c0 = bits.Add64(t2, c1, c0) 559 u3, c1 = bits.Mul64(m, q3) 560 t2, c0 = bits.Add64(t3, c1, c0) 561 u4, c1 = bits.Mul64(m, q4) 562 t3, c0 = bits.Add64(t4, c1, c0) 563 u5, c1 = bits.Mul64(m, q5) 564 565 t4, c0 = bits.Add64(0, c1, c0) 566 u5, _ = bits.Add64(u5, 0, c0) 567 t0, c0 = bits.Add64(u0, t0, 0) 568 t1, c0 = bits.Add64(u1, t1, c0) 569 t2, c0 = bits.Add64(u2, t2, c0) 570 t3, c0 = bits.Add64(u3, t3, c0) 571 t4, c0 = bits.Add64(u4, t4, c0) 572 c2, _ = bits.Add64(c2, 0, c0) 573 t4, c0 = bits.Add64(t5, t4, 0) 574 t5, _ = bits.Add64(u5, c2, c0) 575 576 } 577 { 578 var c0, c1, c2 uint64 579 v := x[3] 580 u0, c1 = bits.Mul64(v, x[0]) 581 t0, c0 = bits.Add64(c1, t0, 0) 582 u1, c1 = bits.Mul64(v, x[1]) 583 t1, c0 = bits.Add64(c1, t1, c0) 584 u2, c1 = bits.Mul64(v, x[2]) 585 t2, c0 = bits.Add64(c1, t2, c0) 586 u3, c1 = bits.Mul64(v, x[3]) 587 t3, c0 = bits.Add64(c1, t3, c0) 588 u4, c1 = bits.Mul64(v, x[4]) 589 t4, c0 = bits.Add64(c1, t4, c0) 590 u5, c1 = bits.Mul64(v, x[5]) 591 t5, c0 = bits.Add64(c1, t5, c0) 592 593 c2, _ = bits.Add64(0, 0, c0) 594 t1, c0 = bits.Add64(u0, t1, 0) 595 t2, c0 = bits.Add64(u1, t2, c0) 596 t3, c0 = bits.Add64(u2, t3, c0) 597 t4, c0 = bits.Add64(u3, t4, c0) 598 t5, c0 = bits.Add64(u4, t5, c0) 599 c2, _ = bits.Add64(u5, c2, c0) 600 601 m := qInvNeg * t0 602 603 u0, c1 = bits.Mul64(m, q0) 604 _, c0 = bits.Add64(t0, c1, 0) 605 u1, c1 = bits.Mul64(m, q1) 606 t0, c0 = bits.Add64(t1, c1, c0) 607 u2, c1 = bits.Mul64(m, q2) 608 t1, c0 = bits.Add64(t2, c1, c0) 609 u3, c1 = bits.Mul64(m, q3) 610 t2, c0 = bits.Add64(t3, c1, c0) 611 u4, c1 = bits.Mul64(m, q4) 612 t3, c0 = bits.Add64(t4, c1, c0) 613 u5, c1 = bits.Mul64(m, q5) 614 615 t4, c0 = bits.Add64(0, c1, c0) 616 u5, _ = bits.Add64(u5, 0, c0) 617 t0, c0 = bits.Add64(u0, t0, 0) 618 t1, c0 = bits.Add64(u1, t1, c0) 619 t2, c0 = bits.Add64(u2, t2, c0) 620 t3, c0 = bits.Add64(u3, t3, c0) 621 t4, c0 = bits.Add64(u4, t4, c0) 622 c2, _ = bits.Add64(c2, 0, c0) 623 t4, c0 = bits.Add64(t5, t4, 0) 624 t5, _ = bits.Add64(u5, c2, c0) 625 626 } 627 { 628 var c0, c1, c2 uint64 629 v := x[4] 630 u0, c1 = bits.Mul64(v, x[0]) 631 t0, c0 = bits.Add64(c1, t0, 0) 632 u1, c1 = bits.Mul64(v, x[1]) 633 t1, c0 = bits.Add64(c1, t1, c0) 634 u2, c1 = bits.Mul64(v, x[2]) 635 t2, c0 = bits.Add64(c1, t2, c0) 636 u3, c1 = bits.Mul64(v, x[3]) 637 t3, c0 = bits.Add64(c1, t3, c0) 638 u4, c1 = bits.Mul64(v, x[4]) 639 t4, c0 = bits.Add64(c1, t4, c0) 640 u5, c1 = bits.Mul64(v, x[5]) 641 t5, c0 = bits.Add64(c1, t5, c0) 642 643 c2, _ = bits.Add64(0, 0, c0) 644 t1, c0 = bits.Add64(u0, t1, 0) 645 t2, c0 = bits.Add64(u1, t2, c0) 646 t3, c0 = bits.Add64(u2, t3, c0) 647 t4, c0 = bits.Add64(u3, t4, c0) 648 t5, c0 = bits.Add64(u4, t5, c0) 649 c2, _ = bits.Add64(u5, c2, c0) 650 651 m := qInvNeg * t0 652 653 u0, c1 = bits.Mul64(m, q0) 654 _, c0 = bits.Add64(t0, c1, 0) 655 u1, c1 = bits.Mul64(m, q1) 656 t0, c0 = bits.Add64(t1, c1, c0) 657 u2, c1 = bits.Mul64(m, q2) 658 t1, c0 = bits.Add64(t2, c1, c0) 659 u3, c1 = bits.Mul64(m, q3) 660 t2, c0 = bits.Add64(t3, c1, c0) 661 u4, c1 = bits.Mul64(m, q4) 662 t3, c0 = bits.Add64(t4, c1, c0) 663 u5, c1 = bits.Mul64(m, q5) 664 665 t4, c0 = bits.Add64(0, c1, c0) 666 u5, _ = bits.Add64(u5, 0, c0) 667 t0, c0 = bits.Add64(u0, t0, 0) 668 t1, c0 = bits.Add64(u1, t1, c0) 669 t2, c0 = bits.Add64(u2, t2, c0) 670 t3, c0 = bits.Add64(u3, t3, c0) 671 t4, c0 = bits.Add64(u4, t4, c0) 672 c2, _ = bits.Add64(c2, 0, c0) 673 t4, c0 = bits.Add64(t5, t4, 0) 674 t5, _ = bits.Add64(u5, c2, c0) 675 676 } 677 { 678 var c0, c1, c2 uint64 679 v := x[5] 680 u0, c1 = bits.Mul64(v, x[0]) 681 t0, c0 = bits.Add64(c1, t0, 0) 682 u1, c1 = bits.Mul64(v, x[1]) 683 t1, c0 = bits.Add64(c1, t1, c0) 684 u2, c1 = bits.Mul64(v, x[2]) 685 t2, c0 = bits.Add64(c1, t2, c0) 686 u3, c1 = bits.Mul64(v, x[3]) 687 t3, c0 = bits.Add64(c1, t3, c0) 688 u4, c1 = bits.Mul64(v, x[4]) 689 t4, c0 = bits.Add64(c1, t4, c0) 690 u5, c1 = bits.Mul64(v, x[5]) 691 t5, c0 = bits.Add64(c1, t5, c0) 692 693 c2, _ = bits.Add64(0, 0, c0) 694 t1, c0 = bits.Add64(u0, t1, 0) 695 t2, c0 = bits.Add64(u1, t2, c0) 696 t3, c0 = bits.Add64(u2, t3, c0) 697 t4, c0 = bits.Add64(u3, t4, c0) 698 t5, c0 = bits.Add64(u4, t5, c0) 699 c2, _ = bits.Add64(u5, c2, c0) 700 701 m := qInvNeg * t0 702 703 u0, c1 = bits.Mul64(m, q0) 704 _, c0 = bits.Add64(t0, c1, 0) 705 u1, c1 = bits.Mul64(m, q1) 706 t0, c0 = bits.Add64(t1, c1, c0) 707 u2, c1 = bits.Mul64(m, q2) 708 t1, c0 = bits.Add64(t2, c1, c0) 709 u3, c1 = bits.Mul64(m, q3) 710 t2, c0 = bits.Add64(t3, c1, c0) 711 u4, c1 = bits.Mul64(m, q4) 712 t3, c0 = bits.Add64(t4, c1, c0) 713 u5, c1 = bits.Mul64(m, q5) 714 715 t4, c0 = bits.Add64(0, c1, c0) 716 u5, _ = bits.Add64(u5, 0, c0) 717 t0, c0 = bits.Add64(u0, t0, 0) 718 t1, c0 = bits.Add64(u1, t1, c0) 719 t2, c0 = bits.Add64(u2, t2, c0) 720 t3, c0 = bits.Add64(u3, t3, c0) 721 t4, c0 = bits.Add64(u4, t4, c0) 722 c2, _ = bits.Add64(c2, 0, c0) 723 t4, c0 = bits.Add64(t5, t4, 0) 724 t5, _ = bits.Add64(u5, c2, c0) 725 726 } 727 z[0] = t0 728 z[1] = t1 729 z[2] = t2 730 z[3] = t3 731 z[4] = t4 732 z[5] = t5 733 734 // if z ⩾ q → z -= q 735 if !z.smallerThanModulus() { 736 var b uint64 737 z[0], b = bits.Sub64(z[0], q0, 0) 738 z[1], b = bits.Sub64(z[1], q1, b) 739 z[2], b = bits.Sub64(z[2], q2, b) 740 z[3], b = bits.Sub64(z[3], q3, b) 741 z[4], b = bits.Sub64(z[4], q4, b) 742 z[5], _ = bits.Sub64(z[5], q5, b) 743 } 744 return z 745 }