github.com/klaytn/klaytn@v1.12.1/crypto/secp256k1/libsecp256k1/src/field_10x26_impl.h (about)

     1  /**********************************************************************
     2   * Copyright (c) 2013, 2014 Pieter Wuille                             *
     3   * Distributed under the MIT software license, see the accompanying   *
     4   * file COPYING or http://www.opensource.org/licenses/mit-license.php.*
     5   **********************************************************************/
     6  
     7  #ifndef _SECP256K1_FIELD_REPR_IMPL_H_
     8  #define _SECP256K1_FIELD_REPR_IMPL_H_
     9  
    10  #include "util.h"
    11  #include "num.h"
    12  #include "field.h"
    13  
    14  #ifdef VERIFY
    15  static void secp256k1_fe_verify(const secp256k1_fe *a) {
    16      const uint32_t *d = a->n;
    17      int m = a->normalized ? 1 : 2 * a->magnitude, r = 1;
    18      r &= (d[0] <= 0x3FFFFFFUL * m);
    19      r &= (d[1] <= 0x3FFFFFFUL * m);
    20      r &= (d[2] <= 0x3FFFFFFUL * m);
    21      r &= (d[3] <= 0x3FFFFFFUL * m);
    22      r &= (d[4] <= 0x3FFFFFFUL * m);
    23      r &= (d[5] <= 0x3FFFFFFUL * m);
    24      r &= (d[6] <= 0x3FFFFFFUL * m);
    25      r &= (d[7] <= 0x3FFFFFFUL * m);
    26      r &= (d[8] <= 0x3FFFFFFUL * m);
    27      r &= (d[9] <= 0x03FFFFFUL * m);
    28      r &= (a->magnitude >= 0);
    29      r &= (a->magnitude <= 32);
    30      if (a->normalized) {
    31          r &= (a->magnitude <= 1);
    32          if (r && (d[9] == 0x03FFFFFUL)) {
    33              uint32_t mid = d[8] & d[7] & d[6] & d[5] & d[4] & d[3] & d[2];
    34              if (mid == 0x3FFFFFFUL) {
    35                  r &= ((d[1] + 0x40UL + ((d[0] + 0x3D1UL) >> 26)) <= 0x3FFFFFFUL);
    36              }
    37          }
    38      }
    39      VERIFY_CHECK(r == 1);
    40  }
    41  #endif
    42  
    43  static void secp256k1_fe_normalize(secp256k1_fe *r) {
    44      uint32_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4],
    45               t5 = r->n[5], t6 = r->n[6], t7 = r->n[7], t8 = r->n[8], t9 = r->n[9];
    46  
    47      /* Reduce t9 at the start so there will be at most a single carry from the first pass */
    48      uint32_t m;
    49      uint32_t x = t9 >> 22; t9 &= 0x03FFFFFUL;
    50  
    51      /* The first pass ensures the magnitude is 1, ... */
    52      t0 += x * 0x3D1UL; t1 += (x << 6);
    53      t1 += (t0 >> 26); t0 &= 0x3FFFFFFUL;
    54      t2 += (t1 >> 26); t1 &= 0x3FFFFFFUL;
    55      t3 += (t2 >> 26); t2 &= 0x3FFFFFFUL; m = t2;
    56      t4 += (t3 >> 26); t3 &= 0x3FFFFFFUL; m &= t3;
    57      t5 += (t4 >> 26); t4 &= 0x3FFFFFFUL; m &= t4;
    58      t6 += (t5 >> 26); t5 &= 0x3FFFFFFUL; m &= t5;
    59      t7 += (t6 >> 26); t6 &= 0x3FFFFFFUL; m &= t6;
    60      t8 += (t7 >> 26); t7 &= 0x3FFFFFFUL; m &= t7;
    61      t9 += (t8 >> 26); t8 &= 0x3FFFFFFUL; m &= t8;
    62  
    63      /* ... except for a possible carry at bit 22 of t9 (i.e. bit 256 of the field element) */
    64      VERIFY_CHECK(t9 >> 23 == 0);
    65  
    66      /* At most a single final reduction is needed; check if the value is >= the field characteristic */
    67      x = (t9 >> 22) | ((t9 == 0x03FFFFFUL) & (m == 0x3FFFFFFUL)
    68          & ((t1 + 0x40UL + ((t0 + 0x3D1UL) >> 26)) > 0x3FFFFFFUL));
    69  
    70      /* Apply the final reduction (for constant-time behaviour, we do it always) */
    71      t0 += x * 0x3D1UL; t1 += (x << 6);
    72      t1 += (t0 >> 26); t0 &= 0x3FFFFFFUL;
    73      t2 += (t1 >> 26); t1 &= 0x3FFFFFFUL;
    74      t3 += (t2 >> 26); t2 &= 0x3FFFFFFUL;
    75      t4 += (t3 >> 26); t3 &= 0x3FFFFFFUL;
    76      t5 += (t4 >> 26); t4 &= 0x3FFFFFFUL;
    77      t6 += (t5 >> 26); t5 &= 0x3FFFFFFUL;
    78      t7 += (t6 >> 26); t6 &= 0x3FFFFFFUL;
    79      t8 += (t7 >> 26); t7 &= 0x3FFFFFFUL;
    80      t9 += (t8 >> 26); t8 &= 0x3FFFFFFUL;
    81  
    82      /* If t9 didn't carry to bit 22 already, then it should have after any final reduction */
    83      VERIFY_CHECK(t9 >> 22 == x);
    84  
    85      /* Mask off the possible multiple of 2^256 from the final reduction */
    86      t9 &= 0x03FFFFFUL;
    87  
    88      r->n[0] = t0; r->n[1] = t1; r->n[2] = t2; r->n[3] = t3; r->n[4] = t4;
    89      r->n[5] = t5; r->n[6] = t6; r->n[7] = t7; r->n[8] = t8; r->n[9] = t9;
    90  
    91  #ifdef VERIFY
    92      r->magnitude = 1;
    93      r->normalized = 1;
    94      secp256k1_fe_verify(r);
    95  #endif
    96  }
    97  
    98  static void secp256k1_fe_normalize_weak(secp256k1_fe *r) {
    99      uint32_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4],
   100               t5 = r->n[5], t6 = r->n[6], t7 = r->n[7], t8 = r->n[8], t9 = r->n[9];
   101  
   102      /* Reduce t9 at the start so there will be at most a single carry from the first pass */
   103      uint32_t x = t9 >> 22; t9 &= 0x03FFFFFUL;
   104  
   105      /* The first pass ensures the magnitude is 1, ... */
   106      t0 += x * 0x3D1UL; t1 += (x << 6);
   107      t1 += (t0 >> 26); t0 &= 0x3FFFFFFUL;
   108      t2 += (t1 >> 26); t1 &= 0x3FFFFFFUL;
   109      t3 += (t2 >> 26); t2 &= 0x3FFFFFFUL;
   110      t4 += (t3 >> 26); t3 &= 0x3FFFFFFUL;
   111      t5 += (t4 >> 26); t4 &= 0x3FFFFFFUL;
   112      t6 += (t5 >> 26); t5 &= 0x3FFFFFFUL;
   113      t7 += (t6 >> 26); t6 &= 0x3FFFFFFUL;
   114      t8 += (t7 >> 26); t7 &= 0x3FFFFFFUL;
   115      t9 += (t8 >> 26); t8 &= 0x3FFFFFFUL;
   116  
   117      /* ... except for a possible carry at bit 22 of t9 (i.e. bit 256 of the field element) */
   118      VERIFY_CHECK(t9 >> 23 == 0);
   119  
   120      r->n[0] = t0; r->n[1] = t1; r->n[2] = t2; r->n[3] = t3; r->n[4] = t4;
   121      r->n[5] = t5; r->n[6] = t6; r->n[7] = t7; r->n[8] = t8; r->n[9] = t9;
   122  
   123  #ifdef VERIFY
   124      r->magnitude = 1;
   125      secp256k1_fe_verify(r);
   126  #endif
   127  }
   128  
   129  static void secp256k1_fe_normalize_var(secp256k1_fe *r) {
   130      uint32_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4],
   131               t5 = r->n[5], t6 = r->n[6], t7 = r->n[7], t8 = r->n[8], t9 = r->n[9];
   132  
   133      /* Reduce t9 at the start so there will be at most a single carry from the first pass */
   134      uint32_t m;
   135      uint32_t x = t9 >> 22; t9 &= 0x03FFFFFUL;
   136  
   137      /* The first pass ensures the magnitude is 1, ... */
   138      t0 += x * 0x3D1UL; t1 += (x << 6);
   139      t1 += (t0 >> 26); t0 &= 0x3FFFFFFUL;
   140      t2 += (t1 >> 26); t1 &= 0x3FFFFFFUL;
   141      t3 += (t2 >> 26); t2 &= 0x3FFFFFFUL; m = t2;
   142      t4 += (t3 >> 26); t3 &= 0x3FFFFFFUL; m &= t3;
   143      t5 += (t4 >> 26); t4 &= 0x3FFFFFFUL; m &= t4;
   144      t6 += (t5 >> 26); t5 &= 0x3FFFFFFUL; m &= t5;
   145      t7 += (t6 >> 26); t6 &= 0x3FFFFFFUL; m &= t6;
   146      t8 += (t7 >> 26); t7 &= 0x3FFFFFFUL; m &= t7;
   147      t9 += (t8 >> 26); t8 &= 0x3FFFFFFUL; m &= t8;
   148  
   149      /* ... except for a possible carry at bit 22 of t9 (i.e. bit 256 of the field element) */
   150      VERIFY_CHECK(t9 >> 23 == 0);
   151  
   152      /* At most a single final reduction is needed; check if the value is >= the field characteristic */
   153      x = (t9 >> 22) | ((t9 == 0x03FFFFFUL) & (m == 0x3FFFFFFUL)
   154          & ((t1 + 0x40UL + ((t0 + 0x3D1UL) >> 26)) > 0x3FFFFFFUL));
   155  
   156      if (x) {
   157          t0 += 0x3D1UL; t1 += (x << 6);
   158          t1 += (t0 >> 26); t0 &= 0x3FFFFFFUL;
   159          t2 += (t1 >> 26); t1 &= 0x3FFFFFFUL;
   160          t3 += (t2 >> 26); t2 &= 0x3FFFFFFUL;
   161          t4 += (t3 >> 26); t3 &= 0x3FFFFFFUL;
   162          t5 += (t4 >> 26); t4 &= 0x3FFFFFFUL;
   163          t6 += (t5 >> 26); t5 &= 0x3FFFFFFUL;
   164          t7 += (t6 >> 26); t6 &= 0x3FFFFFFUL;
   165          t8 += (t7 >> 26); t7 &= 0x3FFFFFFUL;
   166          t9 += (t8 >> 26); t8 &= 0x3FFFFFFUL;
   167  
   168          /* If t9 didn't carry to bit 22 already, then it should have after any final reduction */
   169          VERIFY_CHECK(t9 >> 22 == x);
   170  
   171          /* Mask off the possible multiple of 2^256 from the final reduction */
   172          t9 &= 0x03FFFFFUL;
   173      }
   174  
   175      r->n[0] = t0; r->n[1] = t1; r->n[2] = t2; r->n[3] = t3; r->n[4] = t4;
   176      r->n[5] = t5; r->n[6] = t6; r->n[7] = t7; r->n[8] = t8; r->n[9] = t9;
   177  
   178  #ifdef VERIFY
   179      r->magnitude = 1;
   180      r->normalized = 1;
   181      secp256k1_fe_verify(r);
   182  #endif
   183  }
   184  
   185  static int secp256k1_fe_normalizes_to_zero(secp256k1_fe *r) {
   186      uint32_t t0 = r->n[0], t1 = r->n[1], t2 = r->n[2], t3 = r->n[3], t4 = r->n[4],
   187               t5 = r->n[5], t6 = r->n[6], t7 = r->n[7], t8 = r->n[8], t9 = r->n[9];
   188  
   189      /* z0 tracks a possible raw value of 0, z1 tracks a possible raw value of P */
   190      uint32_t z0, z1;
   191  
   192      /* Reduce t9 at the start so there will be at most a single carry from the first pass */
   193      uint32_t x = t9 >> 22; t9 &= 0x03FFFFFUL;
   194  
   195      /* The first pass ensures the magnitude is 1, ... */
   196      t0 += x * 0x3D1UL; t1 += (x << 6);
   197      t1 += (t0 >> 26); t0 &= 0x3FFFFFFUL; z0  = t0; z1  = t0 ^ 0x3D0UL;
   198      t2 += (t1 >> 26); t1 &= 0x3FFFFFFUL; z0 |= t1; z1 &= t1 ^ 0x40UL;
   199      t3 += (t2 >> 26); t2 &= 0x3FFFFFFUL; z0 |= t2; z1 &= t2;
   200      t4 += (t3 >> 26); t3 &= 0x3FFFFFFUL; z0 |= t3; z1 &= t3;
   201      t5 += (t4 >> 26); t4 &= 0x3FFFFFFUL; z0 |= t4; z1 &= t4;
   202      t6 += (t5 >> 26); t5 &= 0x3FFFFFFUL; z0 |= t5; z1 &= t5;
   203      t7 += (t6 >> 26); t6 &= 0x3FFFFFFUL; z0 |= t6; z1 &= t6;
   204      t8 += (t7 >> 26); t7 &= 0x3FFFFFFUL; z0 |= t7; z1 &= t7;
   205      t9 += (t8 >> 26); t8 &= 0x3FFFFFFUL; z0 |= t8; z1 &= t8;
   206                                           z0 |= t9; z1 &= t9 ^ 0x3C00000UL;
   207  
   208      /* ... except for a possible carry at bit 22 of t9 (i.e. bit 256 of the field element) */
   209      VERIFY_CHECK(t9 >> 23 == 0);
   210  
   211      return (z0 == 0) | (z1 == 0x3FFFFFFUL);
   212  }
   213  
   214  static int secp256k1_fe_normalizes_to_zero_var(secp256k1_fe *r) {
   215      uint32_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9;
   216      uint32_t z0, z1;
   217      uint32_t x;
   218  
   219      t0 = r->n[0];
   220      t9 = r->n[9];
   221  
   222      /* Reduce t9 at the start so there will be at most a single carry from the first pass */
   223      x = t9 >> 22;
   224  
   225      /* The first pass ensures the magnitude is 1, ... */
   226      t0 += x * 0x3D1UL;
   227  
   228      /* z0 tracks a possible raw value of 0, z1 tracks a possible raw value of P */
   229      z0 = t0 & 0x3FFFFFFUL;
   230      z1 = z0 ^ 0x3D0UL;
   231  
   232      /* Fast return path should catch the majority of cases */
   233      if ((z0 != 0UL) & (z1 != 0x3FFFFFFUL)) {
   234          return 0;
   235      }
   236  
   237      t1 = r->n[1];
   238      t2 = r->n[2];
   239      t3 = r->n[3];
   240      t4 = r->n[4];
   241      t5 = r->n[5];
   242      t6 = r->n[6];
   243      t7 = r->n[7];
   244      t8 = r->n[8];
   245  
   246      t9 &= 0x03FFFFFUL;
   247      t1 += (x << 6);
   248  
   249      t1 += (t0 >> 26);
   250      t2 += (t1 >> 26); t1 &= 0x3FFFFFFUL; z0 |= t1; z1 &= t1 ^ 0x40UL;
   251      t3 += (t2 >> 26); t2 &= 0x3FFFFFFUL; z0 |= t2; z1 &= t2;
   252      t4 += (t3 >> 26); t3 &= 0x3FFFFFFUL; z0 |= t3; z1 &= t3;
   253      t5 += (t4 >> 26); t4 &= 0x3FFFFFFUL; z0 |= t4; z1 &= t4;
   254      t6 += (t5 >> 26); t5 &= 0x3FFFFFFUL; z0 |= t5; z1 &= t5;
   255      t7 += (t6 >> 26); t6 &= 0x3FFFFFFUL; z0 |= t6; z1 &= t6;
   256      t8 += (t7 >> 26); t7 &= 0x3FFFFFFUL; z0 |= t7; z1 &= t7;
   257      t9 += (t8 >> 26); t8 &= 0x3FFFFFFUL; z0 |= t8; z1 &= t8;
   258                                           z0 |= t9; z1 &= t9 ^ 0x3C00000UL;
   259  
   260      /* ... except for a possible carry at bit 22 of t9 (i.e. bit 256 of the field element) */
   261      VERIFY_CHECK(t9 >> 23 == 0);
   262  
   263      return (z0 == 0) | (z1 == 0x3FFFFFFUL);
   264  }
   265  
   266  SECP256K1_INLINE static void secp256k1_fe_set_int(secp256k1_fe *r, int a) {
   267      r->n[0] = a;
   268      r->n[1] = r->n[2] = r->n[3] = r->n[4] = r->n[5] = r->n[6] = r->n[7] = r->n[8] = r->n[9] = 0;
   269  #ifdef VERIFY
   270      r->magnitude = 1;
   271      r->normalized = 1;
   272      secp256k1_fe_verify(r);
   273  #endif
   274  }
   275  
   276  SECP256K1_INLINE static int secp256k1_fe_is_zero(const secp256k1_fe *a) {
   277      const uint32_t *t = a->n;
   278  #ifdef VERIFY
   279      VERIFY_CHECK(a->normalized);
   280      secp256k1_fe_verify(a);
   281  #endif
   282      return (t[0] | t[1] | t[2] | t[3] | t[4] | t[5] | t[6] | t[7] | t[8] | t[9]) == 0;
   283  }
   284  
   285  SECP256K1_INLINE static int secp256k1_fe_is_odd(const secp256k1_fe *a) {
   286  #ifdef VERIFY
   287      VERIFY_CHECK(a->normalized);
   288      secp256k1_fe_verify(a);
   289  #endif
   290      return a->n[0] & 1;
   291  }
   292  
   293  SECP256K1_INLINE static void secp256k1_fe_clear(secp256k1_fe *a) {
   294      int i;
   295  #ifdef VERIFY
   296      a->magnitude = 0;
   297      a->normalized = 1;
   298  #endif
   299      for (i=0; i<10; i++) {
   300          a->n[i] = 0;
   301      }
   302  }
   303  
   304  static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) {
   305      int i;
   306  #ifdef VERIFY
   307      VERIFY_CHECK(a->normalized);
   308      VERIFY_CHECK(b->normalized);
   309      secp256k1_fe_verify(a);
   310      secp256k1_fe_verify(b);
   311  #endif
   312      for (i = 9; i >= 0; i--) {
   313          if (a->n[i] > b->n[i]) {
   314              return 1;
   315          }
   316          if (a->n[i] < b->n[i]) {
   317              return -1;
   318          }
   319      }
   320      return 0;
   321  }
   322  
   323  static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) {
   324      int i;
   325      r->n[0] = r->n[1] = r->n[2] = r->n[3] = r->n[4] = 0;
   326      r->n[5] = r->n[6] = r->n[7] = r->n[8] = r->n[9] = 0;
   327      for (i=0; i<32; i++) {
   328          int j;
   329          for (j=0; j<4; j++) {
   330              int limb = (8*i+2*j)/26;
   331              int shift = (8*i+2*j)%26;
   332              r->n[limb] |= (uint32_t)((a[31-i] >> (2*j)) & 0x3) << shift;
   333          }
   334      }
   335      if (r->n[9] == 0x3FFFFFUL && (r->n[8] & r->n[7] & r->n[6] & r->n[5] & r->n[4] & r->n[3] & r->n[2]) == 0x3FFFFFFUL && (r->n[1] + 0x40UL + ((r->n[0] + 0x3D1UL) >> 26)) > 0x3FFFFFFUL) {
   336          return 0;
   337      }
   338  #ifdef VERIFY
   339      r->magnitude = 1;
   340      r->normalized = 1;
   341      secp256k1_fe_verify(r);
   342  #endif
   343      return 1;
   344  }
   345  
   346  /** Convert a field element to a 32-byte big endian value. Requires the input to be normalized */
   347  static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) {
   348      int i;
   349  #ifdef VERIFY
   350      VERIFY_CHECK(a->normalized);
   351      secp256k1_fe_verify(a);
   352  #endif
   353      for (i=0; i<32; i++) {
   354          int j;
   355          int c = 0;
   356          for (j=0; j<4; j++) {
   357              int limb = (8*i+2*j)/26;
   358              int shift = (8*i+2*j)%26;
   359              c |= ((a->n[limb] >> shift) & 0x3) << (2 * j);
   360          }
   361          r[31-i] = c;
   362      }
   363  }
   364  
   365  SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k1_fe *a, int m) {
   366  #ifdef VERIFY
   367      VERIFY_CHECK(a->magnitude <= m);
   368      secp256k1_fe_verify(a);
   369  #endif
   370      r->n[0] = 0x3FFFC2FUL * 2 * (m + 1) - a->n[0];
   371      r->n[1] = 0x3FFFFBFUL * 2 * (m + 1) - a->n[1];
   372      r->n[2] = 0x3FFFFFFUL * 2 * (m + 1) - a->n[2];
   373      r->n[3] = 0x3FFFFFFUL * 2 * (m + 1) - a->n[3];
   374      r->n[4] = 0x3FFFFFFUL * 2 * (m + 1) - a->n[4];
   375      r->n[5] = 0x3FFFFFFUL * 2 * (m + 1) - a->n[5];
   376      r->n[6] = 0x3FFFFFFUL * 2 * (m + 1) - a->n[6];
   377      r->n[7] = 0x3FFFFFFUL * 2 * (m + 1) - a->n[7];
   378      r->n[8] = 0x3FFFFFFUL * 2 * (m + 1) - a->n[8];
   379      r->n[9] = 0x03FFFFFUL * 2 * (m + 1) - a->n[9];
   380  #ifdef VERIFY
   381      r->magnitude = m + 1;
   382      r->normalized = 0;
   383      secp256k1_fe_verify(r);
   384  #endif
   385  }
   386  
   387  SECP256K1_INLINE static void secp256k1_fe_mul_int(secp256k1_fe *r, int a) {
   388      r->n[0] *= a;
   389      r->n[1] *= a;
   390      r->n[2] *= a;
   391      r->n[3] *= a;
   392      r->n[4] *= a;
   393      r->n[5] *= a;
   394      r->n[6] *= a;
   395      r->n[7] *= a;
   396      r->n[8] *= a;
   397      r->n[9] *= a;
   398  #ifdef VERIFY
   399      r->magnitude *= a;
   400      r->normalized = 0;
   401      secp256k1_fe_verify(r);
   402  #endif
   403  }
   404  
   405  SECP256K1_INLINE static void secp256k1_fe_add(secp256k1_fe *r, const secp256k1_fe *a) {
   406  #ifdef VERIFY
   407      secp256k1_fe_verify(a);
   408  #endif
   409      r->n[0] += a->n[0];
   410      r->n[1] += a->n[1];
   411      r->n[2] += a->n[2];
   412      r->n[3] += a->n[3];
   413      r->n[4] += a->n[4];
   414      r->n[5] += a->n[5];
   415      r->n[6] += a->n[6];
   416      r->n[7] += a->n[7];
   417      r->n[8] += a->n[8];
   418      r->n[9] += a->n[9];
   419  #ifdef VERIFY
   420      r->magnitude += a->magnitude;
   421      r->normalized = 0;
   422      secp256k1_fe_verify(r);
   423  #endif
   424  }
   425  
   426  #if defined(USE_EXTERNAL_ASM)
   427  
   428  /* External assembler implementation */
   429  void secp256k1_fe_mul_inner(uint32_t *r, const uint32_t *a, const uint32_t * SECP256K1_RESTRICT b);
   430  void secp256k1_fe_sqr_inner(uint32_t *r, const uint32_t *a);
   431  
   432  #else
   433  
   434  #ifdef VERIFY
   435  #define VERIFY_BITS(x, n) VERIFY_CHECK(((x) >> (n)) == 0)
   436  #else
   437  #define VERIFY_BITS(x, n) do { } while(0)
   438  #endif
   439  
   440  SECP256K1_INLINE static void secp256k1_fe_mul_inner(uint32_t *r, const uint32_t *a, const uint32_t * SECP256K1_RESTRICT b) {
   441      uint64_t c, d;
   442      uint64_t u0, u1, u2, u3, u4, u5, u6, u7, u8;
   443      uint32_t t9, t1, t0, t2, t3, t4, t5, t6, t7;
   444      const uint32_t M = 0x3FFFFFFUL, R0 = 0x3D10UL, R1 = 0x400UL;
   445  
   446      VERIFY_BITS(a[0], 30);
   447      VERIFY_BITS(a[1], 30);
   448      VERIFY_BITS(a[2], 30);
   449      VERIFY_BITS(a[3], 30);
   450      VERIFY_BITS(a[4], 30);
   451      VERIFY_BITS(a[5], 30);
   452      VERIFY_BITS(a[6], 30);
   453      VERIFY_BITS(a[7], 30);
   454      VERIFY_BITS(a[8], 30);
   455      VERIFY_BITS(a[9], 26);
   456      VERIFY_BITS(b[0], 30);
   457      VERIFY_BITS(b[1], 30);
   458      VERIFY_BITS(b[2], 30);
   459      VERIFY_BITS(b[3], 30);
   460      VERIFY_BITS(b[4], 30);
   461      VERIFY_BITS(b[5], 30);
   462      VERIFY_BITS(b[6], 30);
   463      VERIFY_BITS(b[7], 30);
   464      VERIFY_BITS(b[8], 30);
   465      VERIFY_BITS(b[9], 26);
   466  
   467      /** [... a b c] is a shorthand for ... + a<<52 + b<<26 + c<<0 mod n.
   468       *  px is a shorthand for sum(a[i]*b[x-i], i=0..x).
   469       *  Note that [x 0 0 0 0 0 0 0 0 0 0] = [x*R1 x*R0].
   470       */
   471  
   472      d  = (uint64_t)a[0] * b[9]
   473         + (uint64_t)a[1] * b[8]
   474         + (uint64_t)a[2] * b[7]
   475         + (uint64_t)a[3] * b[6]
   476         + (uint64_t)a[4] * b[5]
   477         + (uint64_t)a[5] * b[4]
   478         + (uint64_t)a[6] * b[3]
   479         + (uint64_t)a[7] * b[2]
   480         + (uint64_t)a[8] * b[1]
   481         + (uint64_t)a[9] * b[0];
   482      /* VERIFY_BITS(d, 64); */
   483      /* [d 0 0 0 0 0 0 0 0 0] = [p9 0 0 0 0 0 0 0 0 0] */
   484      t9 = d & M; d >>= 26;
   485      VERIFY_BITS(t9, 26);
   486      VERIFY_BITS(d, 38);
   487      /* [d t9 0 0 0 0 0 0 0 0 0] = [p9 0 0 0 0 0 0 0 0 0] */
   488  
   489      c  = (uint64_t)a[0] * b[0];
   490      VERIFY_BITS(c, 60);
   491      /* [d t9 0 0 0 0 0 0 0 0 c] = [p9 0 0 0 0 0 0 0 0 p0] */
   492      d += (uint64_t)a[1] * b[9]
   493         + (uint64_t)a[2] * b[8]
   494         + (uint64_t)a[3] * b[7]
   495         + (uint64_t)a[4] * b[6]
   496         + (uint64_t)a[5] * b[5]
   497         + (uint64_t)a[6] * b[4]
   498         + (uint64_t)a[7] * b[3]
   499         + (uint64_t)a[8] * b[2]
   500         + (uint64_t)a[9] * b[1];
   501      VERIFY_BITS(d, 63);
   502      /* [d t9 0 0 0 0 0 0 0 0 c] = [p10 p9 0 0 0 0 0 0 0 0 p0] */
   503      u0 = d & M; d >>= 26; c += u0 * R0;
   504      VERIFY_BITS(u0, 26);
   505      VERIFY_BITS(d, 37);
   506      VERIFY_BITS(c, 61);
   507      /* [d u0 t9 0 0 0 0 0 0 0 0 c-u0*R0] = [p10 p9 0 0 0 0 0 0 0 0 p0] */
   508      t0 = c & M; c >>= 26; c += u0 * R1;
   509      VERIFY_BITS(t0, 26);
   510      VERIFY_BITS(c, 37);
   511      /* [d u0 t9 0 0 0 0 0 0 0 c-u0*R1 t0-u0*R0] = [p10 p9 0 0 0 0 0 0 0 0 p0] */
   512      /* [d 0 t9 0 0 0 0 0 0 0 c t0] = [p10 p9 0 0 0 0 0 0 0 0 p0] */
   513  
   514      c += (uint64_t)a[0] * b[1]
   515         + (uint64_t)a[1] * b[0];
   516      VERIFY_BITS(c, 62);
   517      /* [d 0 t9 0 0 0 0 0 0 0 c t0] = [p10 p9 0 0 0 0 0 0 0 p1 p0] */
   518      d += (uint64_t)a[2] * b[9]
   519         + (uint64_t)a[3] * b[8]
   520         + (uint64_t)a[4] * b[7]
   521         + (uint64_t)a[5] * b[6]
   522         + (uint64_t)a[6] * b[5]
   523         + (uint64_t)a[7] * b[4]
   524         + (uint64_t)a[8] * b[3]
   525         + (uint64_t)a[9] * b[2];
   526      VERIFY_BITS(d, 63);
   527      /* [d 0 t9 0 0 0 0 0 0 0 c t0] = [p11 p10 p9 0 0 0 0 0 0 0 p1 p0] */
   528      u1 = d & M; d >>= 26; c += u1 * R0;
   529      VERIFY_BITS(u1, 26);
   530      VERIFY_BITS(d, 37);
   531      VERIFY_BITS(c, 63);
   532      /* [d u1 0 t9 0 0 0 0 0 0 0 c-u1*R0 t0] = [p11 p10 p9 0 0 0 0 0 0 0 p1 p0] */
   533      t1 = c & M; c >>= 26; c += u1 * R1;
   534      VERIFY_BITS(t1, 26);
   535      VERIFY_BITS(c, 38);
   536      /* [d u1 0 t9 0 0 0 0 0 0 c-u1*R1 t1-u1*R0 t0] = [p11 p10 p9 0 0 0 0 0 0 0 p1 p0] */
   537      /* [d 0 0 t9 0 0 0 0 0 0 c t1 t0] = [p11 p10 p9 0 0 0 0 0 0 0 p1 p0] */
   538  
   539      c += (uint64_t)a[0] * b[2]
   540         + (uint64_t)a[1] * b[1]
   541         + (uint64_t)a[2] * b[0];
   542      VERIFY_BITS(c, 62);
   543      /* [d 0 0 t9 0 0 0 0 0 0 c t1 t0] = [p11 p10 p9 0 0 0 0 0 0 p2 p1 p0] */
   544      d += (uint64_t)a[3] * b[9]
   545         + (uint64_t)a[4] * b[8]
   546         + (uint64_t)a[5] * b[7]
   547         + (uint64_t)a[6] * b[6]
   548         + (uint64_t)a[7] * b[5]
   549         + (uint64_t)a[8] * b[4]
   550         + (uint64_t)a[9] * b[3];
   551      VERIFY_BITS(d, 63);
   552      /* [d 0 0 t9 0 0 0 0 0 0 c t1 t0] = [p12 p11 p10 p9 0 0 0 0 0 0 p2 p1 p0] */
   553      u2 = d & M; d >>= 26; c += u2 * R0;
   554      VERIFY_BITS(u2, 26);
   555      VERIFY_BITS(d, 37);
   556      VERIFY_BITS(c, 63);
   557      /* [d u2 0 0 t9 0 0 0 0 0 0 c-u2*R0 t1 t0] = [p12 p11 p10 p9 0 0 0 0 0 0 p2 p1 p0] */
   558      t2 = c & M; c >>= 26; c += u2 * R1;
   559      VERIFY_BITS(t2, 26);
   560      VERIFY_BITS(c, 38);
   561      /* [d u2 0 0 t9 0 0 0 0 0 c-u2*R1 t2-u2*R0 t1 t0] = [p12 p11 p10 p9 0 0 0 0 0 0 p2 p1 p0] */
   562      /* [d 0 0 0 t9 0 0 0 0 0 c t2 t1 t0] = [p12 p11 p10 p9 0 0 0 0 0 0 p2 p1 p0] */
   563  
   564      c += (uint64_t)a[0] * b[3]
   565         + (uint64_t)a[1] * b[2]
   566         + (uint64_t)a[2] * b[1]
   567         + (uint64_t)a[3] * b[0];
   568      VERIFY_BITS(c, 63);
   569      /* [d 0 0 0 t9 0 0 0 0 0 c t2 t1 t0] = [p12 p11 p10 p9 0 0 0 0 0 p3 p2 p1 p0] */
   570      d += (uint64_t)a[4] * b[9]
   571         + (uint64_t)a[5] * b[8]
   572         + (uint64_t)a[6] * b[7]
   573         + (uint64_t)a[7] * b[6]
   574         + (uint64_t)a[8] * b[5]
   575         + (uint64_t)a[9] * b[4];
   576      VERIFY_BITS(d, 63);
   577      /* [d 0 0 0 t9 0 0 0 0 0 c t2 t1 t0] = [p13 p12 p11 p10 p9 0 0 0 0 0 p3 p2 p1 p0] */
   578      u3 = d & M; d >>= 26; c += u3 * R0;
   579      VERIFY_BITS(u3, 26);
   580      VERIFY_BITS(d, 37);
   581      /* VERIFY_BITS(c, 64); */
   582      /* [d u3 0 0 0 t9 0 0 0 0 0 c-u3*R0 t2 t1 t0] = [p13 p12 p11 p10 p9 0 0 0 0 0 p3 p2 p1 p0] */
   583      t3 = c & M; c >>= 26; c += u3 * R1;
   584      VERIFY_BITS(t3, 26);
   585      VERIFY_BITS(c, 39);
   586      /* [d u3 0 0 0 t9 0 0 0 0 c-u3*R1 t3-u3*R0 t2 t1 t0] = [p13 p12 p11 p10 p9 0 0 0 0 0 p3 p2 p1 p0] */
   587      /* [d 0 0 0 0 t9 0 0 0 0 c t3 t2 t1 t0] = [p13 p12 p11 p10 p9 0 0 0 0 0 p3 p2 p1 p0] */
   588  
   589      c += (uint64_t)a[0] * b[4]
   590         + (uint64_t)a[1] * b[3]
   591         + (uint64_t)a[2] * b[2]
   592         + (uint64_t)a[3] * b[1]
   593         + (uint64_t)a[4] * b[0];
   594      VERIFY_BITS(c, 63);
   595      /* [d 0 0 0 0 t9 0 0 0 0 c t3 t2 t1 t0] = [p13 p12 p11 p10 p9 0 0 0 0 p4 p3 p2 p1 p0] */
   596      d += (uint64_t)a[5] * b[9]
   597         + (uint64_t)a[6] * b[8]
   598         + (uint64_t)a[7] * b[7]
   599         + (uint64_t)a[8] * b[6]
   600         + (uint64_t)a[9] * b[5];
   601      VERIFY_BITS(d, 62);
   602      /* [d 0 0 0 0 t9 0 0 0 0 c t3 t2 t1 t0] = [p14 p13 p12 p11 p10 p9 0 0 0 0 p4 p3 p2 p1 p0] */
   603      u4 = d & M; d >>= 26; c += u4 * R0;
   604      VERIFY_BITS(u4, 26);
   605      VERIFY_BITS(d, 36);
   606      /* VERIFY_BITS(c, 64); */
   607      /* [d u4 0 0 0 0 t9 0 0 0 0 c-u4*R0 t3 t2 t1 t0] = [p14 p13 p12 p11 p10 p9 0 0 0 0 p4 p3 p2 p1 p0] */
   608      t4 = c & M; c >>= 26; c += u4 * R1;
   609      VERIFY_BITS(t4, 26);
   610      VERIFY_BITS(c, 39);
   611      /* [d u4 0 0 0 0 t9 0 0 0 c-u4*R1 t4-u4*R0 t3 t2 t1 t0] = [p14 p13 p12 p11 p10 p9 0 0 0 0 p4 p3 p2 p1 p0] */
   612      /* [d 0 0 0 0 0 t9 0 0 0 c t4 t3 t2 t1 t0] = [p14 p13 p12 p11 p10 p9 0 0 0 0 p4 p3 p2 p1 p0] */
   613  
   614      c += (uint64_t)a[0] * b[5]
   615         + (uint64_t)a[1] * b[4]
   616         + (uint64_t)a[2] * b[3]
   617         + (uint64_t)a[3] * b[2]
   618         + (uint64_t)a[4] * b[1]
   619         + (uint64_t)a[5] * b[0];
   620      VERIFY_BITS(c, 63);
   621      /* [d 0 0 0 0 0 t9 0 0 0 c t4 t3 t2 t1 t0] = [p14 p13 p12 p11 p10 p9 0 0 0 p5 p4 p3 p2 p1 p0] */
   622      d += (uint64_t)a[6] * b[9]
   623         + (uint64_t)a[7] * b[8]
   624         + (uint64_t)a[8] * b[7]
   625         + (uint64_t)a[9] * b[6];
   626      VERIFY_BITS(d, 62);
   627      /* [d 0 0 0 0 0 t9 0 0 0 c t4 t3 t2 t1 t0] = [p15 p14 p13 p12 p11 p10 p9 0 0 0 p5 p4 p3 p2 p1 p0] */
   628      u5 = d & M; d >>= 26; c += u5 * R0;
   629      VERIFY_BITS(u5, 26);
   630      VERIFY_BITS(d, 36);
   631      /* VERIFY_BITS(c, 64); */
   632      /* [d u5 0 0 0 0 0 t9 0 0 0 c-u5*R0 t4 t3 t2 t1 t0] = [p15 p14 p13 p12 p11 p10 p9 0 0 0 p5 p4 p3 p2 p1 p0] */
   633      t5 = c & M; c >>= 26; c += u5 * R1;
   634      VERIFY_BITS(t5, 26);
   635      VERIFY_BITS(c, 39);
   636      /* [d u5 0 0 0 0 0 t9 0 0 c-u5*R1 t5-u5*R0 t4 t3 t2 t1 t0] = [p15 p14 p13 p12 p11 p10 p9 0 0 0 p5 p4 p3 p2 p1 p0] */
   637      /* [d 0 0 0 0 0 0 t9 0 0 c t5 t4 t3 t2 t1 t0] = [p15 p14 p13 p12 p11 p10 p9 0 0 0 p5 p4 p3 p2 p1 p0] */
   638  
   639      c += (uint64_t)a[0] * b[6]
   640         + (uint64_t)a[1] * b[5]
   641         + (uint64_t)a[2] * b[4]
   642         + (uint64_t)a[3] * b[3]
   643         + (uint64_t)a[4] * b[2]
   644         + (uint64_t)a[5] * b[1]
   645         + (uint64_t)a[6] * b[0];
   646      VERIFY_BITS(c, 63);
   647      /* [d 0 0 0 0 0 0 t9 0 0 c t5 t4 t3 t2 t1 t0] = [p15 p14 p13 p12 p11 p10 p9 0 0 p6 p5 p4 p3 p2 p1 p0] */
   648      d += (uint64_t)a[7] * b[9]
   649         + (uint64_t)a[8] * b[8]
   650         + (uint64_t)a[9] * b[7];
   651      VERIFY_BITS(d, 61);
   652      /* [d 0 0 0 0 0 0 t9 0 0 c t5 t4 t3 t2 t1 t0] = [p16 p15 p14 p13 p12 p11 p10 p9 0 0 p6 p5 p4 p3 p2 p1 p0] */
   653      u6 = d & M; d >>= 26; c += u6 * R0;
   654      VERIFY_BITS(u6, 26);
   655      VERIFY_BITS(d, 35);
   656      /* VERIFY_BITS(c, 64); */
   657      /* [d u6 0 0 0 0 0 0 t9 0 0 c-u6*R0 t5 t4 t3 t2 t1 t0] = [p16 p15 p14 p13 p12 p11 p10 p9 0 0 p6 p5 p4 p3 p2 p1 p0] */
   658      t6 = c & M; c >>= 26; c += u6 * R1;
   659      VERIFY_BITS(t6, 26);
   660      VERIFY_BITS(c, 39);
   661      /* [d u6 0 0 0 0 0 0 t9 0 c-u6*R1 t6-u6*R0 t5 t4 t3 t2 t1 t0] = [p16 p15 p14 p13 p12 p11 p10 p9 0 0 p6 p5 p4 p3 p2 p1 p0] */
   662      /* [d 0 0 0 0 0 0 0 t9 0 c t6 t5 t4 t3 t2 t1 t0] = [p16 p15 p14 p13 p12 p11 p10 p9 0 0 p6 p5 p4 p3 p2 p1 p0] */
   663  
   664      c += (uint64_t)a[0] * b[7]
   665         + (uint64_t)a[1] * b[6]
   666         + (uint64_t)a[2] * b[5]
   667         + (uint64_t)a[3] * b[4]
   668         + (uint64_t)a[4] * b[3]
   669         + (uint64_t)a[5] * b[2]
   670         + (uint64_t)a[6] * b[1]
   671         + (uint64_t)a[7] * b[0];
   672      /* VERIFY_BITS(c, 64); */
   673      VERIFY_CHECK(c <= 0x8000007C00000007ULL);
   674      /* [d 0 0 0 0 0 0 0 t9 0 c t6 t5 t4 t3 t2 t1 t0] = [p16 p15 p14 p13 p12 p11 p10 p9 0 p7 p6 p5 p4 p3 p2 p1 p0] */
   675      d += (uint64_t)a[8] * b[9]
   676         + (uint64_t)a[9] * b[8];
   677      VERIFY_BITS(d, 58);
   678      /* [d 0 0 0 0 0 0 0 t9 0 c t6 t5 t4 t3 t2 t1 t0] = [p17 p16 p15 p14 p13 p12 p11 p10 p9 0 p7 p6 p5 p4 p3 p2 p1 p0] */
   679      u7 = d & M; d >>= 26; c += u7 * R0;
   680      VERIFY_BITS(u7, 26);
   681      VERIFY_BITS(d, 32);
   682      /* VERIFY_BITS(c, 64); */
   683      VERIFY_CHECK(c <= 0x800001703FFFC2F7ULL);
   684      /* [d u7 0 0 0 0 0 0 0 t9 0 c-u7*R0 t6 t5 t4 t3 t2 t1 t0] = [p17 p16 p15 p14 p13 p12 p11 p10 p9 0 p7 p6 p5 p4 p3 p2 p1 p0] */
   685      t7 = c & M; c >>= 26; c += u7 * R1;
   686      VERIFY_BITS(t7, 26);
   687      VERIFY_BITS(c, 38);
   688      /* [d u7 0 0 0 0 0 0 0 t9 c-u7*R1 t7-u7*R0 t6 t5 t4 t3 t2 t1 t0] = [p17 p16 p15 p14 p13 p12 p11 p10 p9 0 p7 p6 p5 p4 p3 p2 p1 p0] */
   689      /* [d 0 0 0 0 0 0 0 0 t9 c t7 t6 t5 t4 t3 t2 t1 t0] = [p17 p16 p15 p14 p13 p12 p11 p10 p9 0 p7 p6 p5 p4 p3 p2 p1 p0] */
   690  
   691      c += (uint64_t)a[0] * b[8]
   692         + (uint64_t)a[1] * b[7]
   693         + (uint64_t)a[2] * b[6]
   694         + (uint64_t)a[3] * b[5]
   695         + (uint64_t)a[4] * b[4]
   696         + (uint64_t)a[5] * b[3]
   697         + (uint64_t)a[6] * b[2]
   698         + (uint64_t)a[7] * b[1]
   699         + (uint64_t)a[8] * b[0];
   700      /* VERIFY_BITS(c, 64); */
   701      VERIFY_CHECK(c <= 0x9000007B80000008ULL);
   702      /* [d 0 0 0 0 0 0 0 0 t9 c t7 t6 t5 t4 t3 t2 t1 t0] = [p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   703      d += (uint64_t)a[9] * b[9];
   704      VERIFY_BITS(d, 57);
   705      /* [d 0 0 0 0 0 0 0 0 t9 c t7 t6 t5 t4 t3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   706      u8 = d & M; d >>= 26; c += u8 * R0;
   707      VERIFY_BITS(u8, 26);
   708      VERIFY_BITS(d, 31);
   709      /* VERIFY_BITS(c, 64); */
   710      VERIFY_CHECK(c <= 0x9000016FBFFFC2F8ULL);
   711      /* [d u8 0 0 0 0 0 0 0 0 t9 c-u8*R0 t7 t6 t5 t4 t3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   712  
   713      r[3] = t3;
   714      VERIFY_BITS(r[3], 26);
   715      /* [d u8 0 0 0 0 0 0 0 0 t9 c-u8*R0 t7 t6 t5 t4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   716      r[4] = t4;
   717      VERIFY_BITS(r[4], 26);
   718      /* [d u8 0 0 0 0 0 0 0 0 t9 c-u8*R0 t7 t6 t5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   719      r[5] = t5;
   720      VERIFY_BITS(r[5], 26);
   721      /* [d u8 0 0 0 0 0 0 0 0 t9 c-u8*R0 t7 t6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   722      r[6] = t6;
   723      VERIFY_BITS(r[6], 26);
   724      /* [d u8 0 0 0 0 0 0 0 0 t9 c-u8*R0 t7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   725      r[7] = t7;
   726      VERIFY_BITS(r[7], 26);
   727      /* [d u8 0 0 0 0 0 0 0 0 t9 c-u8*R0 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   728  
   729      r[8] = c & M; c >>= 26; c += u8 * R1;
   730      VERIFY_BITS(r[8], 26);
   731      VERIFY_BITS(c, 39);
   732      /* [d u8 0 0 0 0 0 0 0 0 t9+c-u8*R1 r8-u8*R0 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   733      /* [d 0 0 0 0 0 0 0 0 0 t9+c r8 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   734      c   += d * R0 + t9;
   735      VERIFY_BITS(c, 45);
   736      /* [d 0 0 0 0 0 0 0 0 0 c-d*R0 r8 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   737      r[9] = c & (M >> 4); c >>= 22; c += d * (R1 << 4);
   738      VERIFY_BITS(r[9], 22);
   739      VERIFY_BITS(c, 46);
   740      /* [d 0 0 0 0 0 0 0 0 r9+((c-d*R1<<4)<<22)-d*R0 r8 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   741      /* [d 0 0 0 0 0 0 0 -d*R1 r9+(c<<22)-d*R0 r8 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   742      /* [r9+(c<<22) r8 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   743  
   744      d    = c * (R0 >> 4) + t0;
   745      VERIFY_BITS(d, 56);
   746      /* [r9+(c<<22) r8 r7 r6 r5 r4 r3 t2 t1 d-c*R0>>4] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   747      r[0] = d & M; d >>= 26;
   748      VERIFY_BITS(r[0], 26);
   749      VERIFY_BITS(d, 30);
   750      /* [r9+(c<<22) r8 r7 r6 r5 r4 r3 t2 t1+d r0-c*R0>>4] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   751      d   += c * (R1 >> 4) + t1;
   752      VERIFY_BITS(d, 53);
   753      VERIFY_CHECK(d <= 0x10000003FFFFBFULL);
   754      /* [r9+(c<<22) r8 r7 r6 r5 r4 r3 t2 d-c*R1>>4 r0-c*R0>>4] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   755      /* [r9 r8 r7 r6 r5 r4 r3 t2 d r0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   756      r[1] = d & M; d >>= 26;
   757      VERIFY_BITS(r[1], 26);
   758      VERIFY_BITS(d, 27);
   759      VERIFY_CHECK(d <= 0x4000000ULL);
   760      /* [r9 r8 r7 r6 r5 r4 r3 t2+d r1 r0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   761      d   += t2;
   762      VERIFY_BITS(d, 27);
   763      /* [r9 r8 r7 r6 r5 r4 r3 d r1 r0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   764      r[2] = d;
   765      VERIFY_BITS(r[2], 27);
   766      /* [r9 r8 r7 r6 r5 r4 r3 r2 r1 r0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   767  }
   768  
   769  SECP256K1_INLINE static void secp256k1_fe_sqr_inner(uint32_t *r, const uint32_t *a) {
   770      uint64_t c, d;
   771      uint64_t u0, u1, u2, u3, u4, u5, u6, u7, u8;
   772      uint32_t t9, t0, t1, t2, t3, t4, t5, t6, t7;
   773      const uint32_t M = 0x3FFFFFFUL, R0 = 0x3D10UL, R1 = 0x400UL;
   774  
   775      VERIFY_BITS(a[0], 30);
   776      VERIFY_BITS(a[1], 30);
   777      VERIFY_BITS(a[2], 30);
   778      VERIFY_BITS(a[3], 30);
   779      VERIFY_BITS(a[4], 30);
   780      VERIFY_BITS(a[5], 30);
   781      VERIFY_BITS(a[6], 30);
   782      VERIFY_BITS(a[7], 30);
   783      VERIFY_BITS(a[8], 30);
   784      VERIFY_BITS(a[9], 26);
   785  
   786      /** [... a b c] is a shorthand for ... + a<<52 + b<<26 + c<<0 mod n.
   787       *  px is a shorthand for sum(a[i]*a[x-i], i=0..x).
   788       *  Note that [x 0 0 0 0 0 0 0 0 0 0] = [x*R1 x*R0].
   789       */
   790  
   791      d  = (uint64_t)(a[0]*2) * a[9]
   792         + (uint64_t)(a[1]*2) * a[8]
   793         + (uint64_t)(a[2]*2) * a[7]
   794         + (uint64_t)(a[3]*2) * a[6]
   795         + (uint64_t)(a[4]*2) * a[5];
   796      /* VERIFY_BITS(d, 64); */
   797      /* [d 0 0 0 0 0 0 0 0 0] = [p9 0 0 0 0 0 0 0 0 0] */
   798      t9 = d & M; d >>= 26;
   799      VERIFY_BITS(t9, 26);
   800      VERIFY_BITS(d, 38);
   801      /* [d t9 0 0 0 0 0 0 0 0 0] = [p9 0 0 0 0 0 0 0 0 0] */
   802  
   803      c  = (uint64_t)a[0] * a[0];
   804      VERIFY_BITS(c, 60);
   805      /* [d t9 0 0 0 0 0 0 0 0 c] = [p9 0 0 0 0 0 0 0 0 p0] */
   806      d += (uint64_t)(a[1]*2) * a[9]
   807         + (uint64_t)(a[2]*2) * a[8]
   808         + (uint64_t)(a[3]*2) * a[7]
   809         + (uint64_t)(a[4]*2) * a[6]
   810         + (uint64_t)a[5] * a[5];
   811      VERIFY_BITS(d, 63);
   812      /* [d t9 0 0 0 0 0 0 0 0 c] = [p10 p9 0 0 0 0 0 0 0 0 p0] */
   813      u0 = d & M; d >>= 26; c += u0 * R0;
   814      VERIFY_BITS(u0, 26);
   815      VERIFY_BITS(d, 37);
   816      VERIFY_BITS(c, 61);
   817      /* [d u0 t9 0 0 0 0 0 0 0 0 c-u0*R0] = [p10 p9 0 0 0 0 0 0 0 0 p0] */
   818      t0 = c & M; c >>= 26; c += u0 * R1;
   819      VERIFY_BITS(t0, 26);
   820      VERIFY_BITS(c, 37);
   821      /* [d u0 t9 0 0 0 0 0 0 0 c-u0*R1 t0-u0*R0] = [p10 p9 0 0 0 0 0 0 0 0 p0] */
   822      /* [d 0 t9 0 0 0 0 0 0 0 c t0] = [p10 p9 0 0 0 0 0 0 0 0 p0] */
   823  
   824      c += (uint64_t)(a[0]*2) * a[1];
   825      VERIFY_BITS(c, 62);
   826      /* [d 0 t9 0 0 0 0 0 0 0 c t0] = [p10 p9 0 0 0 0 0 0 0 p1 p0] */
   827      d += (uint64_t)(a[2]*2) * a[9]
   828         + (uint64_t)(a[3]*2) * a[8]
   829         + (uint64_t)(a[4]*2) * a[7]
   830         + (uint64_t)(a[5]*2) * a[6];
   831      VERIFY_BITS(d, 63);
   832      /* [d 0 t9 0 0 0 0 0 0 0 c t0] = [p11 p10 p9 0 0 0 0 0 0 0 p1 p0] */
   833      u1 = d & M; d >>= 26; c += u1 * R0;
   834      VERIFY_BITS(u1, 26);
   835      VERIFY_BITS(d, 37);
   836      VERIFY_BITS(c, 63);
   837      /* [d u1 0 t9 0 0 0 0 0 0 0 c-u1*R0 t0] = [p11 p10 p9 0 0 0 0 0 0 0 p1 p0] */
   838      t1 = c & M; c >>= 26; c += u1 * R1;
   839      VERIFY_BITS(t1, 26);
   840      VERIFY_BITS(c, 38);
   841      /* [d u1 0 t9 0 0 0 0 0 0 c-u1*R1 t1-u1*R0 t0] = [p11 p10 p9 0 0 0 0 0 0 0 p1 p0] */
   842      /* [d 0 0 t9 0 0 0 0 0 0 c t1 t0] = [p11 p10 p9 0 0 0 0 0 0 0 p1 p0] */
   843  
   844      c += (uint64_t)(a[0]*2) * a[2]
   845         + (uint64_t)a[1] * a[1];
   846      VERIFY_BITS(c, 62);
   847      /* [d 0 0 t9 0 0 0 0 0 0 c t1 t0] = [p11 p10 p9 0 0 0 0 0 0 p2 p1 p0] */
   848      d += (uint64_t)(a[3]*2) * a[9]
   849         + (uint64_t)(a[4]*2) * a[8]
   850         + (uint64_t)(a[5]*2) * a[7]
   851         + (uint64_t)a[6] * a[6];
   852      VERIFY_BITS(d, 63);
   853      /* [d 0 0 t9 0 0 0 0 0 0 c t1 t0] = [p12 p11 p10 p9 0 0 0 0 0 0 p2 p1 p0] */
   854      u2 = d & M; d >>= 26; c += u2 * R0;
   855      VERIFY_BITS(u2, 26);
   856      VERIFY_BITS(d, 37);
   857      VERIFY_BITS(c, 63);
   858      /* [d u2 0 0 t9 0 0 0 0 0 0 c-u2*R0 t1 t0] = [p12 p11 p10 p9 0 0 0 0 0 0 p2 p1 p0] */
   859      t2 = c & M; c >>= 26; c += u2 * R1;
   860      VERIFY_BITS(t2, 26);
   861      VERIFY_BITS(c, 38);
   862      /* [d u2 0 0 t9 0 0 0 0 0 c-u2*R1 t2-u2*R0 t1 t0] = [p12 p11 p10 p9 0 0 0 0 0 0 p2 p1 p0] */
   863      /* [d 0 0 0 t9 0 0 0 0 0 c t2 t1 t0] = [p12 p11 p10 p9 0 0 0 0 0 0 p2 p1 p0] */
   864  
   865      c += (uint64_t)(a[0]*2) * a[3]
   866         + (uint64_t)(a[1]*2) * a[2];
   867      VERIFY_BITS(c, 63);
   868      /* [d 0 0 0 t9 0 0 0 0 0 c t2 t1 t0] = [p12 p11 p10 p9 0 0 0 0 0 p3 p2 p1 p0] */
   869      d += (uint64_t)(a[4]*2) * a[9]
   870         + (uint64_t)(a[5]*2) * a[8]
   871         + (uint64_t)(a[6]*2) * a[7];
   872      VERIFY_BITS(d, 63);
   873      /* [d 0 0 0 t9 0 0 0 0 0 c t2 t1 t0] = [p13 p12 p11 p10 p9 0 0 0 0 0 p3 p2 p1 p0] */
   874      u3 = d & M; d >>= 26; c += u3 * R0;
   875      VERIFY_BITS(u3, 26);
   876      VERIFY_BITS(d, 37);
   877      /* VERIFY_BITS(c, 64); */
   878      /* [d u3 0 0 0 t9 0 0 0 0 0 c-u3*R0 t2 t1 t0] = [p13 p12 p11 p10 p9 0 0 0 0 0 p3 p2 p1 p0] */
   879      t3 = c & M; c >>= 26; c += u3 * R1;
   880      VERIFY_BITS(t3, 26);
   881      VERIFY_BITS(c, 39);
   882      /* [d u3 0 0 0 t9 0 0 0 0 c-u3*R1 t3-u3*R0 t2 t1 t0] = [p13 p12 p11 p10 p9 0 0 0 0 0 p3 p2 p1 p0] */
   883      /* [d 0 0 0 0 t9 0 0 0 0 c t3 t2 t1 t0] = [p13 p12 p11 p10 p9 0 0 0 0 0 p3 p2 p1 p0] */
   884  
   885      c += (uint64_t)(a[0]*2) * a[4]
   886         + (uint64_t)(a[1]*2) * a[3]
   887         + (uint64_t)a[2] * a[2];
   888      VERIFY_BITS(c, 63);
   889      /* [d 0 0 0 0 t9 0 0 0 0 c t3 t2 t1 t0] = [p13 p12 p11 p10 p9 0 0 0 0 p4 p3 p2 p1 p0] */
   890      d += (uint64_t)(a[5]*2) * a[9]
   891         + (uint64_t)(a[6]*2) * a[8]
   892         + (uint64_t)a[7] * a[7];
   893      VERIFY_BITS(d, 62);
   894      /* [d 0 0 0 0 t9 0 0 0 0 c t3 t2 t1 t0] = [p14 p13 p12 p11 p10 p9 0 0 0 0 p4 p3 p2 p1 p0] */
   895      u4 = d & M; d >>= 26; c += u4 * R0;
   896      VERIFY_BITS(u4, 26);
   897      VERIFY_BITS(d, 36);
   898      /* VERIFY_BITS(c, 64); */
   899      /* [d u4 0 0 0 0 t9 0 0 0 0 c-u4*R0 t3 t2 t1 t0] = [p14 p13 p12 p11 p10 p9 0 0 0 0 p4 p3 p2 p1 p0] */
   900      t4 = c & M; c >>= 26; c += u4 * R1;
   901      VERIFY_BITS(t4, 26);
   902      VERIFY_BITS(c, 39);
   903      /* [d u4 0 0 0 0 t9 0 0 0 c-u4*R1 t4-u4*R0 t3 t2 t1 t0] = [p14 p13 p12 p11 p10 p9 0 0 0 0 p4 p3 p2 p1 p0] */
   904      /* [d 0 0 0 0 0 t9 0 0 0 c t4 t3 t2 t1 t0] = [p14 p13 p12 p11 p10 p9 0 0 0 0 p4 p3 p2 p1 p0] */
   905  
   906      c += (uint64_t)(a[0]*2) * a[5]
   907         + (uint64_t)(a[1]*2) * a[4]
   908         + (uint64_t)(a[2]*2) * a[3];
   909      VERIFY_BITS(c, 63);
   910      /* [d 0 0 0 0 0 t9 0 0 0 c t4 t3 t2 t1 t0] = [p14 p13 p12 p11 p10 p9 0 0 0 p5 p4 p3 p2 p1 p0] */
   911      d += (uint64_t)(a[6]*2) * a[9]
   912         + (uint64_t)(a[7]*2) * a[8];
   913      VERIFY_BITS(d, 62);
   914      /* [d 0 0 0 0 0 t9 0 0 0 c t4 t3 t2 t1 t0] = [p15 p14 p13 p12 p11 p10 p9 0 0 0 p5 p4 p3 p2 p1 p0] */
   915      u5 = d & M; d >>= 26; c += u5 * R0;
   916      VERIFY_BITS(u5, 26);
   917      VERIFY_BITS(d, 36);
   918      /* VERIFY_BITS(c, 64); */
   919      /* [d u5 0 0 0 0 0 t9 0 0 0 c-u5*R0 t4 t3 t2 t1 t0] = [p15 p14 p13 p12 p11 p10 p9 0 0 0 p5 p4 p3 p2 p1 p0] */
   920      t5 = c & M; c >>= 26; c += u5 * R1;
   921      VERIFY_BITS(t5, 26);
   922      VERIFY_BITS(c, 39);
   923      /* [d u5 0 0 0 0 0 t9 0 0 c-u5*R1 t5-u5*R0 t4 t3 t2 t1 t0] = [p15 p14 p13 p12 p11 p10 p9 0 0 0 p5 p4 p3 p2 p1 p0] */
   924      /* [d 0 0 0 0 0 0 t9 0 0 c t5 t4 t3 t2 t1 t0] = [p15 p14 p13 p12 p11 p10 p9 0 0 0 p5 p4 p3 p2 p1 p0] */
   925  
   926      c += (uint64_t)(a[0]*2) * a[6]
   927         + (uint64_t)(a[1]*2) * a[5]
   928         + (uint64_t)(a[2]*2) * a[4]
   929         + (uint64_t)a[3] * a[3];
   930      VERIFY_BITS(c, 63);
   931      /* [d 0 0 0 0 0 0 t9 0 0 c t5 t4 t3 t2 t1 t0] = [p15 p14 p13 p12 p11 p10 p9 0 0 p6 p5 p4 p3 p2 p1 p0] */
   932      d += (uint64_t)(a[7]*2) * a[9]
   933         + (uint64_t)a[8] * a[8];
   934      VERIFY_BITS(d, 61);
   935      /* [d 0 0 0 0 0 0 t9 0 0 c t5 t4 t3 t2 t1 t0] = [p16 p15 p14 p13 p12 p11 p10 p9 0 0 p6 p5 p4 p3 p2 p1 p0] */
   936      u6 = d & M; d >>= 26; c += u6 * R0;
   937      VERIFY_BITS(u6, 26);
   938      VERIFY_BITS(d, 35);
   939      /* VERIFY_BITS(c, 64); */
   940      /* [d u6 0 0 0 0 0 0 t9 0 0 c-u6*R0 t5 t4 t3 t2 t1 t0] = [p16 p15 p14 p13 p12 p11 p10 p9 0 0 p6 p5 p4 p3 p2 p1 p0] */
   941      t6 = c & M; c >>= 26; c += u6 * R1;
   942      VERIFY_BITS(t6, 26);
   943      VERIFY_BITS(c, 39);
   944      /* [d u6 0 0 0 0 0 0 t9 0 c-u6*R1 t6-u6*R0 t5 t4 t3 t2 t1 t0] = [p16 p15 p14 p13 p12 p11 p10 p9 0 0 p6 p5 p4 p3 p2 p1 p0] */
   945      /* [d 0 0 0 0 0 0 0 t9 0 c t6 t5 t4 t3 t2 t1 t0] = [p16 p15 p14 p13 p12 p11 p10 p9 0 0 p6 p5 p4 p3 p2 p1 p0] */
   946  
   947      c += (uint64_t)(a[0]*2) * a[7]
   948         + (uint64_t)(a[1]*2) * a[6]
   949         + (uint64_t)(a[2]*2) * a[5]
   950         + (uint64_t)(a[3]*2) * a[4];
   951      /* VERIFY_BITS(c, 64); */
   952      VERIFY_CHECK(c <= 0x8000007C00000007ULL);
   953      /* [d 0 0 0 0 0 0 0 t9 0 c t6 t5 t4 t3 t2 t1 t0] = [p16 p15 p14 p13 p12 p11 p10 p9 0 p7 p6 p5 p4 p3 p2 p1 p0] */
   954      d += (uint64_t)(a[8]*2) * a[9];
   955      VERIFY_BITS(d, 58);
   956      /* [d 0 0 0 0 0 0 0 t9 0 c t6 t5 t4 t3 t2 t1 t0] = [p17 p16 p15 p14 p13 p12 p11 p10 p9 0 p7 p6 p5 p4 p3 p2 p1 p0] */
   957      u7 = d & M; d >>= 26; c += u7 * R0;
   958      VERIFY_BITS(u7, 26);
   959      VERIFY_BITS(d, 32);
   960      /* VERIFY_BITS(c, 64); */
   961      VERIFY_CHECK(c <= 0x800001703FFFC2F7ULL);
   962      /* [d u7 0 0 0 0 0 0 0 t9 0 c-u7*R0 t6 t5 t4 t3 t2 t1 t0] = [p17 p16 p15 p14 p13 p12 p11 p10 p9 0 p7 p6 p5 p4 p3 p2 p1 p0] */
   963      t7 = c & M; c >>= 26; c += u7 * R1;
   964      VERIFY_BITS(t7, 26);
   965      VERIFY_BITS(c, 38);
   966      /* [d u7 0 0 0 0 0 0 0 t9 c-u7*R1 t7-u7*R0 t6 t5 t4 t3 t2 t1 t0] = [p17 p16 p15 p14 p13 p12 p11 p10 p9 0 p7 p6 p5 p4 p3 p2 p1 p0] */
   967      /* [d 0 0 0 0 0 0 0 0 t9 c t7 t6 t5 t4 t3 t2 t1 t0] = [p17 p16 p15 p14 p13 p12 p11 p10 p9 0 p7 p6 p5 p4 p3 p2 p1 p0] */
   968  
   969      c += (uint64_t)(a[0]*2) * a[8]
   970         + (uint64_t)(a[1]*2) * a[7]
   971         + (uint64_t)(a[2]*2) * a[6]
   972         + (uint64_t)(a[3]*2) * a[5]
   973         + (uint64_t)a[4] * a[4];
   974      /* VERIFY_BITS(c, 64); */
   975      VERIFY_CHECK(c <= 0x9000007B80000008ULL);
   976      /* [d 0 0 0 0 0 0 0 0 t9 c t7 t6 t5 t4 t3 t2 t1 t0] = [p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   977      d += (uint64_t)a[9] * a[9];
   978      VERIFY_BITS(d, 57);
   979      /* [d 0 0 0 0 0 0 0 0 t9 c t7 t6 t5 t4 t3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   980      u8 = d & M; d >>= 26; c += u8 * R0;
   981      VERIFY_BITS(u8, 26);
   982      VERIFY_BITS(d, 31);
   983      /* VERIFY_BITS(c, 64); */
   984      VERIFY_CHECK(c <= 0x9000016FBFFFC2F8ULL);
   985      /* [d u8 0 0 0 0 0 0 0 0 t9 c-u8*R0 t7 t6 t5 t4 t3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   986  
   987      r[3] = t3;
   988      VERIFY_BITS(r[3], 26);
   989      /* [d u8 0 0 0 0 0 0 0 0 t9 c-u8*R0 t7 t6 t5 t4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   990      r[4] = t4;
   991      VERIFY_BITS(r[4], 26);
   992      /* [d u8 0 0 0 0 0 0 0 0 t9 c-u8*R0 t7 t6 t5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   993      r[5] = t5;
   994      VERIFY_BITS(r[5], 26);
   995      /* [d u8 0 0 0 0 0 0 0 0 t9 c-u8*R0 t7 t6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   996      r[6] = t6;
   997      VERIFY_BITS(r[6], 26);
   998      /* [d u8 0 0 0 0 0 0 0 0 t9 c-u8*R0 t7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
   999      r[7] = t7;
  1000      VERIFY_BITS(r[7], 26);
  1001      /* [d u8 0 0 0 0 0 0 0 0 t9 c-u8*R0 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1002  
  1003      r[8] = c & M; c >>= 26; c += u8 * R1;
  1004      VERIFY_BITS(r[8], 26);
  1005      VERIFY_BITS(c, 39);
  1006      /* [d u8 0 0 0 0 0 0 0 0 t9+c-u8*R1 r8-u8*R0 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1007      /* [d 0 0 0 0 0 0 0 0 0 t9+c r8 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1008      c   += d * R0 + t9;
  1009      VERIFY_BITS(c, 45);
  1010      /* [d 0 0 0 0 0 0 0 0 0 c-d*R0 r8 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1011      r[9] = c & (M >> 4); c >>= 22; c += d * (R1 << 4);
  1012      VERIFY_BITS(r[9], 22);
  1013      VERIFY_BITS(c, 46);
  1014      /* [d 0 0 0 0 0 0 0 0 r9+((c-d*R1<<4)<<22)-d*R0 r8 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1015      /* [d 0 0 0 0 0 0 0 -d*R1 r9+(c<<22)-d*R0 r8 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1016      /* [r9+(c<<22) r8 r7 r6 r5 r4 r3 t2 t1 t0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1017  
  1018      d    = c * (R0 >> 4) + t0;
  1019      VERIFY_BITS(d, 56);
  1020      /* [r9+(c<<22) r8 r7 r6 r5 r4 r3 t2 t1 d-c*R0>>4] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1021      r[0] = d & M; d >>= 26;
  1022      VERIFY_BITS(r[0], 26);
  1023      VERIFY_BITS(d, 30);
  1024      /* [r9+(c<<22) r8 r7 r6 r5 r4 r3 t2 t1+d r0-c*R0>>4] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1025      d   += c * (R1 >> 4) + t1;
  1026      VERIFY_BITS(d, 53);
  1027      VERIFY_CHECK(d <= 0x10000003FFFFBFULL);
  1028      /* [r9+(c<<22) r8 r7 r6 r5 r4 r3 t2 d-c*R1>>4 r0-c*R0>>4] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1029      /* [r9 r8 r7 r6 r5 r4 r3 t2 d r0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1030      r[1] = d & M; d >>= 26;
  1031      VERIFY_BITS(r[1], 26);
  1032      VERIFY_BITS(d, 27);
  1033      VERIFY_CHECK(d <= 0x4000000ULL);
  1034      /* [r9 r8 r7 r6 r5 r4 r3 t2+d r1 r0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1035      d   += t2;
  1036      VERIFY_BITS(d, 27);
  1037      /* [r9 r8 r7 r6 r5 r4 r3 d r1 r0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1038      r[2] = d;
  1039      VERIFY_BITS(r[2], 27);
  1040      /* [r9 r8 r7 r6 r5 r4 r3 r2 r1 r0] = [p18 p17 p16 p15 p14 p13 p12 p11 p10 p9 p8 p7 p6 p5 p4 p3 p2 p1 p0] */
  1041  }
  1042  #endif
  1043  
  1044  static void secp256k1_fe_mul(secp256k1_fe *r, const secp256k1_fe *a, const secp256k1_fe * SECP256K1_RESTRICT b) {
  1045  #ifdef VERIFY
  1046      VERIFY_CHECK(a->magnitude <= 8);
  1047      VERIFY_CHECK(b->magnitude <= 8);
  1048      secp256k1_fe_verify(a);
  1049      secp256k1_fe_verify(b);
  1050      VERIFY_CHECK(r != b);
  1051  #endif
  1052      secp256k1_fe_mul_inner(r->n, a->n, b->n);
  1053  #ifdef VERIFY
  1054      r->magnitude = 1;
  1055      r->normalized = 0;
  1056      secp256k1_fe_verify(r);
  1057  #endif
  1058  }
  1059  
  1060  static void secp256k1_fe_sqr(secp256k1_fe *r, const secp256k1_fe *a) {
  1061  #ifdef VERIFY
  1062      VERIFY_CHECK(a->magnitude <= 8);
  1063      secp256k1_fe_verify(a);
  1064  #endif
  1065      secp256k1_fe_sqr_inner(r->n, a->n);
  1066  #ifdef VERIFY
  1067      r->magnitude = 1;
  1068      r->normalized = 0;
  1069      secp256k1_fe_verify(r);
  1070  #endif
  1071  }
  1072  
  1073  static SECP256K1_INLINE void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag) {
  1074      uint32_t mask0, mask1;
  1075      mask0 = flag + ~((uint32_t)0);
  1076      mask1 = ~mask0;
  1077      r->n[0] = (r->n[0] & mask0) | (a->n[0] & mask1);
  1078      r->n[1] = (r->n[1] & mask0) | (a->n[1] & mask1);
  1079      r->n[2] = (r->n[2] & mask0) | (a->n[2] & mask1);
  1080      r->n[3] = (r->n[3] & mask0) | (a->n[3] & mask1);
  1081      r->n[4] = (r->n[4] & mask0) | (a->n[4] & mask1);
  1082      r->n[5] = (r->n[5] & mask0) | (a->n[5] & mask1);
  1083      r->n[6] = (r->n[6] & mask0) | (a->n[6] & mask1);
  1084      r->n[7] = (r->n[7] & mask0) | (a->n[7] & mask1);
  1085      r->n[8] = (r->n[8] & mask0) | (a->n[8] & mask1);
  1086      r->n[9] = (r->n[9] & mask0) | (a->n[9] & mask1);
  1087  #ifdef VERIFY
  1088      if (a->magnitude > r->magnitude) {
  1089          r->magnitude = a->magnitude;
  1090      }
  1091      r->normalized &= a->normalized;
  1092  #endif
  1093  }
  1094  
  1095  static SECP256K1_INLINE void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r, const secp256k1_fe_storage *a, int flag) {
  1096      uint32_t mask0, mask1;
  1097      mask0 = flag + ~((uint32_t)0);
  1098      mask1 = ~mask0;
  1099      r->n[0] = (r->n[0] & mask0) | (a->n[0] & mask1);
  1100      r->n[1] = (r->n[1] & mask0) | (a->n[1] & mask1);
  1101      r->n[2] = (r->n[2] & mask0) | (a->n[2] & mask1);
  1102      r->n[3] = (r->n[3] & mask0) | (a->n[3] & mask1);
  1103      r->n[4] = (r->n[4] & mask0) | (a->n[4] & mask1);
  1104      r->n[5] = (r->n[5] & mask0) | (a->n[5] & mask1);
  1105      r->n[6] = (r->n[6] & mask0) | (a->n[6] & mask1);
  1106      r->n[7] = (r->n[7] & mask0) | (a->n[7] & mask1);
  1107  }
  1108  
  1109  static void secp256k1_fe_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a) {
  1110  #ifdef VERIFY
  1111      VERIFY_CHECK(a->normalized);
  1112  #endif
  1113      r->n[0] = a->n[0] | a->n[1] << 26;
  1114      r->n[1] = a->n[1] >> 6 | a->n[2] << 20;
  1115      r->n[2] = a->n[2] >> 12 | a->n[3] << 14;
  1116      r->n[3] = a->n[3] >> 18 | a->n[4] << 8;
  1117      r->n[4] = a->n[4] >> 24 | a->n[5] << 2 | a->n[6] << 28;
  1118      r->n[5] = a->n[6] >> 4 | a->n[7] << 22;
  1119      r->n[6] = a->n[7] >> 10 | a->n[8] << 16;
  1120      r->n[7] = a->n[8] >> 16 | a->n[9] << 10;
  1121  }
  1122  
  1123  static SECP256K1_INLINE void secp256k1_fe_from_storage(secp256k1_fe *r, const secp256k1_fe_storage *a) {
  1124      r->n[0] = a->n[0] & 0x3FFFFFFUL;
  1125      r->n[1] = a->n[0] >> 26 | ((a->n[1] << 6) & 0x3FFFFFFUL);
  1126      r->n[2] = a->n[1] >> 20 | ((a->n[2] << 12) & 0x3FFFFFFUL);
  1127      r->n[3] = a->n[2] >> 14 | ((a->n[3] << 18) & 0x3FFFFFFUL);
  1128      r->n[4] = a->n[3] >> 8 | ((a->n[4] << 24) & 0x3FFFFFFUL);
  1129      r->n[5] = (a->n[4] >> 2) & 0x3FFFFFFUL;
  1130      r->n[6] = a->n[4] >> 28 | ((a->n[5] << 4) & 0x3FFFFFFUL);
  1131      r->n[7] = a->n[5] >> 22 | ((a->n[6] << 10) & 0x3FFFFFFUL);
  1132      r->n[8] = a->n[6] >> 16 | ((a->n[7] << 16) & 0x3FFFFFFUL);
  1133      r->n[9] = a->n[7] >> 10;
  1134  #ifdef VERIFY
  1135      r->magnitude = 1;
  1136      r->normalized = 1;
  1137  #endif
  1138  }
  1139  
  1140  #endif