nettle-bugs

nettle-bugs@lists.lysator.liu.se

2196 discussions

Fwd: [PowerPC] GCM optimization
by Maamoun TK 01 Dec '20

01 Dec '20

---------- Forwarded message --------- From: Maamoun TK <maamoun.tk(a)googlemail.com> Date: Thu, Nov 12, 2020 at 7:42 PM Subject: Re: [PowerPC] GCM optimization To: Niels Möller <nisse(a)lysator.liu.se> On Thu, Nov 12, 2020 at 6:40 PM Niels Möller <nisse(a)lysator.liu.se> wrote: > I gave it a test run on gcc112 in the gcc compile farm, and speedup of > gcm update seems to be 26 times(!) compared to the C version. > That's reasonable, I got similar speedup on more stable POWER instances than gcc compile farm. > Where would that documentation be published? In the Nettle manual, as > some IBM white paper, or as a more-or-less academic paper, e.g., on > arxiv? I will not be able to spend much time on writing, but I'd be > happy to review. > I'll start writing the papers once I got more details from IBM, similar to intel documents, the document will be academic and practical at the same time, I'll dive into finite field equations to demonstrate how we get there as well as I'll add a practical example to clarify the preference of this method in addition to the expected speedup of this method. My intention that other crypto libraries could take advantage of this document or maybe be a starting point for further improvements to the algorithm so I'm checking if IBM would publish or approve such a document the same as intel. > I have a sketch of ARM Neon code doing the equivalent of two vpmsumd, > with reasonable parallelism. Quite a lot of instructions needed. > If you don't have much time, you can send it here and I'll continue from that point. I'm planning to compare the new method with the usual method with and without the karatsuba algorithm. > +C Alignment of gcm_key table elements, which is declared in gcm.h > > +define(`TableElemAlign', `0x100') > > I still find this large constant puzzling. If I try > > struct gcm_key key; > printf("sizeof (key): %zd, sizeof(key.h[0]): %zd\n", sizeof(key), > sizeof(key.h[0])); > > (I added it to the start of test_main in gcm-test.c) and run on the > gcc112 machine, I get > > sizeof (key): 4096, sizeof(key.h[0]): 16 > > Which is what I'd expect, with elements of size 16 bytes, not 256 bytes. > > I haven't yet had the time to read the code carefully. > You see, the alignment of each element is 0x100 (256). The table has 16 elements and you got the size of the table 4096 which is reasonable because 16*256=4096 regards, Mamone

3 5

[PowerPC] GCM optimization
by Maamoun TK 28 Nov '20

28 Nov '20

This implementation takes advantage of research made by Niels Möller to optimize GCM on PowerPC, this optimization yields a +27.7% performance boost on POWER8 over the previous implementation that was based on intel documents. The performance comparison is made by processing 4 blocks per loop without any further optimizations. I made some documentations between the lines but I suggest writing a document similar to the intel ones that go into more details and clarify the preference of this method. I'm also curious if this method can also make a difference in other architectures like ARM, I'm planning to try it out for ARM to figure that out. --- configure.ac | 6 +- gcm.c | 49 +++-- powerpc64/p8/gcm-hash.asm | 502 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 542 insertions(+), 15 deletions(-) create mode 100644 powerpc64/p8/gcm-hash.asm diff --git a/configure.ac b/configure.ac index 2a47f940..20f7cf74 100644 --- a/configure.ac +++ b/configure.ac @@ -497,7 +497,7 @@ asm_replace_list="aes-encrypt-internal.asm aes-decrypt-internal.asm \ sha3-permute.asm umac-nh.asm umac-nh-n.asm machine.m4" # Assembler files which generate additional object files if they are used. -asm_nettle_optional_list="gcm-hash8.asm cpuid.asm \ +asm_nettle_optional_list="gcm-hash.asm gcm-hash8.asm cpuid.asm \ aes-encrypt-internal-2.asm aes-decrypt-internal-2.asm memxor-2.asm \ chacha-3core.asm chacha-core-internal-2.asm salsa20-2core.asm \ salsa20-core-internal-2.asm sha1-compress-2.asm sha256-compress-2.asm \ @@ -621,9 +621,9 @@ AH_VERBATIM([HAVE_NATIVE], #undef HAVE_NATIVE_ecc_secp384r1_redc #undef HAVE_NATIVE_ecc_secp521r1_modp #undef HAVE_NATIVE_ecc_secp521r1_redc -#undef HAVE_NATIVE_gcm_init_key8 +#undef HAVE_NATIVE_gcm_init_key +#undef HAVE_NATIVE_gcm_hash #undef HAVE_NATIVE_gcm_hash8 -#undef HAVE_NATIVE_gcm_fill #undef HAVE_NATIVE_salsa20_core #undef HAVE_NATIVE_salsa20_2core #undef HAVE_NATIVE_fat_salsa20_2core diff --git a/gcm.c b/gcm.c index 48b3e75a..81981c1c 100644 --- a/gcm.c +++ b/gcm.c @@ -140,6 +140,19 @@ gcm_gf_mul (union nettle_block16 *x, const union nettle_block16 *table) memcpy (x->b, Z.b, sizeof(Z)); } # elif GCM_TABLE_BITS == 8 +# if HAVE_NATIVE_gcm_init_key + +#define gcm_init_key _nettle_gcm_init_key +void +_nettle_gcm_init_key (union nettle_block16 *table); +# endif /* HAVE_NATIVE_gcm_init_key */ +# if HAVE_NATIVE_gcm_hash + +#define gcm_hash _nettle_gcm_hash +void +_nettle_gcm_hash (const struct gcm_key *key, union nettle_block16 *x, + size_t length, const uint8_t *data); +# endif /* HAVE_NATIVE_gcm_hash */ # if HAVE_NATIVE_gcm_hash8 #define gcm_hash _nettle_gcm_hash8 @@ -228,6 +241,29 @@ gcm_gf_mul (union nettle_block16 *x, const union nettle_block16 *table) /* Increment the rightmost 32 bits. */ #define INC32(block) INCREMENT(4, (block.b) + GCM_BLOCK_SIZE - 4) +#ifndef gcm_init_key +static void +gcm_init_key(union nettle_block16 *table) +{ +#if GCM_TABLE_BITS + /* Middle element if GCM_TABLE_BITS > 0, otherwise the first + element */ + unsigned i = (1<<GCM_TABLE_BITS)/2; + + /* Algorithm 3 from the gcm paper. First do powers of two, then do + the rest by adding. */ + while (i /= 2) + block16_mulx_ghash(&table[i], &table[2*i]); + for (i = 2; i < 1<<GCM_TABLE_BITS; i *= 2) + { + unsigned j; + for (j = 1; j < i; j++) + block16_xor3(&table[i+j], &table[i], &table[j]); + } +#endif +} +#endif /* !gcm_init_key */ + /* Initialization of GCM. * @ctx: The context of GCM * @cipher: The context of the underlying block cipher @@ -245,18 +281,7 @@ gcm_set_key(struct gcm_key *key, memset(key->h[0].b, 0, GCM_BLOCK_SIZE); f (cipher, GCM_BLOCK_SIZE, key->h[i].b, key->h[0].b); -#if GCM_TABLE_BITS - /* Algorithm 3 from the gcm paper. First do powers of two, then do - the rest by adding. */ - while (i /= 2) - block16_mulx_ghash(&key->h[i], &key->h[2*i]); - for (i = 2; i < 1<<GCM_TABLE_BITS; i *= 2) - { - unsigned j; - for (j = 1; j < i; j++) - block16_xor3(&key->h[i+j], &key->h[i],&key->h[j]); - } -#endif + gcm_init_key(key->h); } #ifndef gcm_hash diff --git a/powerpc64/p8/gcm-hash.asm b/powerpc64/p8/gcm-hash.asm new file mode 100644 index 00000000..e79fbdc2 --- /dev/null +++ b/powerpc64/p8/gcm-hash.asm @@ -0,0 +1,502 @@ +C powerpc64/p8/gcm-hash.asm + +ifelse(` + Copyright (C) 2020 Niels Möller and Mamone Tarsha + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +') + +C Alignment of gcm_key table elements, which is declared in gcm.h +define(`TableElemAlign', `0x100') + +C Register usage: + +define(`SP', `r1') +define(`TOCP', `r2') + +define(`TABLE', `r3') + +define(`ZERO', `v0') +define(`B1', `v1') +define(`EMSB', `v16') +define(`POLY', `v17') +define(`POLY_L', `v1') + +define(`H', `v2') +define(`H2', `v3') +define(`H3', `v4') +define(`H4', `v5') +define(`H1M', `v6') +define(`H1L', `v7') +define(`H2M', `v8') +define(`H2L', `v9') +define(`Hl', `v10') +define(`Hm', `v11') +define(`Hp', `v12') +define(`Hl2', `v13') +define(`Hm2', `v14') +define(`Hp2', `v15') +define(`R', `v13') +define(`F', `v14') +define(`T', `v15') +define(`R2', `v16') +define(`F2', `v17') +define(`T2', `v18') + +define(`LE_TEMP', `v18') +define(`LE_MASK', `v19') + +.file "gcm-hash.asm" + +.text + + C void gcm_init_key (union gcm_block *table) + +C This function populates the gcm table as the following layout +C ******************************************************************************* +C | H1M = (H1 div x⁶⁴)||((H1 mod x⁶⁴) × (x⁶⁴+x⁶³+x⁶²+x⁵⁷)) div x⁶⁴ | +C | H1L = (H1 mod x⁶⁴)||(((H1 mod x⁶⁴) × (x⁶³+x⁶²+x⁵⁷)) mod x⁶⁴) + (H1 div x⁶⁴) | +C | | +C | H2M = (H2 div x⁶⁴)||((H2 mod x⁶⁴) × (x⁶⁴+x⁶³+x⁶²+x⁵⁷)) div x⁶⁴ | +C | H2L = (H2 mod x⁶⁴)||(((H2 mod x⁶⁴) × (x⁶³+x⁶²+x⁵⁷)) mod x⁶⁴) + (H2 div x⁶⁴) | +C | | +C | H3M = (H3 div x⁶⁴)||((H3 mod x⁶⁴) × (x⁶⁴+x⁶³+x⁶²+x⁵⁷)) div x⁶⁴ | +C | H3L = (H3 mod x⁶⁴)||(((H3 mod x⁶⁴) × (x⁶³+x⁶²+x⁵⁷)) mod x⁶⁴) + (H3 div x⁶⁴) | +C | | +C | H4M = (H3 div x⁶⁴)||((H4 mod x⁶⁴) × (x⁶⁴+x⁶³+x⁶²+x⁵⁷)) div x⁶⁴ | +C | H4L = (H3 mod x⁶⁴)||(((H4 mod x⁶⁴) × (x⁶³+x⁶²+x⁵⁷)) mod x⁶⁴) + (H4 div x⁶⁴) | +C ******************************************************************************* + +define(`FUNC_ALIGN', `5') +PROLOGUE(_nettle_gcm_init_key) + DATA_LOAD_VEC(POLY,.polynomial,r7) C 0xC2000000000000000000000000000001 +IF_LE(` + li r8,0 + lvsl LE_MASK,0,r8 C 0x000102030405060708090A0B0C0D0E0F + vspltisb LE_TEMP,0x07 C 0x07070707070707070707070707070707 + vxor LE_MASK,LE_MASK,LE_TEMP C 0x07060504030201000F0E0D0C0B0A0908 +') + + C 'H' is assigned by gcm_set_key() to the middle element of the table + li r10,8*TableElemAlign + lxvd2x VSR(H),r10,TABLE C load 'H' + C byte-reverse of each doubleword permuting on little-endian mode +IF_LE(` + vperm H,H,H,LE_MASK +') + + C --- calculate H = H << 1 mod P(X), P(X) = (x¹²⁸+x¹²⁷+x¹²⁶+x¹²¹+1) --- + + vupkhsb EMSB,H C extend most significant bit to first byte + vspltisb B1,1 C 0x01010101010101010101010101010101 + vspltb EMSB,EMSB,0 C first byte quadword-extend + vsl H,H,B1 C H = H << 1 + vand EMSB,EMSB,POLY C EMSB &= 0xC2000000000000000000000000000001 + vxor ZERO,ZERO,ZERO C 0x00000000000000000000000000000000 + vxor H,H,EMSB C H ^= EMSB + + C --- calculate H^2 = H*H --- + + xxmrghd VSR(POLY_L),VSR(ZERO),VSR(POLY) C 0x0000000000000000C200000000000000 + + C --- Hp = (H mod x⁶⁴) / x⁶⁴ mod P(X) --- + C --- Hp = (H mod x⁶⁴) × (x⁶⁴+x⁶³+x⁶²+x⁵⁷) mod P(X), deg(Hp) ≤ 127 --- + C --- Hp = (H mod x⁶⁴) × (x⁶⁴+x⁶³+x⁶²+x⁵⁷) --- + vpmsumd Hp,H,POLY_L C Hp = (H mod x⁶⁴) × (x⁶³+x⁶²+x⁵⁷) + xxmrgld VSR(Hl),VSR(H),VSR(ZERO) C Hl = (H mod x⁶⁴) × x⁶⁴ + xxswapd VSR(Hm),VSR(H) + vxor Hl,Hl,Hp C Hl = Hl + Hp + vxor Hm,Hm,Hp C Hm = Hm + Hp + xxmrghd VSR(H1M),VSR(H),VSR(Hl) C H1M = (H div x⁶⁴)||(Hl div x⁶⁴) + xxmrgld VSR(H1L),VSR(H),VSR(Hm) C H1L = (H mod x⁶⁴)||(Hl mod x⁶⁴) + + vpmsumd F,H1L,H C F = (H1Lh × Hh) + (H1Ll × Hl) + vpmsumd R,H1M,H C R = (H1Mh × Hh) + (H1Ml × Hl) + + C --- rduction --- + vpmsumd T,F,POLY_L C T = (F mod x⁶⁴) × (x⁶³+x⁶²+x⁵⁷) + xxswapd VSR(H2),VSR(F) + vxor R,R,T C R = R + T + vxor H2,R,H2 + + xxmrgld VSR(Hl),VSR(H2),VSR(ZERO) + xxswapd VSR(Hm),VSR(H2) + vpmsumd Hp,H2,POLY_L + vxor Hl,Hl,Hp + vxor Hm,Hm,Hp + xxmrghd VSR(H2M),VSR(H2),VSR(Hl) + xxmrgld VSR(H2L),VSR(H2),VSR(Hm) + + C store H1M, H1L, H2M, H2L + li r8,1*TableElemAlign + li r9,2*TableElemAlign + li r10,3*TableElemAlign + stxvd2x VSR(H1M),0,TABLE + stxvd2x VSR(H1L),r8,TABLE + stxvd2x VSR(H2M),r9,TABLE + stxvd2x VSR(H2L),r10,TABLE + + C --- calculate H^3 = H^1*H^2, H^4 = H^2*H^2 --- + + vpmsumd F,H1L,H2 + vpmsumd F2,H2L,H2 + vpmsumd R,H1M,H2 + vpmsumd R2,H2M,H2 + + vpmsumd T,F,POLY_L + vpmsumd T2,F2,POLY_L + xxswapd VSR(H3),VSR(F) + xxswapd VSR(H4),VSR(F2) + vxor R,R,T + vxor R2,R2,T2 + vxor H3,R,H3 + vxor H4,R2,H4 + + xxmrgld VSR(Hl),VSR(H3),VSR(ZERO) + xxmrgld VSR(Hl2),VSR(H4),VSR(ZERO) + xxswapd VSR(Hm),VSR(H3) + xxswapd VSR(Hm2),VSR(H4) + vpmsumd Hp,H3,POLY_L + vpmsumd Hp2,H4,POLY_L + vxor Hl,Hl,Hp + vxor Hl2,Hl2,Hp2 + vxor Hm,Hm,Hp + vxor Hm2,Hm2,Hp2 + xxmrghd VSR(H1M),VSR(H3),VSR(Hl) + xxmrghd VSR(H2M),VSR(H4),VSR(Hl2) + xxmrgld VSR(H1L),VSR(H3),VSR(Hm) + xxmrgld VSR(H2L),VSR(H4),VSR(Hm2) + + C store H3M, H3L, H4M, H4L + li r7,4*TableElemAlign + li r8,5*TableElemAlign + li r9,6*TableElemAlign + li r10,7*TableElemAlign + stxvd2x VSR(H1M),r7,TABLE + stxvd2x VSR(H1L),r8,TABLE + stxvd2x VSR(H2M),r9,TABLE + stxvd2x VSR(H2L),r10,TABLE + + blr +EPILOGUE(_nettle_gcm_init_key) + +define(`TABLE', `r3') +define(`X', `r4') +define(`LENGTH', `r5') +define(`DATA', `r6') + +define(`ZERO', `v16') +define(`POLY', `v17') +define(`POLY_L', `v0') + +define(`D', `v1') +define(`C0', `v2') +define(`C1', `v3') +define(`C2', `v4') +define(`C3', `v5') +define(`H1M', `v6') +define(`H1L', `v7') +define(`H2M', `v8') +define(`H2L', `v9') +define(`H3M', `v10') +define(`H3L', `v11') +define(`H4M', `v12') +define(`H4L', `v13') +define(`R', `v14') +define(`F', `v15') +define(`R2', `v16') +define(`F2', `v17') +define(`R3', `v18') +define(`F3', `v20') +define(`R4', `v21') +define(`F4', `v22') +define(`T', `v23') + +define(`LE_TEMP', `v18') +define(`LE_MASK', `v19') + + C void gcm_hash (const struct gcm_key *key, union gcm_block *x, + C size_t length, const uint8_t *data) + +define(`FUNC_ALIGN', `5') +PROLOGUE(_nettle_gcm_hash) + DATA_LOAD_VEC(POLY,.polynomial,r7) +IF_LE(` + li r8,0 + lvsl LE_MASK,0,r8 + vspltisb LE_TEMP,0x07 + vxor LE_MASK,LE_MASK,LE_TEMP +') + vxor ZERO,ZERO,ZERO + xxmrghd VSR(POLY_L),VSR(ZERO),VSR(POLY) + + lxvd2x VSR(D),0,X C load 'X' pointer + C byte-reverse of each doubleword permuting on little-endian mode +IF_LE(` + vperm D,D,D,LE_MASK +') + + C --- process 4 blocks '128-bit each' per one loop --- + + srdi r7,LENGTH,6 C 4-blocks loop count 'LENGTH / (4 * 16)' + cmpldi r7,0 + beq L2x + + mtctr r7 C assign counter register to loop count + + C store non-volatile vector registers + addi r8,SP,-64 + stvx 20,0,r8 + addi r8,r8,16 + stvx 21,0,r8 + addi r8,r8,16 + stvx 22,0,r8 + addi r8,r8,16 + stvx 23,0,r8 + + C load table elements + li r8,1*TableElemAlign + li r9,2*TableElemAlign + li r10,3*TableElemAlign + lxvd2x VSR(H1M),0,TABLE + lxvd2x VSR(H1L),r8,TABLE + lxvd2x VSR(H2M),r9,TABLE + lxvd2x VSR(H2L),r10,TABLE + li r7,4*TableElemAlign + li r8,5*TableElemAlign + li r9,6*TableElemAlign + li r10,7*TableElemAlign + lxvd2x VSR(H3M),r7,TABLE + lxvd2x VSR(H3L),r8,TABLE + lxvd2x VSR(H4M),r9,TABLE + lxvd2x VSR(H4L),r10,TABLE + + li r8,0x10 + li r9,0x20 + li r10,0x30 +.align 5 +L4x_loop: + C input loading + lxvd2x VSR(C0),0,DATA C load C0 + lxvd2x VSR(C1),r8,DATA C load C1 + lxvd2x VSR(C2),r9,DATA C load C2 + lxvd2x VSR(C3),r10,DATA C load C3 + +IF_LE(` + vperm C0,C0,C0,LE_MASK + vperm C1,C1,C1,LE_MASK + vperm C2,C2,C2,LE_MASK + vperm C3,C3,C3,LE_MASK +') + + C previous digest combining + vxor C0,C0,D + + C polynomial multiplication + vpmsumd F2,H3L,C1 + vpmsumd R2,H3M,C1 + vpmsumd F3,H2L,C2 + vpmsumd R3,H2M,C2 + vpmsumd F4,H1L,C3 + vpmsumd R4,H1M,C3 + vpmsumd F,H4L,C0 + vpmsumd R,H4M,C0 + + C deferred recombination of partial products + vxor F3,F3,F4 + vxor R3,R3,R4 + vxor F,F,F2 + vxor R,R,R2 + vxor F,F,F3 + vxor R,R,R3 + + C reduction + vpmsumd T,F,POLY_L + xxswapd VSR(D),VSR(F) + vxor R,R,T + vxor D,R,D + + addi DATA,DATA,0x40 + bdnz L4x_loop + + C restore non-volatile vector registers + addi r8,SP,-64 + lvx 20,0,r8 + addi r8,r8,16 + lvx 21,0,r8 + addi r8,r8,16 + lvx 22,0,r8 + addi r8,r8,16 + lvx 23,0,r8 + + clrldi LENGTH,LENGTH,58 C 'set the high-order 58 bits to zeros' +L2x: + C --- process 2 blocks --- + + srdi r7,LENGTH,5 C 'LENGTH / (2 * 16)' + cmpldi r7,0 + beq L1x + + C load table elements + li r8,1*TableElemAlign + li r9,2*TableElemAlign + li r10,3*TableElemAlign + lxvd2x VSR(H1M),0,TABLE + lxvd2x VSR(H1L),r8,TABLE + lxvd2x VSR(H2M),r9,TABLE + lxvd2x VSR(H2L),r10,TABLE + + C input loading + li r10,0x10 + lxvd2x VSR(C0),0,DATA C load C0 + lxvd2x VSR(C1),r10,DATA C load C1 + +IF_LE(` + vperm C0,C0,C0,LE_MASK + vperm C1,C1,C1,LE_MASK +') + + C previous digest combining + vxor C0,C0,D + + C polynomial multiplication + vpmsumd F2,H1L,C1 + vpmsumd R2,H1M,C1 + vpmsumd F,H2L,C0 + vpmsumd R,H2M,C0 + + C deferred recombination of partial products + vxor F,F,F2 + vxor R,R,R2 + + C reduction + vpmsumd T,F,POLY_L + xxswapd VSR(D),VSR(F) + vxor R,R,T + vxor D,R,D + + addi DATA,DATA,0x20 + clrldi LENGTH,LENGTH,59 C 'set the high-order 59 bits to zeros' +L1x: + C --- process 1 block --- + + srdi r7,LENGTH,4 C 'LENGTH / (1 * 16)' + cmpldi r7,0 + beq Lmod + + C load table elements + li r8,1*TableElemAlign + lxvd2x VSR(H1M),0,TABLE + lxvd2x VSR(H1L),r8,TABLE + + C input loading + lxvd2x VSR(C0),0,DATA C load C0 + +IF_LE(` + vperm C0,C0,C0,LE_MASK +') + + C previous digest combining + vxor C0,C0,D + + C polynomial multiplication + vpmsumd F,H1L,C0 + vpmsumd R,H1M,C0 + + C reduction + vpmsumd T,F,POLY_L + xxswapd VSR(D),VSR(F) + vxor R,R,T + vxor D,R,D + + addi DATA,DATA,0x10 + clrldi LENGTH,LENGTH,60 C 'set the high-order 60 bits to zeros' +Lmod: + C --- process the modulo bytes, padding the low-order bytes with zeros --- + + cmpldi LENGTH,0 + beq Ldone + + C load table elements + li r8,1*TableElemAlign + lxvd2x VSR(H1M),0,TABLE + lxvd2x VSR(H1L),r8,TABLE + + C push every modulo byte to the stack and load them with padding into vector register + vxor ZERO,ZERO,ZERO + addi r8,SP,-16 + stvx ZERO,0,r8 +Lstb_loop: + subic. LENGTH,LENGTH,1 + lbzx r7,LENGTH,DATA + stbx r7,LENGTH,r8 + bne Lstb_loop + lxvd2x VSR(C0),0,r8 + +IF_LE(` + vperm C0,C0,C0,LE_MASK +') + + C previous digest combining + vxor C0,C0,D + + C polynomial multiplication + vpmsumd F,H1L,C0 + vpmsumd R,H1M,C0 + + C reduction + vpmsumd T,F,POLY_L + xxswapd VSR(D),VSR(F) + vxor R,R,T + vxor D,R,D + +Ldone: + C byte-reverse of each doubleword permuting on little-endian mode +IF_LE(` + vperm D,D,D,LE_MASK +') + stxvd2x VSR(D),0,X C store digest 'D' + + blr +EPILOGUE(_nettle_gcm_hash) + +.data + C 0xC2000000000000000000000000000001 +.polynomial: +.align 4 +IF_BE(` +.byte 0xC2 +.rept 14 +.byte 0x00 +.endr +.byte 0x01 +',` +.byte 0x01 +.rept 14 +.byte 0x00 +.endr +.byte 0xC2 +') -- 2.17.1

3 28

[PATCH] "PowerPC" Detect VSX support on AIX and FreeBSD
by Maamoun TK 11 Nov '20

11 Nov '20

--- fat-ppc.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/fat-ppc.c b/fat-ppc.c index 2bfd649f..ec971706 100644 --- a/fat-ppc.c +++ b/fat-ppc.c @@ -43,8 +43,13 @@ #if defined(_AIX) # include <sys/systemcfg.h> #elif defined(__linux__) +# include <asm/cputable.h> # include <sys/auxv.h> #elif defined(__FreeBSD__) +# include <machine/cpu.h> +# ifdef PPC_FEATURE2_HAS_VEC_CRYPTO +# define PPC_FEATURE2_VEC_CRYPTO PPC_FEATURE2_HAS_VEC_CRYPTO +# endif # if __FreeBSD__ >= 12 # include <sys/auxv.h> # else @@ -58,7 +63,13 @@ #include "gcm.h" #include "fat-setup.h" -/* Define from arch/powerpc/include/uapi/asm/cputable.h in Linux kernel */ +/* Defines from arch/powerpc/include/uapi/asm/cputable.h in Linux kernel */ +#ifndef PPC_FEATURE_HAS_ALTIVEC +#define PPC_FEATURE_HAS_ALTIVEC 0x10000000 +#endif +#ifndef PPC_FEATURE_HAS_VSX +#define PPC_FEATURE_HAS_VSX 0x00000080 +#endif #ifndef PPC_FEATURE2_VEC_CRYPTO #define PPC_FEATURE2_VEC_CRYPTO 0x02000000 #endif @@ -96,8 +107,10 @@ get_ppc_features (struct ppc_features *features) } else { -#if defined(_AIX) && defined(__power_8_andup) - features->have_crypto_ext = __power_8_andup() != 0 ? 1 : 0; +#if defined(_AIX) + features->have_crypto_ext + = _system_configuration.implementation >= 0x10000u; + features->have_altivec = _system_configuration.vmx_version > 1; #else unsigned long hwcap = 0; unsigned long hwcap2 = 0; @@ -106,9 +119,13 @@ get_ppc_features (struct ppc_features *features) hwcap2 = getauxval(AT_HWCAP2); # elif defined(__FreeBSD__) # if __FreeBSD__ >= 12 + elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap)); elf_aux_info(AT_HWCAP2, &hwcap2, sizeof(hwcap2)); # else - size_t len = sizeof(hwcap2); + size_t len; + len = sizeof(hwcap); + sysctlbyname("hw.cpu_features", &hwcap, &len, NULL, 0); + len = sizeof(hwcap2); sysctlbyname("hw.cpu_features2", &hwcap2, &len, NULL, 0); # endif # endif -- 2.17.1

2 1

Modular inversion via powering
by nisse＠lysator.liu.se 09 Nov '20

09 Nov '20

Hi, Nettle includes a function for side-channel silent modular inversion, which asymptotically is O(n^2), like binary gcd but slower by a pretty large constant factor. For prime moduli, inversion can also be done via a^{-1} = a^{p-2} (mod p). That's asymptotically O(n^3) (if the underlying multiplies are done with the basic O(n^2) algorithm), but it may nevertheless be faster than the other method for numbers of the size used for Nettle's elliptic curves. The code for curve25519 and curve448 has been using powering to invert for a long time. I've now spent some time writing specific powering code for the five secp curves as well. I've found fairly efficient addition chains where powering for a prime of n bits needs n-1 squarings and about a dozen multiplies. (I don't know what the *optimal* addition chains are, if you know of tools for that, let me know). Code is on the branch optimize-ecc-invert. However, there's some risk the new code is slower on some platforms, in particular platforms with slow multiplication. The main benchmark is ./examples/hogweed-benchmark ecdsa To get numbers for just the changed function, one can run ./examples-ecc-benchmark and look at the modinv column. Please compare performance between this branch and master, on the platforms that are important to you. And to get relevant numbers, make sure to build using a recent GMP library; performance when built with mini-gmp is not so important. I will merge these changes to master in a week or two, if no problems show up I've benchmarked on one recent and one older x86_64 machine, and on raspberry-pi version 1 (the slowest ARM machine I had easy access to). I've seen improvements in ecdsa signing performance for all the secp curves, ranging from 5% up to 20% depending on curve and platform. Not all inversions are rewritten. I haven't changed the modq inversion which is needed for ecdsa, and I haven't changed the code for the two supported gost curves. Regards, /Niels -- Niels Möller. PGP-encrypted email is preferred. Keyid 368C6677. Internet email is subject to wholesale government surveillance.

2 1

[PATCH] "PowerPC64" chacha-core big-endian support "Shorter version"
by Maamoun TK 07 Nov '20

07 Nov '20

The last patch follows the C implementation but I just figured out a decent way to do it. --- powerpc64/p7/chacha-core-internal.asm | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/powerpc64/p7/chacha-core-internal.asm b/powerpc64/p7/chacha-core-internal.asm index 33c721c1..76ca0d45 100644 --- a/powerpc64/p7/chacha-core-internal.asm +++ b/powerpc64/p7/chacha-core-internal.asm @@ -53,6 +53,10 @@ define(`S1', `v9') define(`S2', `v10') define(`S3', `v11') +C Big-endian working state +define(`LE_MASK', `v12') +define(`LE_TEMP', `v13') + C QROUND(X0, X1, X2, X3) define(`QROUND', ` C x0 += x1, x3 ^= x0, x3 lrot 16 @@ -77,10 +81,18 @@ define(`QROUND', ` vrlw $2, $2, ROT7 ') +C LE_SWAP32(X0, X1, X2, X3) +define(`LE_SWAP32', `IF_BE(` + vperm X0, X0, X0, LE_MASK + vperm X1, X1, X1, LE_MASK + vperm X2, X2, X2, LE_MASK + vperm X3, X3, X3, LE_MASK +')') + .text - .align 4 C _chacha_core(uint32_t *dst, const uint32_t *src, unsigned rounds) +define(`FUNC_ALIGN', `5') PROLOGUE(_nettle_chacha_core) li r6, 0x10 C set up some... @@ -91,6 +103,12 @@ PROLOGUE(_nettle_chacha_core) vspltisw ROT12, 12 vspltisw ROT8, 8 vspltisw ROT7, 7 +IF_BE(` + li r9, 0 + lvsl LE_MASK, r9, r9 + vspltisb LE_TEMP, 0x03 + vxor LE_MASK, LE_MASK, LE_TEMP +') lxvw4x VSR(X0), 0, SRC lxvw4x VSR(X1), r6, SRC @@ -131,6 +149,8 @@ PROLOGUE(_nettle_chacha_core) vadduwm X2, X2, S2 vadduwm X3, X3, S3 + LE_SWAP32(X0, X1, X2, X3) + stxvw4x VSR(X0), 0, DST stxvw4x VSR(X1), r6, DST stxvw4x VSR(X2), r7, DST -- 2.17.1

3 4

SHA1 Collision Detection
by Neal H. Walfield 04 Nov '20

04 Nov '20

Hi, It's well known that SHA-1 is broken. I don't want to save it. But, particularly when dealing with data at rest, there are cases where one has to use SHA-1. It would be nice if Nettle integrated SHA-1 collision detection to make that a tiny bit safer: https://github.com/cr-marcstevens/sha1collisiondetection That library is under the MIT license, and apparently detects known attacks against SHA-1: [The routines] will compute the SHA-1 hash of any given file and additionally will detect cryptanalytic collision attacks against SHA-1 present in each file. It is very fast and takes less than twice the amount of time as regular SHA-1. More specifically they will detect any cryptanalytic collision attack against SHA-1 using any of the top 32 SHA-1 disturbance vectors with probability 1: ... The possibility of false positives can be neglected as the probability is smaller than 2^-90. Thanks, :) Neal

4 8

Blowfish integer overshift (undefined behavior)
by Guido Vranken 29 Oct '20

29 Oct '20

Hi all, My project is Cryptofuzz (https://github.com/guidovranken/cryptofuzz) which uses differential fuzzing to find correctness bugs (and memory bugs as well) in popular cryptographic libraries. It has bindings for Nettle and it tests many of the library's features: https://github.com/guidovranken/cryptofuzz/blob/master/modules/nettle/modul… I've been running this on Google's OSS-Fuzz ( https://github.com/google/oss-fuzz) for a while, and today it found a bug in the blowfish key setter function. Niels and other maintainers (if any), if you would like to be notified by e-mail of bugs found by Cryptofuzz, please send me your e-mail address and I will add you to the project. The e-mail address needs to be linked to a Google account in order to access the dashboard at oss-fuzz.com. Bug reproducer below: ------- #include <nettle/blowfish.h> int main(void) { const unsigned char key[] = {0xec, 0x00, 0x3a, 0x06, 0x73, 0x61, 0x74, 0x20, 0x74, 0xab, 0xe2, 0xc6, 0x61, 0x8b, 0x98, 0x89}; struct blowfish_ctx ctx; blowfish_set_key(&ctx, sizeof(key), key); return 0; } ------- If you compile with Clang and -fsanitize=undefined, this will print: blowfish.c:388:22: runtime error: left shift of 236 by 24 places cannot be represented in type 'int' Explicit casting around the shifted values will fix this.

2 1

[PATCH] "PowerPC64" GCM support
by Maamoun TK 11 Oct '20

11 Oct '20

--- configure.ac | 6 +- gcm.c | 49 +++- powerpc64/p8/gcm-hash.asm | 607 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 647 insertions(+), 15 deletions(-) create mode 100644 powerpc64/p8/gcm-hash.asm diff --git a/configure.ac b/configure.ac index e9983697..0129f950 100644 --- a/configure.ac +++ b/configure.ac @@ -488,7 +488,7 @@ asm_replace_list="aes-encrypt-internal.asm aes-decrypt-internal.asm \ sha3-permute.asm umac-nh.asm umac-nh-n.asm machine.m4" # Assembler files which generate additional object files if they are used. -asm_nettle_optional_list="gcm-hash8.asm cpuid.asm \ +asm_nettle_optional_list="gcm-hash.asm gcm-hash8.asm cpuid.asm \ aes-encrypt-internal-2.asm aes-decrypt-internal-2.asm memxor-2.asm \ chacha-3core.asm chacha-core-internal-2.asm salsa20-2core.asm \ salsa20-core-internal-2.asm sha1-compress-2.asm sha256-compress-2.asm \ @@ -612,9 +612,9 @@ AH_VERBATIM([HAVE_NATIVE], #undef HAVE_NATIVE_ecc_secp384r1_redc #undef HAVE_NATIVE_ecc_secp521r1_modp #undef HAVE_NATIVE_ecc_secp521r1_redc -#undef HAVE_NATIVE_gcm_init_key8 +#undef HAVE_NATIVE_gcm_init_key +#undef HAVE_NATIVE_gcm_hash #undef HAVE_NATIVE_gcm_hash8 -#undef HAVE_NATIVE_gcm_fill #undef HAVE_NATIVE_salsa20_core #undef HAVE_NATIVE_salsa20_2core #undef HAVE_NATIVE_fat_salsa20_2core diff --git a/gcm.c b/gcm.c index 48b3e75a..81981c1c 100644 --- a/gcm.c +++ b/gcm.c @@ -140,6 +140,19 @@ gcm_gf_mul (union nettle_block16 *x, const union nettle_block16 *table) memcpy (x->b, Z.b, sizeof(Z)); } # elif GCM_TABLE_BITS == 8 +# if HAVE_NATIVE_gcm_init_key + +#define gcm_init_key _nettle_gcm_init_key +void +_nettle_gcm_init_key (union nettle_block16 *table); +# endif /* HAVE_NATIVE_gcm_init_key */ +# if HAVE_NATIVE_gcm_hash + +#define gcm_hash _nettle_gcm_hash +void +_nettle_gcm_hash (const struct gcm_key *key, union nettle_block16 *x, + size_t length, const uint8_t *data); +# endif /* HAVE_NATIVE_gcm_hash */ # if HAVE_NATIVE_gcm_hash8 #define gcm_hash _nettle_gcm_hash8 @@ -228,6 +241,29 @@ gcm_gf_mul (union nettle_block16 *x, const union nettle_block16 *table) /* Increment the rightmost 32 bits. */ #define INC32(block) INCREMENT(4, (block.b) + GCM_BLOCK_SIZE - 4) +#ifndef gcm_init_key +static void +gcm_init_key(union nettle_block16 *table) +{ +#if GCM_TABLE_BITS + /* Middle element if GCM_TABLE_BITS > 0, otherwise the first + element */ + unsigned i = (1<<GCM_TABLE_BITS)/2; + + /* Algorithm 3 from the gcm paper. First do powers of two, then do + the rest by adding. */ + while (i /= 2) + block16_mulx_ghash(&table[i], &table[2*i]); + for (i = 2; i < 1<<GCM_TABLE_BITS; i *= 2) + { + unsigned j; + for (j = 1; j < i; j++) + block16_xor3(&table[i+j], &table[i], &table[j]); + } +#endif +} +#endif /* !gcm_init_key */ + /* Initialization of GCM. * @ctx: The context of GCM * @cipher: The context of the underlying block cipher @@ -245,18 +281,7 @@ gcm_set_key(struct gcm_key *key, memset(key->h[0].b, 0, GCM_BLOCK_SIZE); f (cipher, GCM_BLOCK_SIZE, key->h[i].b, key->h[0].b); -#if GCM_TABLE_BITS - /* Algorithm 3 from the gcm paper. First do powers of two, then do - the rest by adding. */ - while (i /= 2) - block16_mulx_ghash(&key->h[i], &key->h[2*i]); - for (i = 2; i < 1<<GCM_TABLE_BITS; i *= 2) - { - unsigned j; - for (j = 1; j < i; j++) - block16_xor3(&key->h[i+j], &key->h[i],&key->h[j]); - } -#endif + gcm_init_key(key->h); } #ifndef gcm_hash diff --git a/powerpc64/p8/gcm-hash.asm b/powerpc64/p8/gcm-hash.asm new file mode 100644 index 00000000..540c9f97 --- /dev/null +++ b/powerpc64/p8/gcm-hash.asm @@ -0,0 +1,607 @@ +C powerpc64/p8/gcm-hash.asm + +ifelse(` + Copyright (D) 2020 Mamone Tarsha + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +') + +C Alignment of gcm_key table elements, which is declared in gcm.h +define(`TableElemAlign', `0x100') + +C Register usage: + +define(`SP', `r1') +define(`TOCP', `r2') + +define(`TABLE', `r3') + +define(`ZERO', `v0') +define(`B1', `v1') +define(`EMSB', `v2') +define(`POLY', `v7') +define(`POLY_L', `v1') +define(`POLY_H', `v2') + +define(`H', `v3') +define(`H3', `v3') +define(`Hl', `v4') +define(`Hm', `v5') +define(`Hh', `v6') +define(`RP', `v7') +define(`Mh', `v8') +define(`Ml', `v9') +define(`H2', `v7') +define(`H4', `v7') +define(`H2m', `v8') +define(`H2l', `v9') +define(`H2h', `v10') +define(`RP2', `v11') +define(`M2h', `v12') +define(`M2l', `v13') + +define(`H2_l', `v14') +define(`H2_m', `v15') +define(`H2_h', `v16') +define(`H3_l', `v14') +define(`H3_m', `v15') +define(`H3_h', `v16') +define(`H4_l', `v17') +define(`H4_m', `v18') +define(`H4_h', `v19') + +define(`H21l', `v16') +define(`H21h', `v17') +define(`H3m', `v16') +define(`H4m', `v17') +define(`H43l', `v18') +define(`H43h', `v19') + +define(`LE_TEMP', `v18') +define(`LE_MASK', `v19') + +.file "gcm-hash.asm" + +.text + + C void gcm_init_key (union gcm_block *table) + +C This function populates the gcm table as the following layout +C ******************************************************************** +C | Hm = low-order doubleword of H^1:high-order doubleword of H^1 | +C | Hl = 64-bits zeros:low-order doubleword of H^1 | +C | Hh = high-order doubleword of H^1:64-bits zeros | +C | | +C | H2m = low-order doubleword of H^2:high-order doubleword of H^2 | +C | H21l = low-order doubleword of H^2:low-order doubleword of H^1 | +C | H21h = high-order doubleword of H^2:high-order doubleword of H^1 | +C | | +C | H3m = low-order doubleword of H^3:high-order doubleword of H^3 | +C | H4m = low-order doubleword of H^4:high-order doubleword of H^4 | +C | H43l = low-order doubleword of H^4:low-order doubleword of H^3 | +C | H43h = high-order doubleword of H^4:high-order doubleword of H^3 | +C ******************************************************************** + +define(`FUNC_ALIGN', `5') +PROLOGUE(_nettle_gcm_init_key) + DATA_LOAD_VEC(POLY,.polynomial,r7) C 0xC2000000000000000000000000000001 +IF_LE(` + li r8,0 + lvsl LE_MASK,0,r8 C 0x000102030405060708090A0B0C0D0E0F + vspltisb LE_TEMP,0x07 C 0x07070707070707070707070707070707 + vxor LE_MASK,LE_MASK,LE_TEMP C 0x07060504030201000F0E0D0C0B0A0908 +') + + C 'H' is assigned by gcm_set_key() to the middle element of the table + li r10,8*TableElemAlign + lxvd2x VSR(H),r10,TABLE C load 'H' + C byte-reverse of each doubleword permuting on little-endian mode +IF_LE(` + vperm H,H,H,LE_MASK +') + + C --- calculate [H = H << 1 modulo polynomial] --- + + vupkhsb EMSB,H C extend most significant bit to first byte + vspltisb B1,1 C 0x01010101010101010101010101010101 + vspltb EMSB,EMSB,0 C first byte quadword-extend + vsl H,H,B1 C H = H << 1 + vand EMSB,EMSB,POLY C EMSB &= 0xC2000000000000000000000000000001 + vxor ZERO,ZERO,ZERO C 0x00000000000000000000000000000000 + vxor H,H,EMSB C H ^= EMSB + + C calculate [Hl = 0:H^1l, Hm = H^1l:H^1h, Hh = H^1h:0] + xxmrgld VSR(Hl),VSR(ZERO),VSR(H) + xxswapd VSR(Hm),VSR(H) + xxmrghd VSR(Hh),VSR(H),VSR(ZERO) + + C --- calculate H^2 = H*H --- + + C reduction pre-processing + xxmrghd VSR(POLY_H),VSR(POLY),VSR(ZERO) C 0xC2000000000000000000000000000000 + xxmrghd VSR(POLY_L),VSR(ZERO),VSR(POLY) C 0x0000000000000000C200000000000000 + + C polynomial multiplication "classical" + vpmsumd H2_l,H,Hl C H^1l*H^1l + vpmsumd H2_m,H,Hm C H^1h*H^1l⊕H^1l*H^1h + vpmsumd H2_h,H,Hh C H^1h*H^1h + + C reduction first phase [1] + vpmsumd RP,H2_l,POLY_L C [1] + + C polynomial multiplication post-processing [2] + xxmrghd VSR(Mh),VSR(ZERO),VSR(H2_m) C [2] + xxmrgld VSR(Ml),VSR(H2_m),VSR(ZERO) C [2] + xxswapd VSR(RP),VSR(RP) C [1] + vxor H2_h,H2_h,Mh C [2] + vxor H2_l,H2_l,Ml C [2] + vxor H2_l,H2_l,RP C [1] + + C reduction second phase + vpmsumd RP,H2_l,POLY_H + vxor H2_h,H2_h,H2_l + vxor H2,H2_h,RP + + C store [H2m = H^2l:H^2h, H2l = 0:H^2l, H2h = H^2h:0] + xxswapd VSR(H2m),VSR(H2) + xxmrgld VSR(H2l),VSR(ZERO),VSR(H2) + xxmrghd VSR(H2h),VSR(H2),VSR(ZERO) + + C calculate [H21l = H^2l:H^1l, H21h = H^2h:H^1h] + xxmrgld VSR(H21l),VSR(H2),VSR(H) + xxmrghd VSR(H21h),VSR(H2),VSR(H) + + C store [Hm, Hl, Hh] + li r9,1*TableElemAlign + li r10,2*TableElemAlign + stxvd2x VSR(Hm),0,TABLE + stxvd2x VSR(Hl),r9,TABLE + stxvd2x VSR(Hh),r10,TABLE + + C store [H2m, H21l, H21h] + li r8,3*TableElemAlign + li r9,4*TableElemAlign + li r10,5*TableElemAlign + stxvd2x VSR(H2m),r8,TABLE + stxvd2x VSR(H21l),r9,TABLE + stxvd2x VSR(H21h),r10,TABLE + + C --- calculate H^3 = H^1*H^2, H^4 = H^2*H^2 --- + + C polynomial multiplication "classical" + vpmsumd H3_l,H,H2l C H^1l*H^2l + vpmsumd H4_l,H2,H2l C H^2l*H^2l + vpmsumd H3_m,H,H2m C H^1h*H^2l⊕H^1l*H^2h + vpmsumd H4_m,H2,H2m C H^2h*H^2l⊕H^2l*H^2h + vpmsumd H3_h,H,H2h C H^1h*H^2h + vpmsumd H4_h,H2,H2h C H^2h*H^2h + + C reduction first phase [1] + vpmsumd RP,H3_l,POLY_L C [1] H^3 + vpmsumd RP2,H4_l,POLY_L C [1] H^4 + + C polynomial multiplication post-processing [2] + xxmrghd VSR(Mh),VSR(ZERO),VSR(H3_m) C [2] H^3 + xxmrghd VSR(M2h),VSR(ZERO),VSR(H4_m) C [2] H^4 + xxmrgld VSR(Ml),VSR(H3_m),VSR(ZERO) C [2] H^3 + xxmrgld VSR(M2l),VSR(H4_m),VSR(ZERO) C [2] H^4 + xxswapd VSR(RP),VSR(RP) C [1] H^3 + xxswapd VSR(RP2),VSR(RP2) C [1] H^4 + vxor H3_h,H3_h,Mh C [2] H^3 + vxor H4_h,H4_h,M2h C [2] H^4 + vxor H3_l,H3_l,Ml C [2] H^3 + vxor H4_l,H4_l,M2l C [2] H^4 + vxor H3_l,H3_l,RP C [1] H^3 + vxor H4_l,H4_l,RP2 C [1] H^4 + + C reduction second phase + vpmsumd RP,H3_l,POLY_H C H^3 + vpmsumd RP2,H4_l,POLY_H C H^4 + vxor H3_h,H3_h,H3_l C H^3 + vxor H4_h,H4_h,H4_l C H^4 + vxor H3,H3_h,RP C H^3 + vxor H4,H4_h,RP2 C H^4 + + C calculate [H3m = H^3l:H^3h, H4m = H^4l:H^4h, H43l = H^4l:H^3l, H43h = H^4h:H^3h] + xxswapd VSR(H3m),VSR(H3) + xxswapd VSR(H4m),VSR(H4) + xxmrgld VSR(H43l),VSR(H4),VSR(H3) + xxmrghd VSR(H43h),VSR(H4),VSR(H3) + + C store [H3m, H4m, H43l, H43h] + li r7,6*TableElemAlign + li r8,7*TableElemAlign + li r9,8*TableElemAlign + li r10,9*TableElemAlign + stxvd2x VSR(H3m),r7,TABLE + stxvd2x VSR(H4m),r8,TABLE + stxvd2x VSR(H43l),r9,TABLE + stxvd2x VSR(H43h),r10,TABLE + + blr +EPILOGUE(_nettle_gcm_init_key) + +define(`TABLE', `r3') +define(`X', `r4') +define(`LENGTH', `r5') +define(`DATA', `r6') + +define(`ZERO', `v0') +define(`POLY', `v3') +define(`POLY_L', `v1') +define(`POLY_H', `v2') + +define(`D', `v3') +define(`C0', `v4') +define(`C1', `v5') +define(`C2', `v6') +define(`C3', `v7') +define(`Mh', `v8') +define(`Ml', `v9') +define(`RP', `v10') +define(`C01h', `v11') +define(`C01l', `v12') +define(`C23h', `v13') +define(`C23l', `v14') + +define(`H1', `v15') +define(`H2', `v16') +define(`H21l', `v17') +define(`H21h', `v18') +define(`H3', `v20') +define(`H4', `v21') +define(`H43l', `v22') +define(`H43h', `v23') + +define(`Cl', `v5') +define(`Cm', `v6') +define(`Ch', `v7') +define(`Hl', `v15') +define(`H', `v16') +define(`Hh', `v17') + +define(`LE_TEMP', `v18') +define(`LE_MASK', `v19') + + C void gcm_hash (const struct gcm_key *key, union gcm_block *x, + C size_t length, const uint8_t *data) + +define(`FUNC_ALIGN', `5') +PROLOGUE(_nettle_gcm_hash) + DATA_LOAD_VEC(POLY,.polynomial,r7) +IF_LE(` + li r8,0 + lvsl LE_MASK,0,r8 + vspltisb LE_TEMP,0x07 + vxor LE_MASK,LE_MASK,LE_TEMP +') + vxor ZERO,ZERO,ZERO + + xxmrghd VSR(POLY_L),VSR(ZERO),VSR(POLY) + xxmrghd VSR(POLY_H),VSR(POLY),VSR(ZERO) + + lxvd2x VSR(D),0,X C load 'X' pointer + C byte-reverse of each doubleword permuting on little-endian mode +IF_LE(` + vperm D,D,D,LE_MASK +') + + C --- process 4 blocks '128-bit each' per one loop --- + + srdi r7,LENGTH,6 C 4-blocks loop count 'LENGTH / (4 * 16)' + cmpldi r7,0 + beq L2x + + mtctr r7 C assign counter register to loop count + + C backup non-volatile vector registers + addi r8,SP,-64 + stvx 20,0,r8 + addi r8,r8,16 + stvx 21,0,r8 + addi r8,r8,16 + stvx 22,0,r8 + addi r8,r8,16 + stvx 23,0,r8 + + C load table elements + li r8,3*TableElemAlign + li r9,4*TableElemAlign + li r10,5*TableElemAlign + lxvd2x VSR(H1),0,TABLE + lxvd2x VSR(H2),r8,TABLE + lxvd2x VSR(H21l),r9,TABLE + lxvd2x VSR(H21h),r10,TABLE + li r7,6*TableElemAlign + li r8,7*TableElemAlign + li r9,8*TableElemAlign + li r10,9*TableElemAlign + lxvd2x VSR(H3),r7,TABLE + lxvd2x VSR(H4),r8,TABLE + lxvd2x VSR(H43l),r9,TABLE + lxvd2x VSR(H43h),r10,TABLE + + li r8,0x10 + li r9,0x20 + li r10,0x30 +.align 5 +L4x_loop: + C input loading + lxvd2x VSR(C0),0,DATA C load C0 + lxvd2x VSR(C1),r8,DATA C load C1 + lxvd2x VSR(C2),r9,DATA C load C2 + lxvd2x VSR(C3),r10,DATA C load C3 + +IF_LE(` + vperm C0,C0,C0,LE_MASK + vperm C1,C1,C1,LE_MASK + vperm C2,C2,C2,LE_MASK + vperm C3,C3,C3,LE_MASK +') + + C previous digest combining + vxor C0,C0,D + + C polynomial multiplication "classical" pre-processing + xxmrghd VSR(C23h),VSR(C2),VSR(C3) + xxmrgld VSR(C23l),VSR(C2),VSR(C3) + xxmrghd VSR(C01h),VSR(C0),VSR(C1) + xxmrgld VSR(C01l),VSR(C0),VSR(C1) + + C polynomial multiplication "classical" + vpmsumd C3,C3,H1 C M3 = H^1l*C3h⊕H^1h*C3l + vpmsumd C2,C2,H2 C M2 = H^2l*C2h⊕H^2h*C2l + vpmsumd C1,C1,H3 C M1 = H^3l*C1h⊕H^3h*C1l + vpmsumd C0,C0,H4 C M0 = H^4l*C0h⊕H^4h*C0l + vpmsumd C23h,C23h,H21h C H23 = H^2h*C2h⊕H^1h*C3h + vpmsumd C23l,C23l,H21l C L23 = H^2l*C2l⊕H^1l*C3l + vpmsumd C01h,C01h,H43h C H01 = H^4h*C0h⊕H^4h*C1h + vpmsumd C01l,C01l,H43l C L01 = H^4l*C0l⊕H^4l*C1l + + C polynomial multiplication "classical" post-processing + vxor C2,C2,C3 C M2 = M2⊕M3 + vxor C0,C0,C1 C M0 = M0⊕M1 + + C deferred recombination of partial products + vxor C01h,C01h,C23h C H0 = H01⊕H23 + vxor C01l,C01l,C23l C L0 = L01⊕L23 + vxor C0,C0,C2 C M0 = M0⊕M2 + + C reduction first phase [1] + vpmsumd RP,C01l,POLY_L C [1] + + C polynomial multiplication post-processing [2] + xxmrghd VSR(Mh),VSR(ZERO),VSR(C0) C [2] + xxmrgld VSR(Ml),VSR(C0),VSR(ZERO) C [2] + xxswapd VSR(RP),VSR(RP) C [1] + vxor C01h,C01h,Mh C [2] + vxor C01l,C01l,Ml C [2] + vxor C01l,C01l,RP C [1] + + C reduction second phase + vpmsumd RP,C01l,POLY_H + vxor C01h,C01l,C01h + vxor D,C01h,RP + + addi DATA,DATA,0x40 + bdnz L4x_loop + + C restore non-volatile vector registers + addi r8,SP,-64 + lvx 20,0,r8 + addi r8,r8,16 + lvx 21,0,r8 + addi r8,r8,16 + lvx 22,0,r8 + addi r8,r8,16 + lvx 23,0,r8 + + clrldi LENGTH,LENGTH,58 C 'set the high-order 58 bits to zeros' +L2x: + C --- process 2 blocks --- + + srdi r7,LENGTH,5 C 'LENGTH / (2 * 16)' + cmpldi r7,0 + beq L1x + + C load table elements + li r8,3*TableElemAlign + li r9,4*TableElemAlign + li r10,5*TableElemAlign + lxvd2x VSR(H1),0,TABLE + lxvd2x VSR(H2),r8,TABLE + lxvd2x VSR(H21l),r9,TABLE + lxvd2x VSR(H21h),r10,TABLE + + C input loading + li r10,0x10 + lxvd2x VSR(C0),0,DATA C load C0 + lxvd2x VSR(C1),r10,DATA C load C1 + +IF_LE(` + vperm C0,C0,C0,LE_MASK + vperm C1,C1,C1,LE_MASK +') + + C previous digest combining + vxor C0,C0,D + + C polynomial multiplication "classical" pre-processing + xxmrghd VSR(C01h),VSR(C0),VSR(C1) + xxmrgld VSR(C01l),VSR(C0),VSR(C1) + + C polynomial multiplication "classical" + vpmsumd C1,C1,H1 C M1 = H^1l*C1h⊕H^1h*C1l + vpmsumd C0,C0,H2 C M0 = H^2l*C0h⊕H^2h*C0l + vpmsumd C01h,C01h,H21h C H01 = H^2h*C0h⊕H^1h*C1h + vpmsumd C01l,C01l,H21l C L01 = H^2l*C0l⊕H^1l*C1l + + C deferred recombination of partial products + vxor C0,C0,C1 C M0 = M0⊕M1 + + C reduction first phase [1] + vpmsumd RP,C01l,POLY_L C [1] + + C polynomial multiplication post-processing [2] + xxmrghd VSR(Mh),VSR(ZERO),VSR(C0) C [2] + xxmrgld VSR(Ml),VSR(C0),VSR(ZERO) C [2] + xxswapd VSR(RP),VSR(RP) C [1] + vxor C01h,C01h,Mh C [2] + vxor C01l,C01l,Ml C [2] + vxor C01l,C01l,RP C [1] + + C reduction second phase + vpmsumd RP,C01l,POLY_H + vxor C01h,C01l,C01h + vxor D,C01h,RP + + addi DATA,DATA,0x20 + clrldi LENGTH,LENGTH,59 C 'set the high-order 59 bits to zeros' +L1x: + C --- process 1 block --- + + srdi r7,LENGTH,4 C 'LENGTH / (1 * 16)' + cmpldi r7,0 + beq Lmod + + C load table elements + li r9,1*TableElemAlign + li r10,2*TableElemAlign + lxvd2x VSR(H),0,TABLE + lxvd2x VSR(Hl),r9,TABLE + lxvd2x VSR(Hh),r10,TABLE + + C input loading + lxvd2x VSR(C0),0,DATA C load C0 + +IF_LE(` + vperm C0,C0,C0,LE_MASK +') + + C previous digest combining + vxor C0,C0,D + + C polynomial multiplication "classical" + vpmsumd Cl,C0,Hl C L = Hl*Cl + vpmsumd Cm,C0,H C M = Hh*Cl⊕Hl*Ch + vpmsumd Ch,C0,Hh C H = Hh*Ch + + C reduction first phase C [1] + vpmsumd RP,Cl,POLY_L C [1] + + C polynomial multiplication post-processing [2] + xxmrghd VSR(Mh),VSR(ZERO),VSR(Cm) C [2] + xxmrgld VSR(Ml),VSR(Cm),VSR(ZERO) C [2] + xxswapd VSR(RP),VSR(RP) C [1] + vxor Ch,Ch,Mh C [2] + vxor Cl,Cl,Ml C [2] + vxor Cl,Cl,RP C [1] + + C reduction second phase + vpmsumd RP,Cl,POLY_H + vxor Ch,Cl,Ch + vxor D,Ch,RP + + addi DATA,DATA,0x10 + clrldi LENGTH,LENGTH,60 C 'set the high-order 60 bits to zeros' +Lmod: + C --- process the modulo bytes, padding the low-order bytes with zeros --- + + cmpldi LENGTH,0 + beq Ldone + + C load table elements + li r9,1*TableElemAlign + li r10,2*TableElemAlign + lxvd2x VSR(H),0,TABLE + lxvd2x VSR(Hl),r9,TABLE + lxvd2x VSR(Hh),r10,TABLE + + C push every modulo byte to the stack and load them with padding into vector register + addi r8,SP,-16 + stvx ZERO,0,r8 +Lstb_loop: + subic. LENGTH,LENGTH,1 + lbzx r7,LENGTH,DATA + stbx r7,LENGTH,r8 + bne Lstb_loop + lxvd2x VSR(C0),0,r8 + +IF_LE(` + vperm C0,C0,C0,LE_MASK +') + + C previous digest combining + vxor C0,C0,D + + C polynomial multiplication "classical" + vpmsumd Cl,C0,Hl C L = Hl*Cl + vpmsumd Cm,C0,H C M = Hh*Cl⊕Hl*Ch + vpmsumd Ch,C0,Hh C H = Hh*Ch + + C reduction first phase [1] + vpmsumd RP,Cl,POLY_L C [1] + + C polynomial multiplication post-processing [2] + xxmrghd VSR(Mh),VSR(ZERO),VSR(Cm) C [2] + xxmrgld VSR(Ml),VSR(Cm),VSR(ZERO) C [2] + xxswapd VSR(RP),VSR(RP) C [1] + vxor Ch,Ch,Mh C [2] + vxor Cl,Cl,Ml C [2] + vxor Cl,Cl,RP C [1] + + C reduction second phase + vpmsumd RP,Cl,POLY_H + vxor Ch,Cl,Ch + vxor D,Ch,RP + +Ldone: + C byte-reverse of each doubleword permuting on little-endian mode +IF_LE(` + vperm D,D,D,LE_MASK +') + stxvd2x VSR(D),0,X C store digest 'D' + + blr +EPILOGUE(_nettle_gcm_hash) + +.data + C 0xC2000000000000000000000000000001 +.polynomial: +.align 4 +IF_BE(` +.byte 0xC2 +.rept 14 +.byte 0x00 +.endr +.byte 0x01 +',` +.byte 0x01 +.rept 14 +.byte 0x00 +.endr +.byte 0xC2 +') -- 2.17.1

3 11

[PATCH] "PowerPC64" GCM support
by Maamoun TK 04 Oct '20

04 Oct '20

--- configure.ac | 6 +- fat-ppc.c | 33 ++ fat-setup.h | 7 + gcm.c | 69 +++- powerpc64/fat/gcm-hash.asm | 39 +++ powerpc64/p8/gcm-hash.asm | 773 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 909 insertions(+), 18 deletions(-) create mode 100644 powerpc64/fat/gcm-hash.asm create mode 100644 powerpc64/p8/gcm-hash.asm diff --git a/configure.ac b/configure.ac index e9983697..0129f950 100644 --- a/configure.ac +++ b/configure.ac @@ -488,7 +488,7 @@ asm_replace_list="aes-encrypt-internal.asm aes-decrypt-internal.asm \ sha3-permute.asm umac-nh.asm umac-nh-n.asm machine.m4" # Assembler files which generate additional object files if they are used. -asm_nettle_optional_list="gcm-hash8.asm cpuid.asm \ +asm_nettle_optional_list="gcm-hash.asm gcm-hash8.asm cpuid.asm \ aes-encrypt-internal-2.asm aes-decrypt-internal-2.asm memxor-2.asm \ chacha-3core.asm chacha-core-internal-2.asm salsa20-2core.asm \ salsa20-core-internal-2.asm sha1-compress-2.asm sha256-compress-2.asm \ @@ -612,9 +612,9 @@ AH_VERBATIM([HAVE_NATIVE], #undef HAVE_NATIVE_ecc_secp384r1_redc #undef HAVE_NATIVE_ecc_secp521r1_modp #undef HAVE_NATIVE_ecc_secp521r1_redc -#undef HAVE_NATIVE_gcm_init_key8 +#undef HAVE_NATIVE_gcm_init_key +#undef HAVE_NATIVE_gcm_hash #undef HAVE_NATIVE_gcm_hash8 -#undef HAVE_NATIVE_gcm_fill #undef HAVE_NATIVE_salsa20_core #undef HAVE_NATIVE_salsa20_2core #undef HAVE_NATIVE_fat_salsa20_2core diff --git a/fat-ppc.c b/fat-ppc.c index 2bc50481..d73b45b0 100644 --- a/fat-ppc.c +++ b/fat-ppc.c @@ -120,6 +120,16 @@ DECLARE_FAT_FUNC(_nettle_aes_decrypt, aes_crypt_internal_func) DECLARE_FAT_FUNC_VAR(aes_decrypt, aes_crypt_internal_func, c) DECLARE_FAT_FUNC_VAR(aes_decrypt, aes_crypt_internal_func, ppc64) +#if GCM_TABLE_BITS == 8 +DECLARE_FAT_FUNC(_nettle_gcm_init_key, gcm_init_key_func) +DECLARE_FAT_FUNC_VAR(gcm_init_key, gcm_init_key_func, c) +DECLARE_FAT_FUNC_VAR(gcm_init_key, gcm_init_key_func, ppc64) + +DECLARE_FAT_FUNC(_nettle_gcm_hash, gcm_hash_func) +DECLARE_FAT_FUNC_VAR(gcm_hash, gcm_hash_func, c) +DECLARE_FAT_FUNC_VAR(gcm_hash, gcm_hash_func, ppc64) +#endif /* GCM_TABLE_BITS == 8 */ + static void CONSTRUCTOR fat_init (void) { @@ -139,11 +149,23 @@ fat_init (void) fprintf (stderr, "libnettle: enabling arch 2.07 code.\n"); _nettle_aes_encrypt_vec = _nettle_aes_encrypt_ppc64; _nettle_aes_decrypt_vec = _nettle_aes_decrypt_ppc64; +#if GCM_TABLE_BITS == 8 + /* Make sure _nettle_gcm_init_key_vec function is compatible + with _nettle_gcm_hash_vec function e.g. _nettle_gcm_init_key_c() + fills gcm_key table with values that are incompatible with + _nettle_gcm_hash_ppc64() */ + _nettle_gcm_init_key_vec = _nettle_gcm_init_key_ppc64; + _nettle_gcm_hash_vec = _nettle_gcm_hash_ppc64; +#endif /* GCM_TABLE_BITS == 8 */ } else { _nettle_aes_encrypt_vec = _nettle_aes_encrypt_c; _nettle_aes_decrypt_vec = _nettle_aes_decrypt_c; +#if GCM_TABLE_BITS == 8 + _nettle_gcm_init_key_vec = _nettle_gcm_init_key_c; + _nettle_gcm_hash_vec = _nettle_gcm_hash_c; +#endif /* GCM_TABLE_BITS == 8 */ } } @@ -160,3 +182,14 @@ DEFINE_FAT_FUNC(_nettle_aes_decrypt, void, size_t length, uint8_t *dst, const uint8_t *src), (rounds, keys, T, length, dst, src)) + +#if GCM_TABLE_BITS == 8 +DEFINE_FAT_FUNC(_nettle_gcm_init_key, void, + (union nettle_block16 *table), + (table)) + +DEFINE_FAT_FUNC(_nettle_gcm_hash, void, + (const struct gcm_key *key, union nettle_block16 *x, + size_t length, const uint8_t *data), + (key, x, length, data)) +#endif /* GCM_TABLE_BITS == 8 */ diff --git a/fat-setup.h b/fat-setup.h index 99f1ea67..4a9f7969 100644 --- a/fat-setup.h +++ b/fat-setup.h @@ -162,6 +162,13 @@ typedef void aes_crypt_internal_func (unsigned rounds, const uint32_t *keys, size_t length, uint8_t *dst, const uint8_t *src); +#if GCM_TABLE_BITS == 8 +typedef void gcm_init_key_func (union nettle_block16 *table); + +typedef void gcm_hash_func (const struct gcm_key *key, union nettle_block16 *x, + size_t length, const uint8_t *data); +#endif /* GCM_TABLE_BITS == 8 */ + typedef void *(memxor_func)(void *dst, const void *src, size_t n); typedef void salsa20_core_func (uint32_t *dst, const uint32_t *src, unsigned rounds); diff --git a/gcm.c b/gcm.c index 48b3e75a..787c303e 100644 --- a/gcm.c +++ b/gcm.c @@ -140,6 +140,26 @@ gcm_gf_mul (union nettle_block16 *x, const union nettle_block16 *table) memcpy (x->b, Z.b, sizeof(Z)); } # elif GCM_TABLE_BITS == 8 +# if HAVE_NATIVE_gcm_init_key + +#define gcm_init_key _nettle_gcm_init_key +void +_nettle_gcm_init_key (union nettle_block16 *table); +/* For fat builds */ +void +_nettle_gcm_init_key_c (union nettle_block16 *table); +# endif /* HAVE_NATIVE_gcm_init_key */ +# if HAVE_NATIVE_gcm_hash + +#define gcm_hash _nettle_gcm_hash +void +_nettle_gcm_hash (const struct gcm_key *key, union nettle_block16 *x, + size_t length, const uint8_t *data); +/* For fat builds */ +void +_nettle_gcm_hash_c (const struct gcm_key *key, union nettle_block16 *x, + size_t length, const uint8_t *data); +# endif /* HAVE_NATIVE_gcm_hash */ # if HAVE_NATIVE_gcm_hash8 #define gcm_hash _nettle_gcm_hash8 @@ -228,6 +248,32 @@ gcm_gf_mul (union nettle_block16 *x, const union nettle_block16 *table) /* Increment the rightmost 32 bits. */ #define INC32(block) INCREMENT(4, (block.b) + GCM_BLOCK_SIZE - 4) +#ifdef gcm_init_key +void +_nettle_gcm_init_key_c(union nettle_block16 *table) +#else +static void +gcm_init_key(union nettle_block16 *table) +#endif /* !gcm_init_key */ +{ +#if GCM_TABLE_BITS + /* Middle element if GCM_TABLE_BITS > 0, otherwise the first + element */ + unsigned i = (1<<GCM_TABLE_BITS)/2; + + /* Algorithm 3 from the gcm paper. First do powers of two, then do + the rest by adding. */ + while (i /= 2) + block16_mulx_ghash(&table[i], &table[2*i]); + for (i = 2; i < 1<<GCM_TABLE_BITS; i *= 2) + { + unsigned j; + for (j = 1; j < i; j++) + block16_xor3(&table[i+j], &table[i], &table[j]); + } +#endif +} + /* Initialization of GCM. * @ctx: The context of GCM * @cipher: The context of the underlying block cipher @@ -244,25 +290,19 @@ gcm_set_key(struct gcm_key *key, /* H */ memset(key->h[0].b, 0, GCM_BLOCK_SIZE); f (cipher, GCM_BLOCK_SIZE, key->h[i].b, key->h[0].b); - -#if GCM_TABLE_BITS - /* Algorithm 3 from the gcm paper. First do powers of two, then do - the rest by adding. */ - while (i /= 2) - block16_mulx_ghash(&key->h[i], &key->h[2*i]); - for (i = 2; i < 1<<GCM_TABLE_BITS; i *= 2) - { - unsigned j; - for (j = 1; j < i; j++) - block16_xor3(&key->h[i+j], &key->h[i],&key->h[j]); - } -#endif + + gcm_init_key(key->h); } -#ifndef gcm_hash +#ifdef gcm_hash +void +_nettle_gcm_hash_c(const struct gcm_key *key, union nettle_block16 *x, + size_t length, const uint8_t *data) +#else static void gcm_hash(const struct gcm_key *key, union nettle_block16 *x, size_t length, const uint8_t *data) +#endif /* !gcm_hash */ { for (; length >= GCM_BLOCK_SIZE; length -= GCM_BLOCK_SIZE, data += GCM_BLOCK_SIZE) @@ -276,7 +316,6 @@ gcm_hash(const struct gcm_key *key, union nettle_block16 *x, gcm_gf_mul (x, key->h); } } -#endif /* !gcm_hash */ static void gcm_hash_sizes(const struct gcm_key *key, union nettle_block16 *x, diff --git a/powerpc64/fat/gcm-hash.asm b/powerpc64/fat/gcm-hash.asm new file mode 100644 index 00000000..7fd77223 --- /dev/null +++ b/powerpc64/fat/gcm-hash.asm @@ -0,0 +1,39 @@ +C powerpc64/fat/gcm-hash.asm + + +ifelse(` + Copyright (C) 2020 Mamone Tarsha + + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +') + +dnl picked up by configure +dnl PROLOGUE(_nettle_gcm_init_key) +dnl PROLOGUE(_nettle_gcm_hash) + +define(`fat_transform', `$1_ppc64') +include_src(`powerpc64/p8/gcm-hash.asm') diff --git a/powerpc64/p8/gcm-hash.asm b/powerpc64/p8/gcm-hash.asm new file mode 100644 index 00000000..f987f2ca --- /dev/null +++ b/powerpc64/p8/gcm-hash.asm @@ -0,0 +1,773 @@ +C powerpc64/p8/gcm-hash.asm + +ifelse(` + Copyright (C) 2020 Mamone Tarsha + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +') + +C Alignment of gcm_key table elements, which is declared in gcm.h +define(`TableElemAlign', `0x100') + +C Register usage: + +define(`SP', `r1') +define(`TOCP', `r2') + +define(`TABLE', `r3') +define(`X', `r4') +define(`LENGTH', `r5') +define(`DATA', `r6') + +define(`zero', `v0') +define(`swap_mask', `v1') +define(`hidw_mask', `v2') +define(`lodw_mask', `v3') +define(`poly', `v4') +define(`poly_h', `v4') +define(`poly_l', `v5') +define(`RP', `v6') +define(`Mh', `v7') +define(`Ml', `v8') +define(`H', `v9') +define(`Hh', `v10') +define(`Hl', `v11') +define(`RP2', `v9') +define(`M2h', `v10') +define(`M2l', `v11') + +define(`sl1', `v1') +define(`msb', `v5') +define(`H2', `v6') +define(`H2h', `v7') +define(`H2l', `v8') +define(`H_h', `v12') +define(`H_m', `v13') +define(`H_l', `v14') +define(`H_Hh', `v12') +define(`H_H', `v13') +define(`H_Hl', `v14') +define(`H_t', `v15') +define(`H2_h', `v16') +define(`H2_m', `v17') +define(`H2_l', `v18') +define(`H2_t', `v19') + +define(`C0', `v6') +define(`C1', `v7') +define(`C2', `v8') +define(`C3', `v12') +define(`C4', `v6') +define(`C5', `v7') +define(`C6', `v8') +define(`C7', `v12') + +define(`C', `v13') + +define(`Ch', `v14') +define(`Cl', `v15') +define(`Cm', `v16') + +define(`C01h', `v14') +define(`C01l', `v15') +define(`C01', `v16') +define(`C23h', `v17') +define(`C23l', `v18') +define(`C23', `v19') +define(`C45h', `v20') +define(`C45l', `v21') +define(`C45', `v22') +define(`C67h', `v6') +define(`C67l', `v7') +define(`C67', `v8') + +define(`H21', `v9') +define(`H21h', `v10') +define(`H21l', `v11') +define(`H43', `v23') +define(`H43h', `v24') +define(`H43l', `v25') +define(`H65', `v26') +define(`H65h', `v27') +define(`H65l', `v28') +define(`H87', `v29') +define(`H87h', `v30') +define(`H87l', `v31') + +.file "gcm-hash.asm" + +.text + + # void gcm_init_key (union gcm_block *table) + +define(`FUNC_ALIGN', `5') +PROLOGUE(_nettle_gcm_init_key) + DATA_LOAD_VEC(poly,.polynomial,r7) +IF_LE(`DATA_LOAD_VEC(swap_mask,.swap_mask,r7)') + DATA_LOAD_VEC(hidw_mask,.hidw_mask,r7) + DATA_LOAD_VEC(lodw_mask,.lodw_mask,r7) + + li r10,8*TableElemAlign + lxvd2x VSR(H),r10,TABLE # load H +IF_LE(`vperm H,H,H,swap_mask') + + # --- calculate H = H shift left 1 modulo polynomial --- + + vupkhsw msb,H # most significant bit word-extend + vspltisb sl1,1 # splat 1 for shift left + vspltw msb,msb,0 # most significant bit extend + vsl H,H,sl1 # H shift left 1 + vand msb,msb,poly + vxor zero,zero,zero + vxor H_t,H,msb + + vsldoi H,H_t,H_t,8 # doubleword swap + vsldoi Hh,H,zero,8 + vsldoi Hl,zero,H,8 + + # --- calculate H^2 = H*H --- + + # reduction pre-processing + vsldoi poly_h,zero,poly,8 + vsldoi poly_l,poly_h,poly_h,8 + + # polynomial multiplication "classical" + vpmsumd H_h,H_t,Hh # H^1h*H^1h + vpmsumd H_l,H_t,Hl # H^1l*H^1l + vpmsumd H_m,H_t,H # H^1h*H^1l⊕H^1l*H^1h + + # reduction first phase # [1] + vpmsumd RP,H_l,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,H_m,8 # [2] + vsldoi Ml,H_m,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor H_h,H_h,Mh # [2] + vxor H_l,H_l,Ml # [2] + vxor H_l,H_l,RP # [1] + + # reduction second phase + vpmsumd RP,H_l,poly_l + vxor H_h,H_l,H_h + vxor H2_t,H_h,RP + + vsldoi H2,H2_t,H2_t,8 + vsldoi H2h,H2,zero,8 + vsldoi H2l,zero,H2,8 + + # --- calculate [H^2.Hi⊕H^2.Lo:H^1.Hi⊕H^1.Lo] --- + + vperm H_Hh,H2,H,lodw_mask + vperm H_Hl,H2,H,hidw_mask + vxor H_H,H_Hh,H_Hl + + # --- store H,[H^2.Hi⊕H^2.Lo:H^1.Hi⊕H^1.Lo] --- + + li r8,0*TableElemAlign + li r9,1*TableElemAlign + li r10,2*TableElemAlign + stxvd2x VSR(Hl),r8,TABLE + stxvd2x VSR(H),r9,TABLE + stxvd2x VSR(Hh),r10,TABLE + + li r8,3*TableElemAlign + li r9,4*TableElemAlign + li r10,5*TableElemAlign + stxvd2x VSR(H_Hh),r8,TABLE + stxvd2x VSR(H_H),r9,TABLE + stxvd2x VSR(H_Hl),r10,TABLE + + # --- calculate H^3,H^4 --- + + # polynomial multiplication "classical" + vpmsumd H_l,H_t,H2l # H^1l*H^2l + vpmsumd H_m,H_t,H2 # H^1h*H^2l⊕H^1l*H^2h + vpmsumd H_h,H_t,H2h # H^1h*H^2h + vpmsumd H2_l,H2_t,H2l # H^2l*H^2l + vpmsumd H2_m,H2_t,H2 # H^2h*H^2l⊕H^2l*H^2h + vpmsumd H2_h,H2_t,H2h # H^2h*H^2h + + # reduction first phase # [1] + vpmsumd RP,H_l,poly_h # [1] H^3 + vpmsumd RP2,H2_l,poly_h # [1] H^4 + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,H_m,8 # [2] H^3 + vsldoi M2h,zero,H2_m,8 # [2] H^4 + vsldoi Ml,H_m,zero,8 # [2] H^3 + vsldoi M2l,H2_m,zero,8 # [2] H^4 + vsldoi RP,RP,RP,8 # [1] H^3 + vsldoi RP2,RP2,RP2,8 # [1] H^4 + vxor H_h,H_h,Mh # [2] H^3 + vxor H2_h,H2_h,M2h # [2] H^4 + vxor H_l,H_l,Ml # [2] H^3 + vxor H2_l,H2_l,M2l # [2] H^4 + vxor H_l,H_l,RP # [1] H^3 + vxor H2_l,H2_l,RP2 # [1] H^4 + + # reduction second phase + vpmsumd RP,H_l,poly_l # H^3 + vpmsumd RP2,H2_l,poly_l # H^4 + vxor H_h,H_l,H_h # H^3 + vxor H2_h,H2_l,H2_h # H^4 + vxor H_h,H_h,RP # H^3 + vxor H2_h,H2_h,RP2 # H^4 + + vsldoi H2,H2_h,H2_h,8 # H^4 + vsldoi H,H_h,H_h,8 # H^3 + vsldoi H2l,zero,H2,8 # H^4 + vsldoi H2h,H2,zero,8 # H^4 + + # --- calculate [H^4.Hi⊕H^4.Lo:H^3.Hi⊕H^3.Lo] --- + + vperm H_Hh,H2,H,lodw_mask + vperm H_Hl,H2,H,hidw_mask + vxor H_H,H_Hh,H_Hl + + # --- store [H^4.Hi⊕H^4.Lo:H^3.Hi⊕H^3.Lo] --- + + li r8,6*TableElemAlign + li r9,7*TableElemAlign + li r10,8*TableElemAlign + stxvd2x VSR(H_Hh),r8,TABLE + stxvd2x VSR(H_H),r9,TABLE + stxvd2x VSR(H_Hl),r10,TABLE + + # --- calculate H^5,H^6 --- + + # polynomial multiplication "classical" + vpmsumd H_l,H_t,H2l # H^1l*H^4l + vpmsumd H_m,H_t,H2 # H^1h*H^4l⊕H^1l*H^4h + vpmsumd H_h,H_t,H2h # H^1h*H^4h + vpmsumd H2_l,H2_t,H2l # H^2l*H^4l + vpmsumd H2_m,H2_t,H2 # H^2h*H^4l⊕H^2l*H^4h + vpmsumd H2_h,H2_t,H2h # H^2h*H^4h + + # reduction first phase # [1] + vpmsumd RP,H_l,poly_h # [1] H^5 + vpmsumd RP2,H2_l,poly_h # [1] H^6 + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,H_m,8 # [2] H^5 + vsldoi M2h,zero,H2_m,8 # [2] H^6 + vsldoi Ml,H_m,zero,8 # [2] H^5 + vsldoi M2l,H2_m,zero,8 # [2] H^6 + vsldoi RP,RP,RP,8 # [1] H^5 + vsldoi RP2,RP2,RP2,8 # [1] H^6 + vxor H_h,H_h,Mh # [2] H^5 + vxor H2_h,H2_h,M2h # [2] H^6 + vxor H_l,H_l,Ml # [2] H^5 + vxor H2_l,H2_l,M2l # [2] H^6 + vxor H_l,H_l,RP # [1] H^5 + vxor H2_l,H2_l,RP2 # [1] H^6 + + # reduction second phase + vpmsumd RP,H_l,poly_l # H^5 + vpmsumd RP2,H2_l,poly_l # H^6 + vxor H_h,H_l,H_h # H^5 + vxor H2_h,H2_l,H2_h # H^6 + vxor H_h,H_h,RP # H^5 + vxor H2_h,H2_h,RP2 # H^6 + + vsldoi H2,H2_h,H2_h,8 # H^6 + vsldoi H,H_h,H_h,8 # H^5 + vsldoi H2l,zero,H2,8 # H^6 + vsldoi H2h,H2,zero,8 # H^6 + + # --- calculate [H^6.Hi⊕H^6.Lo:H^5.Hi⊕H^5.Lo] --- + + vperm H_Hh,H2,H,lodw_mask + vperm H_Hl,H2,H,hidw_mask + vxor H_H,H_Hh,H_Hl + + # --- store [H^6.Hi⊕H^6.Lo:H^5.Hi⊕H^5.Lo] --- + + li r8,9*TableElemAlign + li r9,10*TableElemAlign + li r10,11*TableElemAlign + stxvd2x VSR(H_Hh),r8,TABLE + stxvd2x VSR(H_H),r9,TABLE + stxvd2x VSR(H_Hl),r10,TABLE + + # --- calculate H^7,H^8 --- + + # polynomial multiplication "classical" + vpmsumd H_l,H_t,H2l # H^1l*H^6l + vpmsumd H_m,H_t,H2 # H^1h*H^6l⊕H^1l*H^6h + vpmsumd H_h,H_t,H2h # H^1h*H^6h + vpmsumd H2_l,H2_t,H2l # H^2l*H^6l + vpmsumd H2_m,H2_t,H2 # H^2h*H^6l⊕H^2l*H^6h + vpmsumd H2_h,H2_t,H2h # H^2h*H^6h + + # reduction first phase # [1] + vpmsumd RP,H_l,poly_h # [1] H^7 + vpmsumd RP2,H2_l,poly_h # [1] H^8 + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,H_m,8 # [2] H^7 + vsldoi M2h,zero,H2_m,8 # [2] H^8 + vsldoi Ml,H_m,zero,8 # [2] H^7 + vsldoi M2l,H2_m,zero,8 # [2] H^8 + vsldoi RP,RP,RP,8 # [1] H^7 + vsldoi RP2,RP2,RP2,8 # [1] H^8 + vxor H_h,H_h,Mh # [2] H^7 + vxor H2_h,H2_h,M2h # [2] H^8 + vxor H_l,H_l,Ml # [2] H^7 + vxor H2_l,H2_l,M2l # [2] H^8 + vxor H_l,H_l,RP # [1] H^7 + vxor H2_l,H2_l,RP2 # [1] H^8 + + # reduction second phase + vpmsumd RP,H_l,poly_l # H^7 + vpmsumd RP2,H2_l,poly_l # H^8 + vxor H_h,H_l,H_h # H^7 + vxor H2_h,H2_l,H2_h # H^8 + vxor H_h,H_h,RP # H^7 + vxor H2_h,H2_h,RP2 # H^8 + + vsldoi H,H_h,H_h,8 # H^7 + vsldoi H2,H2_h,H2_h,8 # H^8 + + # --- calculate [H^8.Hi⊕H^8.Lo:H^7.Hi⊕H^7.Lo] --- + + vperm H_Hh,H2,H,lodw_mask + vperm H_Hl,H2,H,hidw_mask + vxor H_H,H_Hh,H_Hl + + # --- store [H^8.Hi⊕H^8.Lo:H^7.Hi⊕H^7.Lo] --- + + li r8,12*TableElemAlign + li r9,13*TableElemAlign + li r10,14*TableElemAlign + stxvd2x VSR(H_Hh),r8,TABLE + stxvd2x VSR(H_H),r9,TABLE + stxvd2x VSR(H_Hl),r10,TABLE + + blr +EPILOGUE(_nettle_gcm_init_key) + + # void gcm_hash (const struct gcm_key *key, union gcm_block *x, + # size_t length, const uint8_t *data) + +define(`FUNC_ALIGN', `5') +PROLOGUE(_nettle_gcm_hash) + vxor zero,zero,zero + + DATA_LOAD_VEC(poly,.polynomial,r7) +IF_LE(`DATA_LOAD_VEC(swap_mask,.swap_mask,r7)') + DATA_LOAD_VEC(hidw_mask,.hidw_mask,r7) + DATA_LOAD_VEC(lodw_mask,.lodw_mask,r7) + + vsldoi poly_h,zero,poly,8 + vsldoi poly_l,poly_h,poly_h,8 + + lxvd2x VSR(C),0,X # load X +IF_LE(`vperm C,C,C,swap_mask') + + srdi r7,LENGTH,7 # 8x loop count + cmpldi r7,0 + beq L2x + + # backup registers + stdu SP,-224(SP) + std r28,216(SP) + std r29,208(SP) + std r30,200(SP) + std r31,192(SP) + li 8,176 + stvx 20,8,SP + subi 8,8,16 + stvx 21,8,SP + subi 8,8,16 + stvx 22,8,SP + subi 8,8,16 + stvx 23,8,SP + subi 8,8,16 + stvx 24,8,SP + subi 8,8,16 + stvx 25,8,SP + subi 8,8,16 + stvx 26,8,SP + subi 8,8,16 + stvx 27,8,SP + subi 8,8,16 + stvx 28,8,SP + subi 8,8,16 + stvx 29,8,SP + subi 8,8,16 + stvx 30,8,SP + subi 8,8,16 + stvx 31,8,SP + + # table loading + li r8,3*TableElemAlign + li r9,4*TableElemAlign + li r10,5*TableElemAlign + lxvd2x VSR(H21h),r8,TABLE + lxvd2x VSR(H21),r9,TABLE + lxvd2x VSR(H21l),r10,TABLE + li r8,6*TableElemAlign + li r9,7*TableElemAlign + li r10,8*TableElemAlign + lxvd2x VSR(H43h),r8,TABLE + lxvd2x VSR(H43),r9,TABLE + lxvd2x VSR(H43l),r10,TABLE + li r8,9*TableElemAlign + li r9,10*TableElemAlign + li r10,11*TableElemAlign + lxvd2x VSR(H65h),r8,TABLE + lxvd2x VSR(H65),r9,TABLE + lxvd2x VSR(H65l),r10,TABLE + li r8,12*TableElemAlign + li r9,13*TableElemAlign + li r10,14*TableElemAlign + lxvd2x VSR(H87h),r8,TABLE + lxvd2x VSR(H87),r9,TABLE + lxvd2x VSR(H87l),r10,TABLE + + li r8,0x10 + li r9,0x20 + li r10,0x30 + li r28,0x40 + li r29,0x50 + li r30,0x60 + li r31,0x70 + + mtctr r7 +.align 5 +L8x_loop: + # input loading + lxvd2x VSR(C0),0,DATA # load C0 + lxvd2x VSR(C1),r8,DATA # load C1 + lxvd2x VSR(C2),r9,DATA # load C2 + lxvd2x VSR(C3),r10,DATA # load C3 + + # swap permuting +IF_LE(`vperm C0,C0,C0,swap_mask + vperm C1,C1,C1,swap_mask + vperm C2,C2,C2,swap_mask + vperm C3,C3,C3,swap_mask') + + # previous digest combining + vxor C0,C0,C + + # polynomial multiplication "karatsuba" pre-processing + vperm C23h,C2,C3,hidw_mask + vperm C23l,C2,C3,lodw_mask + vperm C01h,C0,C1,hidw_mask + vperm C01l,C0,C1,lodw_mask + + # input loading + lxvd2x VSR(C4),r28,DATA # load C4 + lxvd2x VSR(C5),r29,DATA # load C5 + lxvd2x VSR(C6),r30,DATA # load C6 + lxvd2x VSR(C7),r31,DATA # load C7 + + # swap permuting +IF_LE(`vperm C4,C4,C4,swap_mask + vperm C5,C5,C5,swap_mask + vperm C6,C6,C6,swap_mask + vperm C7,C7,C7,swap_mask') + + # polynomial multiplication "karatsuba" pre-processing + vperm C45h,C4,C5,hidw_mask + vperm C45l,C4,C5,lodw_mask + vperm C67h,C6,C7,hidw_mask + vperm C67l,C6,C7,lodw_mask + vxor C23,C23h,C23l + vxor C01,C01h,C01l + vxor C45,C45h,C45l + vxor C67,C67h,C67l + + # polynomial multiplication "karatsuba" + vpmsumd C23h,C23h,H65h # H23 = H^6h*C2h⊕H^5h*C3h + vpmsumd C23l,C23l,H65l # L23 = H^6l*C2l⊕H^5l*C3l + vpmsumd C01h,C01h,H87h # H01 = H^8h*C0h⊕H^7h*C1h + vpmsumd C01l,C01l,H87l # L01 = H^8l*C0l⊕H^7l*C1l + vpmsumd C67h,C67h,H21h # H67 = H^2h*C6h⊕H^1h*C7h + vpmsumd C67l,C67l,H21l # L67 = H^2l*C6l⊕H^1l*C7l + vpmsumd C45h,C45h,H43h # H45 = H^4h*C4h⊕H^3h*C5h + vpmsumd C45l,C45l,H43l # L45 = H^4l*C4l⊕H^3l*C5l + vpmsumd C23,C23,H65 # M23 = (H^6h⊕H^5h)*(C2h⊕C3h)⊕(H^6l⊕H^5l)*(C2l⊕C3l) + vpmsumd C01,C01,H87 # M01 = (H^8h⊕H^7h)*(C0h⊕C1h)⊕(H^8l⊕H^7l)*(C0l⊕C1l) + vpmsumd C45,C45,H43 # M45 = (H^4h⊕H^3h)*(C4h⊕C5h)⊕(H^4l⊕H^3l)*(C4l⊕C5l) + vpmsumd C67,C67,H21 # M67 = (H^2h⊕H^1h)*(C6h⊕C7h)⊕(H^2l⊕H^1l)*(C6l⊕C7l) + + # polynomial multiplication "karatsuba" post-processing + vxor C23,C23,C23h + vxor C01,C01,C01h + vxor C45,C45,C45h + vxor C67,C67,C67h + vxor C23,C23,C23l + vxor C01,C01,C01l + vxor C45,C45,C45l + vxor C67,C67,C67l + + # deferred recombination of partial products + vxor C01h,C01h,C23h # H0 = H01⊕H23 + vxor C45h,C45h,C67h # H1 = H45⊕H67 + vxor C01l,C01l,C23l # L0 = L01⊕L23 + vxor C45l,C45l,C67l # L1 = L45⊕L45 + vxor C01,C01,C23 # M0 = M01⊕M23 + vxor C45,C45,C67 # M1 = M45⊕M45 + vxor C01h,C01h,C45h # H = H0⊕H1 + vxor C01l,C01l,C45l # L = L0⊕L1 + vxor C01,C01,C45 # M = M0⊕M1 + + # reduction first phase # [1] + vpmsumd RP,C01l,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,C01,8 # [2] + vsldoi Ml,C01,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor C01h,C01h,Mh # [2] + vxor C01l,C01l,Ml # [2] + vxor C01l,C01l,RP # [1] + + # reduction second phase + vpmsumd RP,C01l,poly_l + vxor C01h,C01l,C01h + vxor C,C01h,RP + + addi DATA,DATA,0x80 + bdnz L8x_loop + + # restore registers + li 8,0 + lvx 31,8,SP + addi 8,8,16 + lvx 30,8,SP + addi 8,8,16 + lvx 29,8,SP + addi 8,8,16 + lvx 28,8,SP + addi 8,8,16 + lvx 27,8,SP + addi 8,8,16 + lvx 26,8,SP + addi 8,8,16 + lvx 25,8,SP + addi 8,8,16 + lvx 24,8,SP + addi 8,8,16 + lvx 23,8,SP + addi 8,8,16 + lvx 22,8,SP + addi 8,8,16 + lvx 21,8,SP + addi 8,8,16 + lvx 20,8,SP + ld r31,192(SP) + ld r30,200(SP) + ld r29,208(SP) + ld r28,216(SP) + addi SP,SP,224 + + clrldi LENGTH,LENGTH,57 +L2x: + srdi r7,LENGTH,5 + cmpldi r7,0 + beq L1x + + # table loading + li r8,3*TableElemAlign + li r9,4*TableElemAlign + li r10,5*TableElemAlign + lxvd2x VSR(H21h),r8,TABLE + lxvd2x VSR(H21),r9,TABLE + lxvd2x VSR(H21l),r10,TABLE + + li r10,0x10 + + mtctr r7 +.align 5 +L2x_loop: + # input loading + lxvd2x VSR(C0),0,DATA # load C0 + lxvd2x VSR(C1),r10,DATA # load C1 + + # swap permuting +IF_LE(`vperm C0,C0,C0,swap_mask + vperm C1,C1,C1,swap_mask') + + # previous digest combining + vxor C0,C0,C + + # polynomial multiplication "karatsuba" pre-processing + vperm C01h,C0,C1,hidw_mask + vperm C01l,C0,C1,lodw_mask + vxor C01,C01h,C01l + + # polynomial multiplication "karatsuba" + vpmsumd C01h,C01h,H21h # H01 = H^2h*C0h⊕H^1h*C1h + vpmsumd C01l,C01l,H21l # L01 = H^2l*C0l⊕H^1l*C1l + vpmsumd C01,C01,H21 # M01 = (H^2h⊕H^1h)*(C0h⊕C1h)⊕(H^2l⊕H^1l)*(C0l⊕C1l) + + # polynomial multiplication "karatsuba" post-processing + vxor C01,C01,C01h + vxor C01,C01,C01l + + # reduction first phase # [1] + vpmsumd RP,C01l,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,C01,8 # [2] + vsldoi Ml,C01,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor C01h,C01h,Mh # [2] + vxor C01l,C01l,Ml # [2] + vxor C01l,C01l,RP # [1] + + # reduction second phase + vpmsumd RP,C01l,poly_l + vxor C01h,C01l,C01h + vxor C,C01h,RP + + addi DATA,DATA,0x20 + bdnz L2x_loop + + clrldi LENGTH,LENGTH,59 +L1x: + srdi r7,LENGTH,4 + cmpldi r7,0 + beq Lrem + + # table loading + li r9,1*TableElemAlign + li r10,2*TableElemAlign + lxvd2x VSR(Hl),0,TABLE + lxvd2x VSR(H), r9,TABLE + lxvd2x VSR(Hh),r10,TABLE + + # input loading + lxvd2x VSR(C0),0,DATA # load C0 + + # swap permuting +IF_LE(`vperm C0,C0,C0,swap_mask') + + # previous digest combining + vxor C0,C0,C + + vpmsumd Cl,C0,Hl # L = Hl*Cl + vpmsumd Cm,C0,H # M = Hh*Cl⊕Hl*Ch + vpmsumd Ch,C0,Hh # H = Hh*Ch + + # reduction first phase # [1] + vpmsumd RP,Cl,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,Cm,8 # [2] + vsldoi Ml,Cm,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor Ch,Ch,Mh # [2] + vxor Cl,Cl,Ml # [2] + vxor Cl,Cl,RP # [1] + + # reduction second phase + vpmsumd RP,Cl,poly_l + vxor Ch,Cl,Ch + vxor C,Ch,RP + + addi DATA,DATA,0x10 + clrldi LENGTH,LENGTH,60 +Lrem: + cmpldi LENGTH,0 + beq Ldone + + # table loading + li r9,1*TableElemAlign + li r10,2*TableElemAlign + lxvd2x VSR(Hl),0,TABLE + lxvd2x VSR(H), r9,TABLE + lxvd2x VSR(Hh),r10,TABLE + + # input loading + stdu SP,-16(SP) + stvx zero,0,SP +Lst_loop: + subic. LENGTH,LENGTH,1 + lbzx r7,LENGTH,DATA + stbx r7,LENGTH,SP + bne Lst_loop + lxvd2x VSR(C0),0,SP + addi SP,SP,16 + + # swap permuting +IF_LE(`vperm C0,C0,C0,swap_mask') + + # previous digest combining + vxor C0,C0,C + + vpmsumd Cl,C0,Hl # L = Hl*Cl + vpmsumd Cm,C0,H # M = Hh*Cl⊕Hl*Ch + vpmsumd Ch,C0,Hh # H = Hh*Ch + + # reduction first phase # [1] + vpmsumd RP,Cl,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,Cm,8 # [2] + vsldoi Ml,Cm,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor Ch,Ch,Mh # [2] + vxor Cl,Cl,Ml # [2] + vxor Cl,Cl,RP # [1] + + # reduction second phase + vpmsumd RP,Cl,poly_l + vxor Ch,Cl,Ch + vxor C,Ch,RP + +Ldone: +IF_LE(`vperm C,C,C,swap_mask') + stxvd2x VSR(C),0,X # store C + blr +EPILOGUE(_nettle_gcm_hash) + + .data +IF_LE(`.align 4 +.polynomial: + .byte 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2 + .align 4 +.swap_mask: + .byte 8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7 + .align 4 +.hidw_mask: + .byte 23,22,21,20,19,18,17,16,7,6,5,4,3,2,1,0 + .align 4 +.lodw_mask: + .byte 31,30,29,28,27,26,25,24,15,14,13,12,11,10,9,8') +IF_BE(`.align 4 +.polynomial: + .byte 0xc2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 + .align 4 +.hidw_mask: + .byte 0,1,2,3,4,5,6,7,16,17,18,19,20,21,22,23 + .align 4 +.lodw_mask: + .byte 8,9,10,11,12,13,14,15,24,25,26,27,28,29,30,31') -- 2.17.1

4 9

[PATCH] "PowerPC64" chacha-core big-endian support
by Maamoun TK 25 Sep '20

25 Sep '20

--- powerpc64/p7/chacha-core-internal.asm | 55 ++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/powerpc64/p7/chacha-core-internal.asm b/powerpc64/p7/chacha-core-internal.asm index 33c721c1..922050ff 100644 --- a/powerpc64/p7/chacha-core-internal.asm +++ b/powerpc64/p7/chacha-core-internal.asm @@ -53,6 +53,18 @@ define(`S1', `v9') define(`S2', `v10') define(`S3', `v11') +C Big-endian working state +define(`ROT24', `v12') +define(`ODD', `v13') +define(`EVEN', `v14') +define(`ZERO', `v15') +define(`NEG', `v16') + +define(`XR0', `v15') +define(`XR1', `v16') +define(`XR2', `v17') +define(`XR3', `v18') + C QROUND(X0, X1, X2, X3) define(`QROUND', ` C x0 += x1, x3 ^= x0, x3 lrot 16 @@ -77,10 +89,42 @@ define(`QROUND', ` vrlw $2, $2, ROT7 ') +C LE_SWAP32(X0, X1, X2, X3) +define(`LE_SWAP32', `IF_BE(` + C xr = x lrot 8, xr &= 0x00FF00FF + C x = x lrot 24, x &= 0xFF00FF00 + C x |= xr + + vrlw XR0, X0, ROT8 + vrlw XR1, X1, ROT8 + vrlw XR2, X2, ROT8 + vrlw XR3, X3, ROT8 + + vand XR0, XR0, ODD + vand XR1, XR1, ODD + vand XR2, XR2, ODD + vand XR3, XR3, ODD + + vrlw X0, X0, ROT24 + vrlw X1, X1, ROT24 + vrlw X2, X2, ROT24 + vrlw X3, X3, ROT24 + + vand X0, X0, EVEN + vand X1, X1, EVEN + vand X2, X2, EVEN + vand X3, X3, EVEN + + vor X0, X0, XR0 + vor X1, X1, XR1 + vor X2, X2, XR2 + vor X3, X3, XR3 +')') + .text - .align 4 C _chacha_core(uint32_t *dst, const uint32_t *src, unsigned rounds) +define(`FUNC_ALIGN', `5') PROLOGUE(_nettle_chacha_core) li r6, 0x10 C set up some... @@ -91,6 +135,13 @@ PROLOGUE(_nettle_chacha_core) vspltisw ROT12, 12 vspltisw ROT8, 8 vspltisw ROT7, 7 +IF_BE(` + vspltisw ZERO, 0 + vspltisw NEG, -1 + vmrghb ODD, ZERO, NEG + vmrghb EVEN, NEG, ZERO + vadduwm ROT24, ROT12, ROT12 +') lxvw4x VSR(X0), 0, SRC lxvw4x VSR(X1), r6, SRC @@ -131,6 +182,8 @@ PROLOGUE(_nettle_chacha_core) vadduwm X2, X2, S2 vadduwm X3, X3, S3 + LE_SWAP32(X0, X1, X2, X3) + stxvw4x VSR(X0), 0, DST stxvw4x VSR(X1), r6, DST stxvw4x VSR(X2), r7, DST -- 2.17.1

1 0

← Newer
1
...
9
10
11
12
13
14
15
...
220
Older →

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

2005

2004

2003

2002

nettle-bugs