nettle-bugs

nettle-bugs@lists.lysator.liu.se

2198 discussions

[PATCH 2/4] Add GHASH optimized implementation for PowerPC64
by Maamoun TK 09 Jul '20

09 Jul '20

--- gcm.c | 19 +- powerpc64/gcm-hash8.asm | 1004 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1022 insertions(+), 1 deletion(-) create mode 100644 powerpc64/gcm-hash8.asm diff --git a/gcm.c b/gcm.c index cf615daf..809c03bc 100644 --- a/gcm.c +++ b/gcm.c @@ -140,6 +140,12 @@ gcm_gf_mul (union nettle_block16 *x, const union nettle_block16 *table) memcpy (x->b, Z.b, sizeof(Z)); } # elif GCM_TABLE_BITS == 8 +# if HAVE_NATIVE_gcm_init_key8 + +#define gcm_init_key _nettle_gcm_init_key8 +void +_nettle_gcm_init_key8 (union nettle_block16 *table); +# endif /* HAVE_NATIVE_gcm_init_key8 */ # if HAVE_NATIVE_gcm_hash8 #define gcm_hash _nettle_gcm_hash8 @@ -225,6 +231,13 @@ gcm_gf_mul (union nettle_block16 *x, const union nettle_block16 *table) #endif /* GCM_TABLE_BITS */ +#if HAVE_NATIVE_gcm_fill + +#define gcm_fill _nettle_gcm_fill +void +_nettle_gcm_fill (uint8_t *ctr, size_t blocks, union nettle_block16 *buffer); +#endif /* HAVE_NATIVE_gcm_fill */ + /* Increment the rightmost 32 bits. */ #define INC32(block) INCREMENT(4, (block.b) + GCM_BLOCK_SIZE - 4) @@ -245,7 +258,9 @@ gcm_set_key(struct gcm_key *key, memset(key->h[0].b, 0, GCM_BLOCK_SIZE); f (cipher, GCM_BLOCK_SIZE, key->h[i].b, key->h[0].b); -#if GCM_TABLE_BITS +#ifdef gcm_init_key + gcm_init_key(key->h); +#elif GCM_TABLE_BITS /* Algorithm 3 from the gcm paper. First do powers of two, then do the rest by adding. */ while (i /= 2) @@ -333,6 +348,7 @@ gcm_update(struct gcm_ctx *ctx, const struct gcm_key *key, ctx->auth_size += length; } +#ifndef gcm_fill static nettle_fill16_func gcm_fill; static void gcm_fill(uint8_t *ctr, size_t blocks, union nettle_block16 *buffer) @@ -349,6 +365,7 @@ gcm_fill(uint8_t *ctr, size_t blocks, union nettle_block16 *buffer) WRITE_UINT32(ctr + GCM_BLOCK_SIZE - 4, c); } +#endif /* !gcm_fill */ void gcm_encrypt (struct gcm_ctx *ctx, const struct gcm_key *key, diff --git a/powerpc64/gcm-hash8.asm b/powerpc64/gcm-hash8.asm new file mode 100644 index 00000000..7d5d3ec9 --- /dev/null +++ b/powerpc64/gcm-hash8.asm @@ -0,0 +1,1004 @@ +C powerpc64/gcm-hash8.asm + +ifelse(< + Copyright (C) 2020 Mamone Tarsha + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +>) + +C Alignment of gcm_key table elements, which is declared in gcm.h +define(<TableElemAlign>, <0x100>) + +C Register usage: + +define(<SP>, <1>) +define(<TOCP>, <2>) + +define(<TABLE>, <3>) +define(<X>, <4>) +define(<LENGTH>, <5>) +define(<DATA>, <6>) + +define(<zero>, <0>) +define(<swap_mask>, <1>) +define(<hidw_mask>, <2>) +define(<lodw_mask>, <3>) +define(<poly>, <4>) +define(<poly_h>, <4>) +define(<poly_l>, <5>) +define(<RP>, <6>) +define(<Mh>, <7>) +define(<Ml>, <8>) +define(<H>, <9>) +define(<Hh>, <10>) +define(<Hl>, <11>) +define(<RP2>, <9>) +define(<M2h>, <10>) +define(<M2l>, <11>) + +define(<HX>, <41>) +define(<HhX>, <42>) +define(<HlX>, <43>) +define(<H_HhX>, <44>) +define(<H_HX>, <45>) +define(<H_HlX>, <46>) + +define(<sl1>, <1>) +define(<msb>, <5>) +define(<H2>, <6>) +define(<H2h>, <7>) +define(<H2l>, <8>) +define(<H_h>, <12>) +define(<H_m>, <13>) +define(<H_l>, <14>) +define(<H_Hh>, <12>) +define(<H_H>, <13>) +define(<H_Hl>, <14>) +define(<H_t>, <15>) +define(<H2_h>, <16>) +define(<H2_m>, <17>) +define(<H2_l>, <18>) +define(<H2_t>, <19>) + +define(<C0X>, <38>) +define(<C1X>, <39>) +define(<C2X>, <40>) +define(<C3X>, <44>) +define(<C4X>, <38>) +define(<C5X>, <39>) +define(<C6X>, <40>) +define(<C7X>, <44>) + +define(<CX>, <45>) + +define(<C0>, <6>) +define(<C1>, <7>) +define(<C2>, <8>) +define(<C3>, <12>) +define(<C4>, <6>) +define(<C5>, <7>) +define(<C6>, <8>) +define(<C7>, <12>) + +define(<C>, <13>) + +define(<Ch>, <14>) +define(<Cl>, <15>) +define(<Cm>, <16>) + +define(<C01h>, <14>) +define(<C01l>, <15>) +define(<C01>, <16>) +define(<C23h>, <17>) +define(<C23l>, <18>) +define(<C23>, <19>) +define(<C45h>, <20>) +define(<C45l>, <21>) +define(<C45>, <22>) +define(<C67h>, <6>) +define(<C67l>, <7>) +define(<C67>, <8>) + +define(<H21>, <9>) +define(<H21h>, <10>) +define(<H21l>, <11>) +define(<H43>, <23>) +define(<H43h>, <24>) +define(<H43l>, <25>) +define(<H65>, <26>) +define(<H65h>, <27>) +define(<H65l>, <28>) +define(<H87>, <29>) +define(<H87h>, <30>) +define(<H87l>, <31>) + +define(<H21X>, <41>) +define(<H21hX>, <42>) +define(<H21lX>, <43>) +define(<H43X>, <55>) +define(<H43hX>, <56>) +define(<H43lX>, <57>) +define(<H65X>, <58>) +define(<H65hX>, <59>) +define(<H65lX>, <60>) +define(<H87X>, <61>) +define(<H87hX>, <62>) +define(<H87lX>, <63>) + +# gcm_fill registers: + +define(<CTR>, <3>) +define(<BLOCKS>, <4>) +define(<BUFFER>, <5>) + +define(<CTR0>, <2>) +define(<CTR0S>, <3>) +define(<CTR1>, <4>) +define(<CTR2>, <5>) +define(<CTR3>, <6>) +define(<CTR4>, <7>) +define(<CTR5>, <8>) +define(<CTR6>, <9>) +define(<CTR7>, <10>) + +define(<CTR0X>, <34>) +define(<CTR0SX>, <35>) +define(<CTR1X>, <36>) +define(<CTR2X>, <37>) +define(<CTR3X>, <38>) +define(<CTR4X>, <39>) +define(<CTR5X>, <40>) +define(<CTR6X>, <41>) +define(<CTR7X>, <42>) + +define(<I1>, <11>) +define(<I2>, <12>) +define(<I3>, <13>) +define(<I4>, <14>) +define(<I5>, <15>) +define(<I6>, <16>) +define(<I7>, <17>) +define(<I8>, <18>) + +.file "gcm-hash8.asm" + +IF_LE(<.abiversion 2>) +.text + + # void gcm_init_key (union gcm_block *table) + +PROLOGUE(_nettle_gcm_init_key8) + ld 7,.polynomial@got(TOCP) + lvx poly,0,7 +IF_LE(<ld 7,.swap_mask@got(TOCP) + lvx swap_mask,0,7>) + ld 7,.hidw_mask@got(TOCP) + lvx hidw_mask,0,7 + ld 7,.lodw_mask@got(TOCP) + lvx lodw_mask,0,7 + + li 10,8*TableElemAlign + lxvd2x HX,10,TABLE # load H +IF_LE(<vperm H,H,H,swap_mask>) + + # --- calculate H = H shift left 1 modulo polynomial --- + + vupkhsw msb,H # most significant bit word-extend + vspltisb sl1,1 # splat 1 for shift left + vspltw msb,msb,0 # most significant bit extend + vsl H,H,sl1 # H shift left 1 + vand msb,msb,poly + vxor zero,zero,zero + vxor H_t,H,msb + + vsldoi H,H_t,H_t,8 # doubleword swap + vsldoi Hh,H,zero,8 + vsldoi Hl,zero,H,8 + + # --- calculate H^2 = H*H --- + + # reduction pre-processing + vsldoi poly_h,zero,poly,8 + vsldoi poly_l,poly_h,poly_h,8 + + # polynomial multiplication "classical" + vpmsumd H_h,H_t,Hh # H^1h*H^1h + vpmsumd H_l,H_t,Hl # H^1l*H^1l + vpmsumd H_m,H_t,H # H^1h*H^1l⊕H^1l*H^1h + + # reduction first phase # [1] + vpmsumd RP,H_l,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,H_m,8 # [2] + vsldoi Ml,H_m,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor H_h,H_h,Mh # [2] + vxor H_l,H_l,Ml # [2] + vxor H_l,H_l,RP # [1] + + # reduction second phase + vpmsumd RP,H_l,poly_l + vxor H_h,H_l,H_h + vxor H2_t,H_h,RP + + vsldoi H2,H2_t,H2_t,8 + vsldoi H2h,H2,zero,8 + vsldoi H2l,zero,H2,8 + + # --- calculate [H^2.Hi⊕H^2.Lo:H^1.Hi⊕H^1.Lo] --- + + vperm H_Hh,H2,H,lodw_mask + vperm H_Hl,H2,H,hidw_mask + vxor H_H,H_Hh,H_Hl + + # --- store H,[H^2.Hi⊕H^2.Lo:H^1.Hi⊕H^1.Lo] --- + + li 8,0*TableElemAlign + li 9,1*TableElemAlign + li 10,2*TableElemAlign + stxvd2x HlX,8,TABLE + stxvd2x HX,9,TABLE + stxvd2x HhX,10,TABLE + + li 8,3*TableElemAlign + li 9,4*TableElemAlign + li 10,5*TableElemAlign + stxvd2x H_HhX,8,TABLE + stxvd2x H_HX,9,TABLE + stxvd2x H_HlX,10,TABLE + + # --- calculate H^3,H^4 --- + + # polynomial multiplication "classical" + vpmsumd H_l,H_t,H2l # H^1l*H^2l + vpmsumd H_m,H_t,H2 # H^1h*H^2l⊕H^1l*H^2h + vpmsumd H_h,H_t,H2h # H^1h*H^2h + vpmsumd H2_l,H2_t,H2l # H^2l*H^2l + vpmsumd H2_m,H2_t,H2 # H^2h*H^2l⊕H^2l*H^2h + vpmsumd H2_h,H2_t,H2h # H^2h*H^2h + + # reduction first phase # [1] + vpmsumd RP,H_l,poly_h # [1] H^3 + vpmsumd RP2,H2_l,poly_h # [1] H^4 + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,H_m,8 # [2] H^3 + vsldoi M2h,zero,H2_m,8 # [2] H^4 + vsldoi Ml,H_m,zero,8 # [2] H^3 + vsldoi M2l,H2_m,zero,8 # [2] H^4 + vsldoi RP,RP,RP,8 # [1] H^3 + vsldoi RP2,RP2,RP2,8 # [1] H^4 + vxor H_h,H_h,Mh # [2] H^3 + vxor H2_h,H2_h,M2h # [2] H^4 + vxor H_l,H_l,Ml # [2] H^3 + vxor H2_l,H2_l,M2l # [2] H^4 + vxor H_l,H_l,RP # [1] H^3 + vxor H2_l,H2_l,RP2 # [1] H^4 + + # reduction second phase + vpmsumd RP,H_l,poly_l # H^3 + vpmsumd RP2,H2_l,poly_l # H^4 + vxor H_h,H_l,H_h # H^3 + vxor H2_h,H2_l,H2_h # H^4 + vxor H_h,H_h,RP # H^3 + vxor H2_h,H2_h,RP2 # H^4 + + vsldoi H2,H2_h,H2_h,8 # H^4 + vsldoi H,H_h,H_h,8 # H^3 + vsldoi H2l,zero,H2,8 # H^4 + vsldoi H2h,H2,zero,8 # H^4 + + # --- calculate [H^4.Hi⊕H^4.Lo:H^3.Hi⊕H^3.Lo] --- + + vperm H_Hh,H2,H,lodw_mask + vperm H_Hl,H2,H,hidw_mask + vxor H_H,H_Hh,H_Hl + + # --- store [H^4.Hi⊕H^4.Lo:H^3.Hi⊕H^3.Lo] --- + + li 8,6*TableElemAlign + li 9,7*TableElemAlign + li 10,8*TableElemAlign + stxvd2x H_HhX,8,TABLE + stxvd2x H_HX,9,TABLE + stxvd2x H_HlX,10,TABLE + + # --- calculate H^5,H^6 --- + + # polynomial multiplication "classical" + vpmsumd H_l,H_t,H2l # H^1l*H^4l + vpmsumd H_m,H_t,H2 # H^1h*H^4l⊕H^1l*H^4h + vpmsumd H_h,H_t,H2h # H^1h*H^4h + vpmsumd H2_l,H2_t,H2l # H^2l*H^4l + vpmsumd H2_m,H2_t,H2 # H^2h*H^4l⊕H^2l*H^4h + vpmsumd H2_h,H2_t,H2h # H^2h*H^4h + + # reduction first phase # [1] + vpmsumd RP,H_l,poly_h # [1] H^5 + vpmsumd RP2,H2_l,poly_h # [1] H^6 + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,H_m,8 # [2] H^5 + vsldoi M2h,zero,H2_m,8 # [2] H^6 + vsldoi Ml,H_m,zero,8 # [2] H^5 + vsldoi M2l,H2_m,zero,8 # [2] H^6 + vsldoi RP,RP,RP,8 # [1] H^5 + vsldoi RP2,RP2,RP2,8 # [1] H^6 + vxor H_h,H_h,Mh # [2] H^5 + vxor H2_h,H2_h,M2h # [2] H^6 + vxor H_l,H_l,Ml # [2] H^5 + vxor H2_l,H2_l,M2l # [2] H^6 + vxor H_l,H_l,RP # [1] H^5 + vxor H2_l,H2_l,RP2 # [1] H^6 + + # reduction second phase + vpmsumd RP,H_l,poly_l # H^5 + vpmsumd RP2,H2_l,poly_l # H^6 + vxor H_h,H_l,H_h # H^5 + vxor H2_h,H2_l,H2_h # H^6 + vxor H_h,H_h,RP # H^5 + vxor H2_h,H2_h,RP2 # H^6 + + vsldoi H2,H2_h,H2_h,8 # H^6 + vsldoi H,H_h,H_h,8 # H^5 + vsldoi H2l,zero,H2,8 # H^6 + vsldoi H2h,H2,zero,8 # H^6 + + # --- calculate [H^6.Hi⊕H^6.Lo:H^5.Hi⊕H^5.Lo] --- + + vperm H_Hh,H2,H,lodw_mask + vperm H_Hl,H2,H,hidw_mask + vxor H_H,H_Hh,H_Hl + + # --- store [H^6.Hi⊕H^6.Lo:H^5.Hi⊕H^5.Lo] --- + + li 8,9*TableElemAlign + li 9,10*TableElemAlign + li 10,11*TableElemAlign + stxvd2x H_HhX,8,TABLE + stxvd2x H_HX,9,TABLE + stxvd2x H_HlX,10,TABLE + + # --- calculate H^7,H^8 --- + + # polynomial multiplication "classical" + vpmsumd H_l,H_t,H2l # H^1l*H^6l + vpmsumd H_m,H_t,H2 # H^1h*H^6l⊕H^1l*H^6h + vpmsumd H_h,H_t,H2h # H^1h*H^6h + vpmsumd H2_l,H2_t,H2l # H^2l*H^6l + vpmsumd H2_m,H2_t,H2 # H^2h*H^6l⊕H^2l*H^6h + vpmsumd H2_h,H2_t,H2h # H^2h*H^6h + + # reduction first phase # [1] + vpmsumd RP,H_l,poly_h # [1] H^7 + vpmsumd RP2,H2_l,poly_h # [1] H^8 + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,H_m,8 # [2] H^7 + vsldoi M2h,zero,H2_m,8 # [2] H^8 + vsldoi Ml,H_m,zero,8 # [2] H^7 + vsldoi M2l,H2_m,zero,8 # [2] H^8 + vsldoi RP,RP,RP,8 # [1] H^7 + vsldoi RP2,RP2,RP2,8 # [1] H^8 + vxor H_h,H_h,Mh # [2] H^7 + vxor H2_h,H2_h,M2h # [2] H^8 + vxor H_l,H_l,Ml # [2] H^7 + vxor H2_l,H2_l,M2l # [2] H^8 + vxor H_l,H_l,RP # [1] H^7 + vxor H2_l,H2_l,RP2 # [1] H^8 + + # reduction second phase + vpmsumd RP,H_l,poly_l # H^7 + vpmsumd RP2,H2_l,poly_l # H^8 + vxor H_h,H_l,H_h # H^7 + vxor H2_h,H2_l,H2_h # H^8 + vxor H_h,H_h,RP # H^7 + vxor H2_h,H2_h,RP2 # H^8 + + vsldoi H,H_h,H_h,8 # H^7 + vsldoi H2,H2_h,H2_h,8 # H^8 + + # --- calculate [H^8.Hi⊕H^8.Lo:H^7.Hi⊕H^7.Lo] --- + + vperm H_Hh,H2,H,lodw_mask + vperm H_Hl,H2,H,hidw_mask + vxor H_H,H_Hh,H_Hl + + # --- store [H^8.Hi⊕H^8.Lo:H^7.Hi⊕H^7.Lo] --- + + li 8,12*TableElemAlign + li 9,13*TableElemAlign + li 10,14*TableElemAlign + stxvd2x H_HhX,8,TABLE + stxvd2x H_HX,9,TABLE + stxvd2x H_HlX,10,TABLE + + blr +EPILOGUE(_nettle_gcm_init_key8) + + # void gcm_hash (const struct gcm_key *key, union gcm_block *x, + # size_t length, const uint8_t *data) + +PROLOGUE(_nettle_gcm_hash8) + vxor zero,zero,zero + + ld 7,.polynomial@got(TOCP) + lvx poly,0,7 +IF_LE(<ld 7,.swap_mask@got(TOCP) + lvx swap_mask,0,7>) + ld 7,.hidw_mask@got(TOCP) + lvx hidw_mask,0,7 + ld 7,.lodw_mask@got(TOCP) + lvx lodw_mask,0,7 + + vsldoi poly_h,zero,poly,8 + vsldoi poly_l,poly_h,poly_h,8 + + lxvd2x CX,0,X # load X +IF_LE(<vperm C,C,C,swap_mask>) + + srdi 7,LENGTH,7 # 8x loop count + cmpldi 7,0 + beq L2x + + # backup registers + stdu SP,-224(SP) + std 28,216(SP) + std 29,208(SP) + std 30,200(SP) + std 31,192(SP) + li 8,176 + stvx 20,8,SP + subi 8,8,16 + stvx 21,8,SP + subi 8,8,16 + stvx 22,8,SP + subi 8,8,16 + stvx 23,8,SP + subi 8,8,16 + stvx 24,8,SP + subi 8,8,16 + stvx 25,8,SP + subi 8,8,16 + stvx 26,8,SP + subi 8,8,16 + stvx 27,8,SP + subi 8,8,16 + stvx 28,8,SP + subi 8,8,16 + stvx 29,8,SP + subi 8,8,16 + stvx 30,8,SP + subi 8,8,16 + stvx 31,8,SP + + # table loading + li 8,3*TableElemAlign + li 9,4*TableElemAlign + li 10,5*TableElemAlign + lxvd2x H21hX,8,TABLE + lxvd2x H21X,9,TABLE + lxvd2x H21lX,10,TABLE + li 8,6*TableElemAlign + li 9,7*TableElemAlign + li 10,8*TableElemAlign + lxvd2x H43hX,8,TABLE + lxvd2x H43X,9,TABLE + lxvd2x H43lX,10,TABLE + li 8,9*TableElemAlign + li 9,10*TableElemAlign + li 10,11*TableElemAlign + lxvd2x H65hX,8,TABLE + lxvd2x H65X,9,TABLE + lxvd2x H65lX,10,TABLE + li 8,12*TableElemAlign + li 9,13*TableElemAlign + li 10,14*TableElemAlign + lxvd2x H87hX,8,TABLE + lxvd2x H87X,9,TABLE + lxvd2x H87lX,10,TABLE + + li 8,0x10 + li 9,0x20 + li 10,0x30 + li 28,0x40 + li 29,0x50 + li 30,0x60 + li 31,0x70 + + mtctr 7 +.align 5 +L8x_loop: + # input loading + lxvd2x C0X,0,DATA # load C0 + lxvd2x C1X,8,DATA # load C1 + lxvd2x C2X,9,DATA # load C2 + lxvd2x C3X,10,DATA # load C3 + + # swap permuting +IF_LE(<vperm C0,C0,C0,swap_mask + vperm C1,C1,C1,swap_mask + vperm C2,C2,C2,swap_mask + vperm C3,C3,C3,swap_mask>) + + # previous digest combining + vxor C0,C0,C + + # polynomial multiplication "karatsuba" pre-processing + vperm C23h,C2,C3,hidw_mask + vperm C23l,C2,C3,lodw_mask + vperm C01h,C0,C1,hidw_mask + vperm C01l,C0,C1,lodw_mask + + # input loading + lxvd2x C4X,28,DATA # load C4 + lxvd2x C5X,29,DATA # load C5 + lxvd2x C6X,30,DATA # load C6 + lxvd2x C7X,31,DATA # load C7 + + # swap permuting +IF_LE(<vperm C4,C4,C4,swap_mask + vperm C5,C5,C5,swap_mask + vperm C6,C6,C6,swap_mask + vperm C7,C7,C7,swap_mask>) + + # polynomial multiplication "karatsuba" pre-processing + vperm C45h,C4,C5,hidw_mask + vperm C45l,C4,C5,lodw_mask + vperm C67h,C6,C7,hidw_mask + vperm C67l,C6,C7,lodw_mask + vxor C23,C23h,C23l + vxor C01,C01h,C01l + vxor C45,C45h,C45l + vxor C67,C67h,C67l + + # polynomial multiplication "karatsuba" + vpmsumd C23h,C23h,H65h # H23 = H^6h*C2h⊕H^5h*C3h + vpmsumd C23l,C23l,H65l # L23 = H^6l*C2l⊕H^5l*C3l + vpmsumd C01h,C01h,H87h # H01 = H^8h*C0h⊕H^7h*C1h + vpmsumd C01l,C01l,H87l # L01 = H^8l*C0l⊕H^7l*C1l + vpmsumd C67h,C67h,H21h # H67 = H^2h*C6h⊕H^1h*C7h + vpmsumd C67l,C67l,H21l # L67 = H^2l*C6l⊕H^1l*C7l + vpmsumd C45h,C45h,H43h # H45 = H^4h*C4h⊕H^3h*C5h + vpmsumd C45l,C45l,H43l # L45 = H^4l*C4l⊕H^3l*C5l + vpmsumd C23,C23,H65 # M23 = (H^6h⊕H^5h)*(C2h⊕C3h)⊕(H^6l⊕H^5l)*(C2l⊕C3l) + vpmsumd C01,C01,H87 # M01 = (H^8h⊕H^7h)*(C0h⊕C1h)⊕(H^8l⊕H^7l)*(C0l⊕C1l) + vpmsumd C45,C45,H43 # M45 = (H^4h⊕H^3h)*(C4h⊕C5h)⊕(H^4l⊕H^3l)*(C4l⊕C5l) + vpmsumd C67,C67,H21 # M67 = (H^2h⊕H^1h)*(C6h⊕C7h)⊕(H^2l⊕H^1l)*(C6l⊕C7l) + + # polynomial multiplication "karatsuba" post-processing + vxor C23,C23,C23h + vxor C01,C01,C01h + vxor C45,C45,C45h + vxor C67,C67,C67h + vxor C23,C23,C23l + vxor C01,C01,C01l + vxor C45,C45,C45l + vxor C67,C67,C67l + + # deferred recombination of partial products + vxor C01h,C01h,C23h # H0 = H01⊕H23 + vxor C45h,C45h,C67h # H1 = H45⊕H67 + vxor C01l,C01l,C23l # L0 = L01⊕L23 + vxor C45l,C45l,C67l # L1 = L45⊕L45 + vxor C01,C01,C23 # M0 = M01⊕M23 + vxor C45,C45,C67 # M1 = M45⊕M45 + vxor C01h,C01h,C45h # H = H0⊕H1 + vxor C01l,C01l,C45l # L = L0⊕L1 + vxor C01,C01,C45 # M = M0⊕M1 + + # reduction first phase # [1] + vpmsumd RP,C01l,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,C01,8 # [2] + vsldoi Ml,C01,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor C01h,C01h,Mh # [2] + vxor C01l,C01l,Ml # [2] + vxor C01l,C01l,RP # [1] + + # reduction second phase + vpmsumd RP,C01l,poly_l + vxor C01h,C01l,C01h + vxor C,C01h,RP + + addi DATA,DATA,0x80 + bdnz L8x_loop + + # restore registers + li 8,0 + lvx 31,8,SP + addi 8,8,16 + lvx 30,8,SP + addi 8,8,16 + lvx 29,8,SP + addi 8,8,16 + lvx 28,8,SP + addi 8,8,16 + lvx 27,8,SP + addi 8,8,16 + lvx 26,8,SP + addi 8,8,16 + lvx 25,8,SP + addi 8,8,16 + lvx 24,8,SP + addi 8,8,16 + lvx 23,8,SP + addi 8,8,16 + lvx 22,8,SP + addi 8,8,16 + lvx 21,8,SP + addi 8,8,16 + lvx 20,8,SP + ld 31,192(SP) + ld 30,200(SP) + ld 29,208(SP) + ld 28,216(SP) + addi SP,SP,224 + + clrldi LENGTH,LENGTH,57 +L2x: + srdi 7,LENGTH,5 + cmpldi 7,0 + beq L1x + + # table loading + li 8,3*TableElemAlign + li 9,4*TableElemAlign + li 10,5*TableElemAlign + lxvd2x H21hX,8,TABLE + lxvd2x H21X,9,TABLE + lxvd2x H21lX,10,TABLE + + li 10,0x10 + + mtctr 7 +.align 5 +L2x_loop: + # input loading + lxvd2x C0X,0,DATA # load C0 + lxvd2x C1X,10,DATA # load C1 + + # swap permuting +IF_LE(<vperm C0,C0,C0,swap_mask + vperm C1,C1,C1,swap_mask>) + + # previous digest combining + vxor C0,C0,C + + # polynomial multiplication "karatsuba" pre-processing + vperm C01h,C0,C1,hidw_mask + vperm C01l,C0,C1,lodw_mask + vxor C01,C01h,C01l + + # polynomial multiplication "karatsuba" + vpmsumd C01h,C01h,H21h # H01 = H^2h*C0h⊕H^1h*C1h + vpmsumd C01l,C01l,H21l # L01 = H^2l*C0l⊕H^1l*C1l + vpmsumd C01,C01,H21 # M01 = (H^2h⊕H^1h)*(C0h⊕C1h)⊕(H^2l⊕H^1l)*(C0l⊕C1l) + + # polynomial multiplication "karatsuba" post-processing + vxor C01,C01,C01h + vxor C01,C01,C01l + + # reduction first phase # [1] + vpmsumd RP,C01l,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,C01,8 # [2] + vsldoi Ml,C01,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor C01h,C01h,Mh # [2] + vxor C01l,C01l,Ml # [2] + vxor C01l,C01l,RP # [1] + + # reduction second phase + vpmsumd RP,C01l,poly_l + vxor C01h,C01l,C01h + vxor C,C01h,RP + + addi DATA,DATA,0x20 + bdnz L2x_loop + + clrldi LENGTH,LENGTH,59 +L1x: + srdi 7,LENGTH,4 + cmpldi 7,0 + beq Lrem + + # table loading + li 9,1*TableElemAlign + li 10,2*TableElemAlign + lxvd2x HlX,0,TABLE + lxvd2x HX, 9,TABLE + lxvd2x HhX,10,TABLE + + # input loading + lxvd2x C0X,0,DATA # load C0 + + # swap permuting +IF_LE(<vperm C0,C0,C0,swap_mask>) + + # previous digest combining + vxor C0,C0,C + + vpmsumd Cl,C0,Hl # L = Hl*Cl + vpmsumd Cm,C0,H # M = Hh*Cl⊕Hl*Ch + vpmsumd Ch,C0,Hh # H = Hh*Ch + + # reduction first phase # [1] + vpmsumd RP,Cl,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,Cm,8 # [2] + vsldoi Ml,Cm,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor Ch,Ch,Mh # [2] + vxor Cl,Cl,Ml # [2] + vxor Cl,Cl,RP # [1] + + # reduction second phase + vpmsumd RP,Cl,poly_l + vxor Ch,Cl,Ch + vxor C,Ch,RP + + addi DATA,DATA,0x10 + clrldi LENGTH,LENGTH,60 +Lrem: + cmpldi LENGTH,0 + beq Ldone + + # table loading + li 9,1*TableElemAlign + li 10,2*TableElemAlign + lxvd2x HlX,0,TABLE + lxvd2x HX, 9,TABLE + lxvd2x HhX,10,TABLE + + # input loading + stdu SP,-16(SP) + stvx zero,0,SP +Lst_loop: + subic. LENGTH,LENGTH,1 + lbzx 7,LENGTH,DATA + stbx 7,LENGTH,SP + bne Lst_loop + lxvd2x C0X,0,SP + addi SP,SP,16 + + # swap permuting +IF_LE(<vperm C0,C0,C0,swap_mask>) + + # previous digest combining + vxor C0,C0,C + + vpmsumd Cl,C0,Hl # L = Hl*Cl + vpmsumd Cm,C0,H # M = Hh*Cl⊕Hl*Ch + vpmsumd Ch,C0,Hh # H = Hh*Ch + + # reduction first phase # [1] + vpmsumd RP,Cl,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,Cm,8 # [2] + vsldoi Ml,Cm,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor Ch,Ch,Mh # [2] + vxor Cl,Cl,Ml # [2] + vxor Cl,Cl,RP # [1] + + # reduction second phase + vpmsumd RP,Cl,poly_l + vxor Ch,Cl,Ch + vxor C,Ch,RP + +Ldone: +IF_LE(<vperm C,C,C,swap_mask>) + stxvd2x CX,0,X # store C + blr +EPILOGUE(_nettle_gcm_hash8) + + # gcm_fill (uint8_t *ctr, size_t blocks, union gcm_block *buffer) + +PROLOGUE(_nettle_gcm_fill) +IF_LE(<ld 6,.swap_mask@got(TOCP) + lvx swap_mask,0,6>) + + vxor zero,zero,zero + vspltisb I1,1 + vspltisb I2,2 + vspltisb I3,3 + vspltisb I4,4 + vspltisb I5,5 + vspltisb I6,6 + vspltisb I7,7 + vspltisb I8,8 + vsldoi I1,zero,I1,1 + vsldoi I2,zero,I2,1 + vsldoi I3,zero,I3,1 + vsldoi I4,zero,I4,1 + vsldoi I5,zero,I5,1 + vsldoi I6,zero,I6,1 + vsldoi I7,zero,I7,1 + vsldoi I8,zero,I8,1 + + lxvd2x CTR0X,0,CTR + IF_LE(<vperm CTR0,CTR0,CTR0,swap_mask>) + + srdi 6,BLOCKS,3 # 8x loop count + cmpldi 6,0 + beq Lfill_4x + + std 25,-56(SP); + std 26,-48(SP); + std 27,-40(SP); + std 28,-32(SP); + std 29,-24(SP); + std 30,-16(SP); + std 31,-8(SP); + + li 25,0x10 + li 26,0x20 + li 27,0x30 + li 28,0x40 + li 29,0x50 + li 30,0x60 + li 31,0x70 + + mtctr 6 + L8x_fill_loop: + vadduwm CTR1,CTR0,I1 + vadduwm CTR2,CTR0,I2 + vadduwm CTR3,CTR0,I3 + vadduwm CTR4,CTR0,I4 + vadduwm CTR5,CTR0,I5 + vadduwm CTR6,CTR0,I6 + vadduwm CTR7,CTR0,I7 + + IF_LE(<vperm CTR0S,CTR0,CTR0,swap_mask + vperm CTR1,CTR1,CTR1,swap_mask + vperm CTR2,CTR2,CTR2,swap_mask + vperm CTR3,CTR3,CTR3,swap_mask + vperm CTR4,CTR4,CTR4,swap_mask + vperm CTR5,CTR5,CTR5,swap_mask + vperm CTR6,CTR6,CTR6,swap_mask + vperm CTR7,CTR7,CTR7,swap_mask>) + + IF_LE(<stxvd2x CTR0SX,0,BUFFER>) + IF_BE(<stxvd2x CTR0X,0,BUFFER>) + stxvd2x CTR1X,25,BUFFER + stxvd2x CTR2X,26,BUFFER + stxvd2x CTR3X,27,BUFFER + stxvd2x CTR4X,28,BUFFER + stxvd2x CTR5X,29,BUFFER + stxvd2x CTR6X,30,BUFFER + stxvd2x CTR7X,31,BUFFER + + vadduwm CTR0,CTR0,I8 + addi BUFFER,BUFFER,0x80 + bdnz L8x_fill_loop + + ld 25,-56(SP); + ld 26,-48(SP); + ld 27,-40(SP); + ld 28,-32(SP); + ld 29,-24(SP); + ld 30,-16(SP); + ld 31,-8(SP); + + clrldi BLOCKS,BLOCKS,61 + + Lfill_4x: + srdi 6,BLOCKS,2 + cmpldi 6,0 + beq Lfill_2x + + li 8,0x10 + li 9,0x20 + li 10,0x30 + + vadduwm CTR1,CTR0,I1 + vadduwm CTR2,CTR0,I2 + vadduwm CTR3,CTR0,I3 + + IF_LE(<vperm CTR0S,CTR0,CTR0,swap_mask + vperm CTR1,CTR1,CTR1,swap_mask + vperm CTR2,CTR2,CTR2,swap_mask + vperm CTR3,CTR3,CTR3,swap_mask>) + + IF_LE(<stxvd2x CTR0SX,0,BUFFER>) + IF_BE(<stxvd2x CTR0X,0,BUFFER>) + stxvd2x CTR1X,8,BUFFER + stxvd2x CTR2X,9,BUFFER + stxvd2x CTR3X,10,BUFFER + + vadduwm CTR0,CTR0,I4 + addi BUFFER,BUFFER,0x40 + + clrldi BLOCKS,BLOCKS,62 + + Lfill_2x: + srdi 6,BLOCKS,1 + cmpldi 6,0 + beq Lfill_1x + + li 10,0x10 + + vadduwm CTR1,CTR0,I1 + + IF_LE(<vperm CTR0S,CTR0,CTR0,swap_mask + vperm CTR1,CTR1,CTR1,swap_mask>) + + IF_LE(<stxvd2x CTR0SX,0,BUFFER>) + IF_BE(<stxvd2x CTR0X,0,BUFFER>) + stxvd2x CTR1X,10,BUFFER + + vadduwm CTR0,CTR0,I2 + addi BUFFER,BUFFER,0x20 + + clrldi BLOCKS,BLOCKS,63 + + Lfill_1x: + cmpldi BLOCKS,0 + beq Lfill_done + + IF_LE(<vperm CTR0S,CTR0,CTR0,swap_mask>) + + IF_LE(<stxvd2x CTR0SX,0,BUFFER>) + IF_BE(<stxvd2x CTR0X,0,BUFFER>) + + vadduwm CTR0,CTR0,I1 + + Lfill_done: + IF_LE(<vperm CTR0,CTR0,CTR0,swap_mask>) + stxvd2x CTR0X,0,CTR + + blr +EPILOGUE(_nettle_gcm_fill) + + .data +IF_LE(<.align 4 +.polynomial: + .byte 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2 + .align 4 +.swap_mask: + .byte 8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7 + .align 4 +.hidw_mask: + .byte 23,22,21,20,19,18,17,16,7,6,5,4,3,2,1,0 + .align 4 +.lodw_mask: + .byte 31,30,29,28,27,26,25,24,15,14,13,12,11,10,9,8>) +IF_BE(<.align 4 +.polynomial: + .byte 0xc2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 + .align 4 +.hidw_mask: + .byte 0,1,2,3,4,5,6,7,16,17,18,19,20,21,22,23 + .align 4 +.lodw_mask: + .byte 8,9,10,11,12,13,14,15,24,25,26,27,28,29,30,31>) -- 2.17.1

1 0

[PATCH] Add bcrypt tests to testsuite.
by Stephen R. van den Berg 06 Jul '20

06 Jul '20

--- testsuite/blowfish-test.c | 47 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/testsuite/blowfish-test.c b/testsuite/blowfish-test.c index cadeda5f..c495b301 100644 --- a/testsuite/blowfish-test.c +++ b/testsuite/blowfish-test.c @@ -45,6 +45,22 @@ test_blowfish(const struct tstring *key, free(data); } +static void +test_bcrypt(int succeed, const struct tstring *key, + const struct tstring *hash) +{ + if (succeed != blowfish_bcrypt_verify(key->length, key->data, + hash->length, hash->data)) + { + fprintf(stderr, "blowfish_bcrypt_verify failed:\nKey:"); + tstring_print_hex(key); + fprintf(stderr, "\nHash: "); + tstring_print_hex(hash); + fprintf(stderr, "\n"); + FAIL(); + } +} + void test_main(void) { @@ -52,4 +68,35 @@ test_main(void) test_blowfish(SDATA("abcdefghijklmnopqrstuvwxyz"), SDATA("BLOWFISH"), SHEX("32 4E D0 FE F4 13 A2 03")); + /* Tests for BSD-style bcrypt. + From John the Ripper 1.7.9 via Phpass */ + test_bcrypt(1, SDATA("U*U"), SDATA("$2a$05$CCCCCCCCCCCCCCCCCCCCC.E5YPO9kmyuRGyh0XouQYb4YMJKvyOeW")); + test_bcrypt(1, SDATA("U*U*"), SDATA("$2a$05$CCCCCCCCCCCCCCCCCCCCC.VGOzA784oUp/Z0DY336zx7pLYAy0lwK")); + test_bcrypt(1, SDATA("U*U*U"), SDATA("$2a$05$XXXXXXXXXXXXXXXXXXXXXOAcXxm9kjPGEMsLznoKqmqw7tc8WCx4a")); + test_bcrypt(1, SDATA(""), SDATA("$2a$05$CCCCCCCCCCCCCCCCCCCCC.7uG0VCzI2bS7j6ymqJi9CdcdxiRTWNy")); + test_bcrypt(1, SDATA("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789chars after 72 are ignored"), SDATA("$2a$05$abcdefghijklmnopqrstuu5s2v8.iXieOjg/.AySBTTZIIVFJeBui")); + test_bcrypt(1, SDATA("\xa3"), SDATA("$2x$05$/OK.fbVrR/bpIqNJ5ianF.CE5elHaaO4EbggVDjb8P19RukzXSM3e")); + test_bcrypt(1, SDATA("\xa3"), SDATA("$2y$05$/OK.fbVrR/bpIqNJ5ianF.Sa7shbm4.OzKpvFnX1pQLmQW96oUlCq")); + test_bcrypt(1, SDATA("\xd1\x91"), SDATA("$2x$05$6bNw2HLQYeqHYyBfLMsv/OiwqTymGIGzFsA4hOTWebfehXHNprcAS")); + test_bcrypt(1, SDATA("\xd0\xc1\xd2\xcf\xcc\xd8"), SDATA("$2x$05$6bNw2HLQYeqHYyBfLMsv/O9LIGgn8OMzuDoHfof8AQimSGfcSWxnS")); + test_bcrypt(1, SDATA("\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa""chars after 72 are ignored as usual"), SDATA("$2a$05$/OK.fbVrR/bpIqNJ5ianF.swQOIzjOiJ9GHEPuhEkvqrUyvWhEMx6")); + test_bcrypt(1, SDATA("\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55\xaa\x55"), SDATA("$2a$05$/OK.fbVrR/bpIqNJ5ianF.R9xrDjiycxMbQE2bp.vgqlYpW5wx2yy")); + test_bcrypt(1, SDATA(""), SDATA("$2a$05$CCCCCCCCCCCCCCCCCCCCC.7uG0VCzI2bS7j6ymqJi9CdcdxiRTWNy")); + test_bcrypt(1, SDATA("\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff\x55\xaa\xff"), SDATA("$2a$05$/OK.fbVrR/bpIqNJ5ianF.9tQZzcJfm3uj2NvJ/n5xkhpqLrMpWCe")); + /* From Openwall's crypt v1.2 via Phpass */ + test_bcrypt(0, SDATA(""), SDATA("$2a$03$CCCCCCCCCCCCCCCCCCCCC.")); + test_bcrypt(0, SDATA(""), SDATA("$2a$32$CCCCCCCCCCCCCCCCCCCCC.")); + test_bcrypt(0, SDATA(""), SDATA("$2z$05$CCCCCCCCCCCCCCCCCCCCC.")); + test_bcrypt(0, SDATA(""), SDATA("$2`$05$CCCCCCCCCCCCCCCCCCCCC.")); + test_bcrypt(0, SDATA(""), SDATA("$2{$05$CCCCCCCCCCCCCCCCCCCCC.")); + /* Stephen's personal tests */ + test_bcrypt(1, SDATA("yawinpassword"), + SDATA("$2a$04$MzVXtd4o0y4DOlyHMMLMDeE4/eezrsT5Xad.2lmGr/NkCpwBgvn3e")); + test_bcrypt(0, SDATA("xawinpassword"), + SDATA("$2a$04$MzVXtd4o0y4DOlyHMMLMDeE4/eezrsT5Xad.2lmGr/NkCpwBgvn3e")); + test_bcrypt(1, SDATA("Bootq9sH5"), + SDATA("$2y$10$1b2lPgo4XumibnJGN3r3sOsXFfVVYlebFjlw47qpaslC4KIwu9dAK")); + test_bcrypt(0, SDATA("Bootq9sH6"), + SDATA("$2y$10$1b2lPgo4XumibnJGN3r3sOsXFfVVYlebFjlw47qpaslC4KIwu9dAK")); + test_bcrypt(0, SDATA("1234"), SDATA("$2y$")); } -- 2.20.1

2 5

[PATCH 3/3] v4.0 Blowfish: Supply lengths instead of C-strings.
by Stephen R. van den Berg 01 Jul '20

01 Jul '20

--- blowfish-bcrypt.c | 56 ++++++++++++++++++++++++----------------------- blowfish.h | 14 +++++++----- nettle.texinfo | 27 +++++++++++++---------- 3 files changed, 52 insertions(+), 45 deletions(-) diff --git a/blowfish-bcrypt.c b/blowfish-bcrypt.c index c06f9e90..491aab82 100644 --- a/blowfish-bcrypt.c +++ b/blowfish-bcrypt.c @@ -76,13 +76,14 @@ static const char radix64_encode_table[64] = "0123456789"; int -blowfish_bcrypt_verify(const char *key, - const char *hashed) +blowfish_bcrypt_verify(size_t lenkey, const char *key, + size_t lenhashed, const char *hashed) { char newhash[BLOWFISH_BCRYPT_HASH_SIZE]; - return blowfish_bcrypt_hash(sizeof newhash, - newhash, key, hashed, -1, (void*)0) + return blowfish_bcrypt_hash(newhash, + lenkey, key, lenhashed, hashed, + -1, (void*)0) && !strcmp(newhash, hashed); } @@ -159,10 +160,12 @@ static void swap32(uint32_t *x, int count) #endif } -static void set_xkey(const char *key, bf_key expanded, bf_key initial, - unsigned bug, uint32_t safety) +static void set_xkey(size_t lenkey, const char *key, + bf_key expanded, bf_key initial, + unsigned bug, uint32_t safety) { const char *ptr = key; + size_t n = lenkey; unsigned i, j; uint32_t sign, diff, tmp[2]; @@ -219,10 +222,10 @@ static void set_xkey(const char *key, bf_key expanded, bf_key initial, */ if (j) sign |= tmp[1] & 0x80; - if (!*ptr) - ptr = key; - else + if (n--) ptr++; + else + ptr = key, n = lenkey; } diff |= tmp[0] ^ tmp[1]; /* Non-zero on any differences */ @@ -259,8 +262,9 @@ static void set_xkey(const char *key, bf_key expanded, bf_key initial, initial[0] ^= sign; } -static int ibcrypt(size_t length, char *dst, - const char *key, const char *scheme, +static int ibcrypt(char *dst, + size_t lenkey, const char *key, + size_t lenscheme, const char *scheme, int minlog2rounds, int log2rounds, const uint8_t *salt) { @@ -277,12 +281,10 @@ static int ibcrypt(size_t length, char *dst, uint32_t *ptr; uint32_t count; int i; - size_t lenscheme = strlen(scheme); unsigned cscheme; unsigned bug = 0; uint32_t safety = 0; - if (length < BLOWFISH_BCRYPT_HASH_SIZE || - lenscheme < 2) + if (lenscheme < 2) return 0; if (lenscheme >= 3 && *scheme++ != '$') @@ -336,7 +338,7 @@ static int ibcrypt(size_t length, char *dst, return 0; count = (uint32_t)1 << log2rounds; - set_xkey(key, data.expanded_key, data.ctx.p, bug, safety); + set_xkey(lenkey, key, data.expanded_key, data.ctx.p, bug, safety); memcpy(data.ctx.s, _nettle_blowfish_initial_ctx.s, sizeof(data.ctx.s)); L = R = 0; @@ -461,12 +463,13 @@ static int ibcrypt(size_t length, char *dst, * The performance cost of this quick self-test is around 0.6% at the "$2a$08" * setting. */ -int blowfish_bcrypt_hash(size_t length, char *dst, - const char *key, const char *scheme, +int blowfish_bcrypt_hash(char *dst, + size_t lenkey, const char *key, + size_t lenscheme, const char *scheme, int log2rounds, const uint8_t *salt) { - const char *test_pw = "8b \xd0\xc1\xd2\xcf\xcc\xd8"; - const char *test_scheme = "$2a$00$abcdefghijklmnopqrstuu"; + const char test_pw[] = "8b \xd0\xc1\xd2\xcf\xcc\xd8"; + const char test_scheme[] = "$2a$00$abcdefghijklmnopqrstuu"; static const char * const test_hashes[2] = {"i1D709vfamulimlGcq0qq3UvuUasvEa\0\x55", /* 'a', 'b', 'y' */ "VUrPmXD6q/nVSSp7pNDhCR9071IfIRe\0\x55"}; /* 'x' */ @@ -478,10 +481,9 @@ int blowfish_bcrypt_hash(size_t length, char *dst, char o[HASHOFFSET + 31 + 1 + 1 + 1]; } buf; - if (length) - *dst = '\0'; + *dst = '\0'; /* Hash the supplied password */ - cscheme = ibcrypt(length, dst, key, scheme, 4, log2rounds, salt); + cscheme = ibcrypt(dst, lenkey, key, lenscheme, scheme, 4, log2rounds, salt); /* * Do a quick self-test. It is important that we make both calls to ibcrypt() @@ -497,18 +499,18 @@ int blowfish_bcrypt_hash(size_t length, char *dst, memset(buf.o, 0x55, sizeof(buf.o)); buf.o[sizeof(buf.o) - 1] = 0; - ok = ibcrypt(sizeof(buf.o) - (1 + 1), buf.o, test_pw, - buf.s, 0, -1, (void*)0); + ok = ibcrypt(buf.o, sizeof(test_pw), test_pw, + sizeof(buf.s), buf.s, 0, -1, (void*)0); ok = (ok && !memcmp(buf.o, buf.s, HASHOFFSET) && !memcmp(buf.o + HASHOFFSET, test_hash, 31 + 1 + 1 + 1)); { - const char *k = "\xff\xa3" "34" "\xff\xff\xff\xa3" "345"; + const char k[] = "\xff\xa3" "34" "\xff\xff\xff\xa3" "345"; bf_key ae, ai, ye, yi; - set_xkey(k, ae, ai, 0, 0x10000); /* $2a$ */ - set_xkey(k, ye, yi, 0, 0); /* $2y$ */ + set_xkey(sizeof(k), k, ae, ai, 0, 0x10000); /* $2a$ */ + set_xkey(sizeof(k), k, ye, yi, 0, 0); /* $2y$ */ ai[0] ^= 0x10000; /* undo the safety (for comparison) */ ok = ok && ai[0] == 0xdb9c59bc && ye[17] == 0x33343500 && !memcmp(ae, ye, sizeof(ae)) && diff --git a/blowfish.h b/blowfish.h index af48e20f..1c5bdbe1 100644 --- a/blowfish.h +++ b/blowfish.h @@ -86,16 +86,18 @@ void blowfish_decrypt(const struct blowfish_ctx *ctx, size_t length, uint8_t *dst, const uint8_t *src); + +/* dst parameter must point to a buffer of minimally + * BLOWFISH_BCRYPT_HASH_SIZE bytes */ int -blowfish_bcrypt_hash(size_t length, - char *dst, - const char *key, - const char *scheme, +blowfish_bcrypt_hash(char *dst, + size_t lenkey, const char *key, + size_t lenscheme, const char *scheme, int log2rounds, const uint8_t *salt); int -blowfish_bcrypt_verify(const char *key, - const char *hashed); +blowfish_bcrypt_verify(size_t lenkey, const char *key, + size_t lenhashed, const char *hashed); #ifdef __cplusplus } diff --git a/nettle.texinfo b/nettle.texinfo index 75e18b58..2269e11d 100644 --- a/nettle.texinfo +++ b/nettle.texinfo @@ -1513,7 +1513,7 @@ in any other way. Analogous to @code{blowfish_encrypt} @end deftypefun -@deftypefun int blowfish_bcrypt_hash (size_t @var{length}, char *@var{dst}, const char *@var{key}, const char *@var{scheme}, int @var{log2rounds}, const uint8_t *@var{salt}) +@deftypefun int blowfish_bcrypt_hash (char *@var{dst}, size_t @var{lenkey}, const char *@var{key}, size_t @var{lenscheme}, const char *@var{scheme}, int @var{log2rounds}, const uint8_t *@var{salt}) Compute the bcrypt password hash. The function will return @code{0} if the hash cannot be computed due to invalid input. @@ -1522,13 +1522,13 @@ in the array pointed to by @var{dst}. The hash is computed based on the chosen @var{scheme}, number of rounds @var{log2rounds} and specified @var{salt}. -@var{length} must be at least @code{BLOWFISH_BCRYPT_HASH_SIZE}. +@var{dst} must point to a character array of at least + @code{BLOWFISH_BCRYPT_HASH_SIZE} bytes. -@var{dst} must point to a character array of the specified @var{length}. +@var{key} contains the plaintext password string of size @var{lenkey}. -@var{key} contains the zero terminated plaintext password string. - -@var{scheme} contains either just the chosen scheme (valid schemes +@var{scheme} is of size @var{lenscheme} and contains either just the +chosen scheme (valid schemes are: @code{2a}, @code{2b}, @code{2x} or @code{2y}), or (the prefix of) an existing hashed password (typically @code{$2b$10$...}). @@ -1543,26 +1543,28 @@ the salt will be extracted from @var{scheme}. Sample code to generate a bcrypt hash: @example char cleartxtpassword[] = "ExamplePassword"; +char scheme[] = "2b"; uint8_t salt[BLOWFISH_BCRYPT_BINSALT_SIZE]; @dots{} /* Make sure that salt is filled with random bytes */ @dots{} char hashedresult[BLOWFISH_BCRYPT_HASH_SIZE]; -int result = blowfish_bcrypt(sizeof(hashedresult), hashedresult, - cleartxtpassword, "2b", 10, salt); +int result = blowfish_bcrypt(hashedresult, + sizeof(cleartxtpassword), cleartxtpassword, + sizeof(scheme), scheme, 10, salt); if (result) printf("%s\n", hashedresult); @end example @end deftypefun -@deftypefun int blowfish_bcrypt_verify (const char *@var{key}, const char *@var{hashed}) +@deftypefun int blowfish_bcrypt_verify (size_t @var{lenkey}, const char *@var{key}, size_t @var{lenhashed}, const char *@var{hashed}) Verifies the bcrypt password hash against the supplied plaintext password. The function will return @code{0} if the password does not match. The function will return @code{1} if the password matches. -@var{key} contains the zero terminated plaintext password string. +@var{key} contains the plaintext password string of size @var{lenkey}. -@var{hashed} contains the zero terminated hashed string to compare with. +@var{hashed} contains the hashed string of size @var{lenhashed} to compare with. Sample code to verify a bcrypt hash: @example @@ -1573,7 +1575,8 @@ char existinghashed[] = "$" /* separator */ "1b2lPgo4XumibnJGN3r3sO" /* base64 encoded 16-byte salt */ "u7wE7xNfYDKlAxZffJDCJdVfFTAyevu"; /* Hashedpart */ -if (blowfish_bcrypt_verify(cleartxtpassword, existinghashed)) +if (blowfish_bcrypt_verify(sizeof(cleartxtpassword), cleartxtpassword, + sizeof(existinghashed), existinghashed)) printf("Password is correct."); else printf("Password is incorrect."); -- 2.20.1

2 6

[Patch] Optimize AES and GHASH for PowerPC64 (support little-endian and big-endian)
by Maamoun TK 01 Jul '20

01 Jul '20

Patch implementation benchmark for GCM_AES (Tested on POWER8): little-endian: - Encrypt x~17.5 of nettle C implementation - Decrypt x~17.5 of nettle C implementation - Update x~30 of nettle C implementation big-endian: - Encrypt x~18.5 of nettle C implementation - Decrypt x~18.5 of nettle C implementation - Update x~28.5 of nettle C implementation I investigated falling back to process 4 blocks instead of 8 blocks, 8x loop outperform 4x loop by x~1.22 on POWER8. Processing 4 blocks leaves plenty of registers on the table, to reduce the performance margin these registers can be used to overlap the instructions inside the loop. GHASH loop has two separate operations: - Digest calculation - Reduction The reduction procedure is hardly parallelized, to overcome that, the two procedures can overlap as follows |`````````````````````````````````````````| |Head: | |Digest calculation [0..3] | |`````````````````````````````````````````| |Loop: | |Overlapped Reduction [i+0..i+3] and | |Digest calculation [i+4..i+7] | |`````````````````````````````````````````| |Tail: | |Reduction [i+4..i+7] | ``````````````````````````````````````````` With that implemented the performance margin is reduced to x~1.15 and the code is messed up a little bit I can do more documentations so the reader can still keep track of what's going on but with that said the performance margin is still considerable and I think it's better to stick with 8x loop. - The patch passed the testsuite for both ppc64 and ppc64el. --- Makefile.in | 2 +- asm.m4 | 17 +- config.m4.in | 1 + configure.ac | 12 + gcm.c | 19 +- powerpc64/aes-decrypt-internal.asm | 584 +++++++++++++++++++++ powerpc64/aes-encrypt-internal.asm | 545 +++++++++++++++++++ powerpc64/gcm-hash8.asm | 1015 ++++++++++++++++++++++++++++++++++++ powerpc64/machine.m4 | 0 testsuite/gcm-test.c | 23 + 10 files changed, 2214 insertions(+), 4 deletions(-) create mode 100644 powerpc64/aes-decrypt-internal.asm create mode 100644 powerpc64/aes-encrypt-internal.asm create mode 100644 powerpc64/gcm-hash8.asm create mode 100644 powerpc64/machine.m4 diff --git a/Makefile.in b/Makefile.in index 64ff1001..fdc819f1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -603,7 +603,7 @@ distdir: $(DISTFILES) done set -e; for d in sparc32 sparc64 x86 \ x86_64 x86_64/aesni x86_64/sha_ni x86_64/fat \ - arm arm/neon arm/v6 arm/fat ; do \ + arm arm/neon arm/v6 arm/fat powerpc64 ; do \ mkdir "$(distdir)/$$d" ; \ find "$(srcdir)/$$d" -maxdepth 1 '(' -name '*.asm' -o -name '*.m4' ')' \ -exec cp '{}' "$(distdir)/$$d" ';' ; \ diff --git a/asm.m4 b/asm.m4 index 59d64098..1033a326 100644 --- a/asm.m4 +++ b/asm.m4 @@ -30,13 +30,26 @@ COFF_STYLE, yes, define(<GMP_NUMB_BITS>,<>)dnl define(<PROLOGUE>, +<ifelse(ASM_POWERPC64,no, <.globl C_NAME($1) DECLARE_FUNC(C_NAME($1)) -C_NAME($1): ASM_X86_ENDBR>) +C_NAME($1): ASM_X86_ENDBR>, +<.globl C_NAME($1) +DECLARE_FUNC(C_NAME($1)) +.section ".opd","aw" +.align 3 +C_NAME($1): +.quad .C_NAME($1),.TOC.@tocbase,0 +.previous +.align 5 +.C_NAME($1):>)>) define(<EPILOGUE>, <ifelse(ELF_STYLE,yes, -<.size C_NAME($1), . - C_NAME($1)>,<>)>) +<ifelse(ASM_POWERPC64,no, +<.size C_NAME($1), . - C_NAME($1)>, +<.size .C_NAME($1), . - .C_NAME($1) +.size C_NAME($1), . - .C_NAME($1)>)>,<>)>) define(<m4_log2>, <m4_log2_internal($1,1,0)>) define(<m4_log2_internal>, diff --git a/config.m4.in b/config.m4.in index f7f5f283..6265a679 100644 --- a/config.m4.in +++ b/config.m4.in @@ -10,6 +10,7 @@ define(<RODATA>, <@ASM_RODATA@>)dnl define(<WORDS_BIGENDIAN>, <@ASM_WORDS_BIGENDIAN@>)dnl define(<ASM_X86_ENDBR>,<@ASM_X86_ENDBR@>)dnl define(<ASM_X86_MARK_CET_ALIGN>,<@ASM_X86_MARK_CET_ALIGN@>)dnl +define(<ASM_POWERPC64>,<@ASM_POWERPC64@>)dnl divert(1) @ASM_X86_MARK_CET@ @ASM_MARK_NOEXEC_STACK@ diff --git a/configure.ac b/configure.ac index 90ea1ea8..d7baface 100644 --- a/configure.ac +++ b/configure.ac @@ -435,6 +435,9 @@ if test "x$enable_assembler" = xyes ; then esac fi ;; + *powerpc64*) + asm_path=powerpc64 + ;; *) enable_assembler=no ;; @@ -572,7 +575,9 @@ AH_VERBATIM([HAVE_NATIVE], #undef HAVE_NATIVE_ecc_secp384r1_redc #undef HAVE_NATIVE_ecc_secp521r1_modp #undef HAVE_NATIVE_ecc_secp521r1_redc +#undef HAVE_NATIVE_gcm_init_key8 #undef HAVE_NATIVE_gcm_hash8 +#undef HAVE_NATIVE_gcm_fill #undef HAVE_NATIVE_salsa20_core #undef HAVE_NATIVE_sha1_compress #undef HAVE_NATIVE_sha256_compress @@ -866,6 +871,12 @@ if test "$nettle_cv_asm_x86_gnu_property" = yes; then .popsection' fi +if test "$host_cpu" = "powerpc64" ; then + ASM_POWERPC64='yes' +else + ASM_POWERPC64='no' +fi + AC_SUBST(ASM_SYMBOL_PREFIX) AC_SUBST(ASM_ELF_STYLE) AC_SUBST(ASM_COFF_STYLE) @@ -879,6 +890,7 @@ AC_SUBST(EMULATOR) AC_SUBST(ASM_X86_ENDBR) AC_SUBST(ASM_X86_MARK_CET) AC_SUBST(ASM_X86_MARK_CET_ALIGN) +AC_SUBST(ASM_POWERPC64) AC_SUBST(LIBNETTLE_MAJOR) AC_SUBST(LIBNETTLE_MINOR) diff --git a/gcm.c b/gcm.c index cf615daf..809c03bc 100644 --- a/gcm.c +++ b/gcm.c @@ -140,6 +140,12 @@ gcm_gf_mul (union nettle_block16 *x, const union nettle_block16 *table) memcpy (x->b, Z.b, sizeof(Z)); } # elif GCM_TABLE_BITS == 8 +# if HAVE_NATIVE_gcm_init_key8 + +#define gcm_init_key _nettle_gcm_init_key8 +void +_nettle_gcm_init_key8 (union nettle_block16 *table); +# endif /* HAVE_NATIVE_gcm_init_key8 */ # if HAVE_NATIVE_gcm_hash8 #define gcm_hash _nettle_gcm_hash8 @@ -225,6 +231,13 @@ gcm_gf_mul (union nettle_block16 *x, const union nettle_block16 *table) #endif /* GCM_TABLE_BITS */ +#if HAVE_NATIVE_gcm_fill + +#define gcm_fill _nettle_gcm_fill +void +_nettle_gcm_fill (uint8_t *ctr, size_t blocks, union nettle_block16 *buffer); +#endif /* HAVE_NATIVE_gcm_fill */ + /* Increment the rightmost 32 bits. */ #define INC32(block) INCREMENT(4, (block.b) + GCM_BLOCK_SIZE - 4) @@ -245,7 +258,9 @@ gcm_set_key(struct gcm_key *key, memset(key->h[0].b, 0, GCM_BLOCK_SIZE); f (cipher, GCM_BLOCK_SIZE, key->h[i].b, key->h[0].b); -#if GCM_TABLE_BITS +#ifdef gcm_init_key + gcm_init_key(key->h); +#elif GCM_TABLE_BITS /* Algorithm 3 from the gcm paper. First do powers of two, then do the rest by adding. */ while (i /= 2) @@ -333,6 +348,7 @@ gcm_update(struct gcm_ctx *ctx, const struct gcm_key *key, ctx->auth_size += length; } +#ifndef gcm_fill static nettle_fill16_func gcm_fill; static void gcm_fill(uint8_t *ctr, size_t blocks, union nettle_block16 *buffer) @@ -349,6 +365,7 @@ gcm_fill(uint8_t *ctr, size_t blocks, union nettle_block16 *buffer) WRITE_UINT32(ctr + GCM_BLOCK_SIZE - 4, c); } +#endif /* !gcm_fill */ void gcm_encrypt (struct gcm_ctx *ctx, const struct gcm_key *key, diff --git a/powerpc64/aes-decrypt-internal.asm b/powerpc64/aes-decrypt-internal.asm new file mode 100644 index 00000000..70fd69f0 --- /dev/null +++ b/powerpc64/aes-decrypt-internal.asm @@ -0,0 +1,584 @@ +C powerpc64/aes-decrypt-internal.asm + +ifelse(< + Copyright (C) 2020 Mamone Tarsha + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +>) + +C Register usage: + +define(<SP>, <1>) +define(<TOCP>, <2>) + +define(<ROUNDS>, <3>) +define(<KEYS>, <4>) +define(<LENGTH>, <6>) +define(<DST>, <7>) +define(<SRC>, <8>) + +define(<swap_mask>, <0>) + +define(<K>, <1>) +define(<S0>, <2>) +define(<S1>, <3>) +define(<S2>, <4>) +define(<S3>, <5>) +define(<S4>, <6>) +define(<S5>, <7>) +define(<S6>, <8>) +define(<S7>, <9>) +define(<S8>, <10>) +define(<S9>, <11>) +define(<S10>, <12>) +define(<S11>, <13>) +define(<S12>, <14>) +define(<S13>, <15>) +define(<S14>, <16>) +define(<S15>, <17>) + +define(<KX>, <33>) +define(<S0X>, <34>) +define(<S1X>, <35>) +define(<S2X>, <36>) +define(<S3X>, <37>) +define(<S4X>, <38>) +define(<S5X>, <39>) +define(<S6X>, <40>) +define(<S7X>, <41>) +define(<S8X>, <42>) +define(<S9X>, <43>) +define(<S10X>, <44>) +define(<S11X>, <45>) +define(<S12X>, <46>) +define(<S13X>, <47>) +define(<S14X>, <48>) +define(<S15X>, <49>) + +C ZERO vector register is used in place of RoundKey +C for vncipher instruction because the order of InvMixColumns +C and Xor processes are flipped in that instruction. +C The Xor process with RoundKey is executed afterward. +define(<ZERO>, <18>) + +.file "aes-decrypt-internal.asm" + +.machine "any" + +IF_LE(<.abiversion 2>) +.text + + C _aes_decrypt(unsigned rounds, const uint32_t *keys, + C const struct aes_table *T, + C size_t length, uint8_t *dst, + C uint8_t *src) + +IF_LE(<.align 5>) +PROLOGUE(_nettle_aes_decrypt) +IF_LE(<.localentry _nettle_aes_decrypt,0>) + + vxor ZERO,ZERO,ZERO + + ld 5,.swap_mask@got(TOCP) + lvx swap_mask,0,5 + + subi ROUNDS,ROUNDS,1 + srdi LENGTH,LENGTH,4 + + srdi 5,LENGTH,4 #16x loop count + cmpldi 5,0 + beq L8x + + std 17,-120(SP); + std 18,-112(SP); + std 19,-104(SP); + std 20,-96(SP); + std 21,-88(SP); + std 22,-80(SP); + std 23,-72(SP); + std 24,-64(SP); + std 25,-56(SP); + std 26,-48(SP); + std 27,-40(SP); + std 28,-32(SP); + std 29,-24(SP); + std 30,-16(SP); + std 31,-8(SP); + + li 17,0x10 + li 18,0x20 + li 19,0x30 + li 20,0x40 + li 21,0x50 + li 22,0x60 + li 23,0x70 + li 24,0x80 + li 25,0x90 + li 26,0xA0 + li 27,0xB0 + li 28,0xC0 + li 29,0xD0 + li 30,0xE0 + li 31,0xF0 + +.align 5 +Lx16_loop: + lxvd2x KX,0,KEYS + vperm K,K,K,swap_mask + + lxvd2x S0X,0,SRC + lxvd2x S1X,17,SRC + lxvd2x S2X,18,SRC + lxvd2x S3X,19,SRC + lxvd2x S4X,20,SRC + lxvd2x S5X,21,SRC + lxvd2x S6X,22,SRC + lxvd2x S7X,23,SRC + lxvd2x S8X,24,SRC + lxvd2x S9X,25,SRC + lxvd2x S10X,26,SRC + lxvd2x S11X,27,SRC + lxvd2x S12X,28,SRC + lxvd2x S13X,29,SRC + lxvd2x S14X,30,SRC + lxvd2x S15X,31,SRC + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask + vperm S2,S2,S2,swap_mask + vperm S3,S3,S3,swap_mask + vperm S4,S4,S4,swap_mask + vperm S5,S5,S5,swap_mask + vperm S6,S6,S6,swap_mask + vperm S7,S7,S7,swap_mask + vperm S8,S8,S8,swap_mask + vperm S9,S9,S9,swap_mask + vperm S10,S10,S10,swap_mask + vperm S11,S11,S11,swap_mask + vperm S12,S12,S12,swap_mask + vperm S13,S13,S13,swap_mask + vperm S14,S14,S14,swap_mask + vperm S15,S15,S15,swap_mask>) + + vxor S0,S0,K + vxor S1,S1,K + vxor S2,S2,K + vxor S3,S3,K + vxor S4,S4,K + vxor S5,S5,K + vxor S6,S6,K + vxor S7,S7,K + vxor S8,S8,K + vxor S9,S9,K + vxor S10,S10,K + vxor S11,S11,K + vxor S12,S12,K + vxor S13,S13,K + vxor S14,S14,K + vxor S15,S15,K + + mtctr ROUNDS + li 10,0x10 +.align 5 +L16x_round_loop: + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vncipher S0,S0,ZERO + vncipher S1,S1,ZERO + vncipher S2,S2,ZERO + vncipher S3,S3,ZERO + vncipher S4,S4,ZERO + vncipher S5,S5,ZERO + vncipher S6,S6,ZERO + vncipher S7,S7,ZERO + vncipher S8,S8,ZERO + vncipher S9,S9,ZERO + vncipher S10,S10,ZERO + vncipher S11,S11,ZERO + vncipher S12,S12,ZERO + vncipher S13,S13,ZERO + vncipher S14,S14,ZERO + vncipher S15,S15,ZERO + vxor S0,S0,K + vxor S1,S1,K + vxor S2,S2,K + vxor S3,S3,K + vxor S4,S4,K + vxor S5,S5,K + vxor S6,S6,K + vxor S7,S7,K + vxor S8,S8,K + vxor S9,S9,K + vxor S10,S10,K + vxor S11,S11,K + vxor S12,S12,K + vxor S13,S13,K + vxor S14,S14,K + vxor S15,S15,K + addi 10,10,0x10 + bdnz L16x_round_loop + + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vncipherlast S0,S0,K + vncipherlast S1,S1,K + vncipherlast S2,S2,K + vncipherlast S3,S3,K + vncipherlast S4,S4,K + vncipherlast S5,S5,K + vncipherlast S6,S6,K + vncipherlast S7,S7,K + vncipherlast S8,S8,K + vncipherlast S9,S9,K + vncipherlast S10,S10,K + vncipherlast S11,S11,K + vncipherlast S12,S12,K + vncipherlast S13,S13,K + vncipherlast S14,S14,K + vncipherlast S15,S15,K + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask + vperm S2,S2,S2,swap_mask + vperm S3,S3,S3,swap_mask + vperm S4,S4,S4,swap_mask + vperm S5,S5,S5,swap_mask + vperm S6,S6,S6,swap_mask + vperm S7,S7,S7,swap_mask + vperm S8,S8,S8,swap_mask + vperm S9,S9,S9,swap_mask + vperm S10,S10,S10,swap_mask + vperm S11,S11,S11,swap_mask + vperm S12,S12,S12,swap_mask + vperm S13,S13,S13,swap_mask + vperm S14,S14,S14,swap_mask + vperm S15,S15,S15,swap_mask>) + + stxvd2x S0X,0,DST + stxvd2x S1X,17,DST + stxvd2x S2X,18,DST + stxvd2x S3X,19,DST + stxvd2x S4X,20,DST + stxvd2x S5X,21,DST + stxvd2x S6X,22,DST + stxvd2x S7X,23,DST + stxvd2x S8X,24,DST + stxvd2x S9X,25,DST + stxvd2x S10X,26,DST + stxvd2x S11X,27,DST + stxvd2x S12X,28,DST + stxvd2x S13X,29,DST + stxvd2x S14X,30,DST + stxvd2x S15X,31,DST + + addi SRC,SRC,0x100 + addi DST,DST,0x100 + subic. 5,5,1 + bne Lx16_loop + + ld 17,-120(SP); + ld 18,-112(SP); + ld 19,-104(SP); + ld 20,-96(SP); + ld 21,-88(SP); + ld 22,-80(SP); + ld 23,-72(SP); + ld 24,-64(SP); + ld 25,-56(SP); + ld 26,-48(SP); + ld 27,-40(SP); + ld 28,-32(SP); + ld 29,-24(SP); + ld 30,-16(SP); + ld 31,-8(SP); + + clrldi LENGTH,LENGTH,60 + +L8x: + srdi 5,LENGTH,3 + cmpldi 5,0 + beq L4x + + lxvd2x KX,0,KEYS + vperm K,K,K,swap_mask + + lxvd2x S0X,0,SRC + li 9,0x10 + lxvd2x S1X,9,SRC + addi 9,9,0x10 + lxvd2x S2X,9,SRC + addi 9,9,0x10 + lxvd2x S3X,9,SRC + addi 9,9,0x10 + lxvd2x S4X,9,SRC + addi 9,9,0x10 + lxvd2x S5X,9,SRC + addi 9,9,0x10 + lxvd2x S6X,9,SRC + addi 9,9,0x10 + lxvd2x S7X,9,SRC + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask + vperm S2,S2,S2,swap_mask + vperm S3,S3,S3,swap_mask + vperm S4,S4,S4,swap_mask + vperm S5,S5,S5,swap_mask + vperm S6,S6,S6,swap_mask + vperm S7,S7,S7,swap_mask>) + + vxor S0,S0,K + vxor S1,S1,K + vxor S2,S2,K + vxor S3,S3,K + vxor S4,S4,K + vxor S5,S5,K + vxor S6,S6,K + vxor S7,S7,K + + mtctr ROUNDS + li 10,0x10 +.align 5 +L8x_round_loop: + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vncipher S0,S0,ZERO + vncipher S1,S1,ZERO + vncipher S2,S2,ZERO + vncipher S3,S3,ZERO + vncipher S4,S4,ZERO + vncipher S5,S5,ZERO + vncipher S6,S6,ZERO + vncipher S7,S7,ZERO + vxor S0,S0,K + vxor S1,S1,K + vxor S2,S2,K + vxor S3,S3,K + vxor S4,S4,K + vxor S5,S5,K + vxor S6,S6,K + vxor S7,S7,K + addi 10,10,0x10 + bdnz L8x_round_loop + + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vncipherlast S0,S0,K + vncipherlast S1,S1,K + vncipherlast S2,S2,K + vncipherlast S3,S3,K + vncipherlast S4,S4,K + vncipherlast S5,S5,K + vncipherlast S6,S6,K + vncipherlast S7,S7,K + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask + vperm S2,S2,S2,swap_mask + vperm S3,S3,S3,swap_mask + vperm S4,S4,S4,swap_mask + vperm S5,S5,S5,swap_mask + vperm S6,S6,S6,swap_mask + vperm S7,S7,S7,swap_mask>) + + stxvd2x S0X,0,DST + li 9,0x10 + stxvd2x S1X,9,DST + addi 9,9,0x10 + stxvd2x S2X,9,DST + addi 9,9,0x10 + stxvd2x S3X,9,DST + addi 9,9,0x10 + stxvd2x S4X,9,DST + addi 9,9,0x10 + stxvd2x S5X,9,DST + addi 9,9,0x10 + stxvd2x S6X,9,DST + addi 9,9,0x10 + stxvd2x S7X,9,DST + + addi SRC,SRC,0x80 + addi DST,DST,0x80 + + clrldi LENGTH,LENGTH,61 + +L4x: + srdi 5,LENGTH,2 + cmpldi 5,0 + beq L2x + + lxvd2x KX,0,KEYS + vperm K,K,K,swap_mask + + lxvd2x S0X,0,SRC + li 9,0x10 + lxvd2x S1X,9,SRC + addi 9,9,0x10 + lxvd2x S2X,9,SRC + addi 9,9,0x10 + lxvd2x S3X,9,SRC + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask + vperm S2,S2,S2,swap_mask + vperm S3,S3,S3,swap_mask>) + + vxor S0,S0,K + vxor S1,S1,K + vxor S2,S2,K + vxor S3,S3,K + + mtctr ROUNDS + li 10,0x10 +.align 5 +L4x_round_loop: + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vncipher S0,S0,ZERO + vncipher S1,S1,ZERO + vncipher S2,S2,ZERO + vncipher S3,S3,ZERO + vxor S0,S0,K + vxor S1,S1,K + vxor S2,S2,K + vxor S3,S3,K + addi 10,10,0x10 + bdnz L4x_round_loop + + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vncipherlast S0,S0,K + vncipherlast S1,S1,K + vncipherlast S2,S2,K + vncipherlast S3,S3,K + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask + vperm S2,S2,S2,swap_mask + vperm S3,S3,S3,swap_mask>) + + stxvd2x S0X,0,DST + li 9,0x10 + stxvd2x S1X,9,DST + addi 9,9,0x10 + stxvd2x S2X,9,DST + addi 9,9,0x10 + stxvd2x S3X,9,DST + + addi SRC,SRC,0x40 + addi DST,DST,0x40 + + clrldi LENGTH,LENGTH,62 + +L2x: + srdi 5,LENGTH,1 + cmpldi 5,0 + beq L1x + + lxvd2x KX,0,KEYS + vperm K,K,K,swap_mask + + lxvd2x S0X,0,SRC + li 9,0x10 + lxvd2x S1X,9,SRC + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask>) + + vxor S0,S0,K + vxor S1,S1,K + + mtctr ROUNDS + li 10,0x10 +.align 5 +L2x_round_loop: + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vncipher S0,S0,ZERO + vncipher S1,S1,ZERO + vxor S0,S0,K + vxor S1,S1,K + addi 10,10,0x10 + bdnz L2x_round_loop + + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vncipherlast S0,S0,K + vncipherlast S1,S1,K + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask>) + + stxvd2x S0X,0,DST + li 9,0x10 + stxvd2x S1X,9,DST + + addi SRC,SRC,0x20 + addi DST,DST,0x20 + + clrldi LENGTH,LENGTH,63 + +L1x: + cmpldi LENGTH,0 + beq Ldone + + lxvd2x KX,0,KEYS + vperm K,K,K,swap_mask + + lxvd2x S0X,0,SRC + +IF_LE(<vperm S0,S0,S0,swap_mask>) + + vxor S0,S0,K + + mtctr ROUNDS + li 10,0x10 +.align 5 +L1x_round_loop: + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vncipher S0,S0,ZERO + vxor S0,S0,K + addi 10,10,0x10 + bdnz L1x_round_loop + + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vncipherlast S0,S0,K + +IF_LE(<vperm S0,S0,S0,swap_mask>) + + stxvd2x S0X,0,DST + +Ldone: + blr +EPILOGUE(_nettle_aes_decrypt) + + .data + .align 4 +.swap_mask: +IF_LE(<.byte 8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7>) +IF_BE(<.byte 3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12>) diff --git a/powerpc64/aes-encrypt-internal.asm b/powerpc64/aes-encrypt-internal.asm new file mode 100644 index 00000000..20711598 --- /dev/null +++ b/powerpc64/aes-encrypt-internal.asm @@ -0,0 +1,545 @@ +C powerpc64/aes-encrypt-internal.asm + +ifelse(< + Copyright (C) 2020 Mamone Tarsha + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +>) + +C Register usage: + +define(<SP>, <1>) +define(<TOCP>, <2>) + +define(<ROUNDS>, <3>) +define(<KEYS>, <4>) +define(<LENGTH>, <6>) +define(<DST>, <7>) +define(<SRC>, <8>) + +define(<swap_mask>, <0>) + +define(<K>, <1>) +define(<S0>, <2>) +define(<S1>, <3>) +define(<S2>, <4>) +define(<S3>, <5>) +define(<S4>, <6>) +define(<S5>, <7>) +define(<S6>, <8>) +define(<S7>, <9>) +define(<S8>, <10>) +define(<S9>, <11>) +define(<S10>, <12>) +define(<S11>, <13>) +define(<S12>, <14>) +define(<S13>, <15>) +define(<S14>, <16>) +define(<S15>, <17>) + +define(<KX>, <33>) +define(<S0X>, <34>) +define(<S1X>, <35>) +define(<S2X>, <36>) +define(<S3X>, <37>) +define(<S4X>, <38>) +define(<S5X>, <39>) +define(<S6X>, <40>) +define(<S7X>, <41>) +define(<S8X>, <42>) +define(<S9X>, <43>) +define(<S10X>, <44>) +define(<S11X>, <45>) +define(<S12X>, <46>) +define(<S13X>, <47>) +define(<S14X>, <48>) +define(<S15X>, <49>) + +.file "aes-encrypt-internal.asm" + +.machine "any" + +IF_LE(<.abiversion 2>) +.text + + C _aes_encrypt(unsigned rounds, const uint32_t *keys, + C const struct aes_table *T, + C size_t length, uint8_t *dst, + C uint8_t *src) + +IF_LE(<.align 5>) +PROLOGUE(_nettle_aes_encrypt) +IF_LE(<.localentry _nettle_aes_encrypt,0>) + + ld 5,.swap_mask@got(TOCP) + lvx swap_mask,0,5 + + subi ROUNDS,ROUNDS,1 + srdi LENGTH,LENGTH,4 + + srdi 5,LENGTH,4 #16x loop count + cmpldi 5,0 + beq L8x + + std 17,-120(SP); + std 18,-112(SP); + std 19,-104(SP); + std 20,-96(SP); + std 21,-88(SP); + std 22,-80(SP); + std 23,-72(SP); + std 24,-64(SP); + std 25,-56(SP); + std 26,-48(SP); + std 27,-40(SP); + std 28,-32(SP); + std 29,-24(SP); + std 30,-16(SP); + std 31,-8(SP); + + li 17,0x10 + li 18,0x20 + li 19,0x30 + li 20,0x40 + li 21,0x50 + li 22,0x60 + li 23,0x70 + li 24,0x80 + li 25,0x90 + li 26,0xA0 + li 27,0xB0 + li 28,0xC0 + li 29,0xD0 + li 30,0xE0 + li 31,0xF0 + +.align 5 +Lx16_loop: + lxvd2x KX,0,KEYS + vperm K,K,K,swap_mask + + lxvd2x S0X,0,SRC + lxvd2x S1X,17,SRC + lxvd2x S2X,18,SRC + lxvd2x S3X,19,SRC + lxvd2x S4X,20,SRC + lxvd2x S5X,21,SRC + lxvd2x S6X,22,SRC + lxvd2x S7X,23,SRC + lxvd2x S8X,24,SRC + lxvd2x S9X,25,SRC + lxvd2x S10X,26,SRC + lxvd2x S11X,27,SRC + lxvd2x S12X,28,SRC + lxvd2x S13X,29,SRC + lxvd2x S14X,30,SRC + lxvd2x S15X,31,SRC + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask + vperm S2,S2,S2,swap_mask + vperm S3,S3,S3,swap_mask + vperm S4,S4,S4,swap_mask + vperm S5,S5,S5,swap_mask + vperm S6,S6,S6,swap_mask + vperm S7,S7,S7,swap_mask + vperm S8,S8,S8,swap_mask + vperm S9,S9,S9,swap_mask + vperm S10,S10,S10,swap_mask + vperm S11,S11,S11,swap_mask + vperm S12,S12,S12,swap_mask + vperm S13,S13,S13,swap_mask + vperm S14,S14,S14,swap_mask + vperm S15,S15,S15,swap_mask>) + + vxor S0,S0,K + vxor S1,S1,K + vxor S2,S2,K + vxor S3,S3,K + vxor S4,S4,K + vxor S5,S5,K + vxor S6,S6,K + vxor S7,S7,K + vxor S8,S8,K + vxor S9,S9,K + vxor S10,S10,K + vxor S11,S11,K + vxor S12,S12,K + vxor S13,S13,K + vxor S14,S14,K + vxor S15,S15,K + + mtctr ROUNDS + li 10,0x10 +.align 5 +L16x_round_loop: + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vcipher S0,S0,K + vcipher S1,S1,K + vcipher S2,S2,K + vcipher S3,S3,K + vcipher S4,S4,K + vcipher S5,S5,K + vcipher S6,S6,K + vcipher S7,S7,K + vcipher S8,S8,K + vcipher S9,S9,K + vcipher S10,S10,K + vcipher S11,S11,K + vcipher S12,S12,K + vcipher S13,S13,K + vcipher S14,S14,K + vcipher S15,S15,K + addi 10,10,0x10 + bdnz L16x_round_loop + + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vcipherlast S0,S0,K + vcipherlast S1,S1,K + vcipherlast S2,S2,K + vcipherlast S3,S3,K + vcipherlast S4,S4,K + vcipherlast S5,S5,K + vcipherlast S6,S6,K + vcipherlast S7,S7,K + vcipherlast S8,S8,K + vcipherlast S9,S9,K + vcipherlast S10,S10,K + vcipherlast S11,S11,K + vcipherlast S12,S12,K + vcipherlast S13,S13,K + vcipherlast S14,S14,K + vcipherlast S15,S15,K + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask + vperm S2,S2,S2,swap_mask + vperm S3,S3,S3,swap_mask + vperm S4,S4,S4,swap_mask + vperm S5,S5,S5,swap_mask + vperm S6,S6,S6,swap_mask + vperm S7,S7,S7,swap_mask + vperm S8,S8,S8,swap_mask + vperm S9,S9,S9,swap_mask + vperm S10,S10,S10,swap_mask + vperm S11,S11,S11,swap_mask + vperm S12,S12,S12,swap_mask + vperm S13,S13,S13,swap_mask + vperm S14,S14,S14,swap_mask + vperm S15,S15,S15,swap_mask>) + + stxvd2x S0X,0,DST + stxvd2x S1X,17,DST + stxvd2x S2X,18,DST + stxvd2x S3X,19,DST + stxvd2x S4X,20,DST + stxvd2x S5X,21,DST + stxvd2x S6X,22,DST + stxvd2x S7X,23,DST + stxvd2x S8X,24,DST + stxvd2x S9X,25,DST + stxvd2x S10X,26,DST + stxvd2x S11X,27,DST + stxvd2x S12X,28,DST + stxvd2x S13X,29,DST + stxvd2x S14X,30,DST + stxvd2x S15X,31,DST + + addi SRC,SRC,0x100 + addi DST,DST,0x100 + subic. 5,5,1 + bne Lx16_loop + + ld 17,-120(SP); + ld 18,-112(SP); + ld 19,-104(SP); + ld 20,-96(SP); + ld 21,-88(SP); + ld 22,-80(SP); + ld 23,-72(SP); + ld 24,-64(SP); + ld 25,-56(SP); + ld 26,-48(SP); + ld 27,-40(SP); + ld 28,-32(SP); + ld 29,-24(SP); + ld 30,-16(SP); + ld 31,-8(SP); + + clrldi LENGTH,LENGTH,60 + +L8x: + srdi 5,LENGTH,3 + cmpldi 5,0 + beq L4x + + lxvd2x KX,0,KEYS + vperm K,K,K,swap_mask + + lxvd2x S0X,0,SRC + li 9,0x10 + lxvd2x S1X,9,SRC + addi 9,9,0x10 + lxvd2x S2X,9,SRC + addi 9,9,0x10 + lxvd2x S3X,9,SRC + addi 9,9,0x10 + lxvd2x S4X,9,SRC + addi 9,9,0x10 + lxvd2x S5X,9,SRC + addi 9,9,0x10 + lxvd2x S6X,9,SRC + addi 9,9,0x10 + lxvd2x S7X,9,SRC + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask + vperm S2,S2,S2,swap_mask + vperm S3,S3,S3,swap_mask + vperm S4,S4,S4,swap_mask + vperm S5,S5,S5,swap_mask + vperm S6,S6,S6,swap_mask + vperm S7,S7,S7,swap_mask>) + + vxor S0,S0,K + vxor S1,S1,K + vxor S2,S2,K + vxor S3,S3,K + vxor S4,S4,K + vxor S5,S5,K + vxor S6,S6,K + vxor S7,S7,K + + mtctr ROUNDS + li 10,0x10 +.align 5 +L8x_round_loop: + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vcipher S0,S0,K + vcipher S1,S1,K + vcipher S2,S2,K + vcipher S3,S3,K + vcipher S4,S4,K + vcipher S5,S5,K + vcipher S6,S6,K + vcipher S7,S7,K + addi 10,10,0x10 + bdnz L8x_round_loop + + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vcipherlast S0,S0,K + vcipherlast S1,S1,K + vcipherlast S2,S2,K + vcipherlast S3,S3,K + vcipherlast S4,S4,K + vcipherlast S5,S5,K + vcipherlast S6,S6,K + vcipherlast S7,S7,K + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask + vperm S2,S2,S2,swap_mask + vperm S3,S3,S3,swap_mask + vperm S4,S4,S4,swap_mask + vperm S5,S5,S5,swap_mask + vperm S6,S6,S6,swap_mask + vperm S7,S7,S7,swap_mask>) + + stxvd2x S0X,0,DST + li 9,0x10 + stxvd2x S1X,9,DST + addi 9,9,0x10 + stxvd2x S2X,9,DST + addi 9,9,0x10 + stxvd2x S3X,9,DST + addi 9,9,0x10 + stxvd2x S4X,9,DST + addi 9,9,0x10 + stxvd2x S5X,9,DST + addi 9,9,0x10 + stxvd2x S6X,9,DST + addi 9,9,0x10 + stxvd2x S7X,9,DST + + addi SRC,SRC,0x80 + addi DST,DST,0x80 + + clrldi LENGTH,LENGTH,61 + +L4x: + srdi 5,LENGTH,2 + cmpldi 5,0 + beq L2x + + lxvd2x KX,0,KEYS + vperm K,K,K,swap_mask + + lxvd2x S0X,0,SRC + li 9,0x10 + lxvd2x S1X,9,SRC + addi 9,9,0x10 + lxvd2x S2X,9,SRC + addi 9,9,0x10 + lxvd2x S3X,9,SRC + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask + vperm S2,S2,S2,swap_mask + vperm S3,S3,S3,swap_mask>) + + vxor S0,S0,K + vxor S1,S1,K + vxor S2,S2,K + vxor S3,S3,K + + mtctr ROUNDS + li 10,0x10 +.align 5 +L4x_round_loop: + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vcipher S0,S0,K + vcipher S1,S1,K + vcipher S2,S2,K + vcipher S3,S3,K + addi 10,10,0x10 + bdnz L4x_round_loop + + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vcipherlast S0,S0,K + vcipherlast S1,S1,K + vcipherlast S2,S2,K + vcipherlast S3,S3,K + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask + vperm S2,S2,S2,swap_mask + vperm S3,S3,S3,swap_mask>) + + stxvd2x S0X,0,DST + li 9,0x10 + stxvd2x S1X,9,DST + addi 9,9,0x10 + stxvd2x S2X,9,DST + addi 9,9,0x10 + stxvd2x S3X,9,DST + + addi SRC,SRC,0x40 + addi DST,DST,0x40 + + clrldi LENGTH,LENGTH,62 + +L2x: + srdi 5,LENGTH,1 + cmpldi 5,0 + beq L1x + + lxvd2x KX,0,KEYS + vperm K,K,K,swap_mask + + lxvd2x S0X,0,SRC + li 9,0x10 + lxvd2x S1X,9,SRC + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask>) + + vxor S0,S0,K + vxor S1,S1,K + + mtctr ROUNDS + li 10,0x10 +.align 5 +L2x_round_loop: + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vcipher S0,S0,K + vcipher S1,S1,K + addi 10,10,0x10 + bdnz L2x_round_loop + + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vcipherlast S0,S0,K + vcipherlast S1,S1,K + +IF_LE(<vperm S0,S0,S0,swap_mask + vperm S1,S1,S1,swap_mask>) + + stxvd2x S0X,0,DST + li 9,0x10 + stxvd2x S1X,9,DST + + addi SRC,SRC,0x20 + addi DST,DST,0x20 + + clrldi LENGTH,LENGTH,63 + +L1x: + cmpldi LENGTH,0 + beq Ldone + + lxvd2x KX,0,KEYS + vperm K,K,K,swap_mask + + lxvd2x S0X,0,SRC + +IF_LE(<vperm S0,S0,S0,swap_mask>) + + vxor S0,S0,K + + mtctr ROUNDS + li 10,0x10 +.align 5 +L1x_round_loop: + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vcipher S0,S0,K + addi 10,10,0x10 + bdnz L1x_round_loop + + lxvd2x KX,10,KEYS + vperm K,K,K,swap_mask + vcipherlast S0,S0,K + +IF_LE(<vperm S0,S0,S0,swap_mask>) + + stxvd2x S0X,0,DST + +Ldone: + blr +EPILOGUE(_nettle_aes_encrypt) + + .data + .align 4 +.swap_mask: +IF_LE(<.byte 8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7>) +IF_BE(<.byte 3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12>) diff --git a/powerpc64/gcm-hash8.asm b/powerpc64/gcm-hash8.asm new file mode 100644 index 00000000..fc9ce739 --- /dev/null +++ b/powerpc64/gcm-hash8.asm @@ -0,0 +1,1015 @@ +C powerpc64/gcm-hash8.asm + +ifelse(< + Copyright (C) 2020 Mamone Tarsha + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +>) + +C Register usage: +C VSX instructions is used to load and store data to memory "lxvd2x, stxvd2x" +C instead of VR instructions "lvx, stvx" as a workaround to access unaligned data +C VSX registers are defined with "X" suffix + +define(<SP>, <1>) +define(<TOCP>, <2>) + +define(<TABLE>, <3>) +define(<X>, <4>) +define(<LENGTH>, <5>) +define(<DATA>, <6>) + +define(<zero>, <0>) +define(<swap_mask>, <1>) +define(<hidw_mask>, <2>) +define(<lodw_mask>, <3>) +define(<poly>, <4>) +define(<poly_h>, <4>) +define(<poly_l>, <5>) +define(<RP>, <6>) +define(<Mh>, <7>) +define(<Ml>, <8>) +define(<H>, <9>) +define(<Hh>, <10>) +define(<Hl>, <11>) +define(<RP2>, <9>) +define(<M2h>, <10>) +define(<M2l>, <11>) + +define(<HX>, <41>) +define(<HhX>, <42>) +define(<HlX>, <43>) +define(<H_HhX>, <44>) +define(<H_HX>, <45>) +define(<H_HlX>, <46>) + +define(<sl1>, <1>) +define(<msb>, <5>) +define(<H2>, <6>) +define(<H2h>, <7>) +define(<H2l>, <8>) +define(<H_h>, <12>) +define(<H_m>, <13>) +define(<H_l>, <14>) +define(<H_Hh>, <12>) +define(<H_H>, <13>) +define(<H_Hl>, <14>) +define(<H_t>, <15>) +define(<H2_h>, <16>) +define(<H2_m>, <17>) +define(<H2_l>, <18>) +define(<H2_t>, <19>) + +define(<C0X>, <38>) +define(<C1X>, <39>) +define(<C2X>, <40>) +define(<C3X>, <44>) +define(<C4X>, <38>) +define(<C5X>, <39>) +define(<C6X>, <40>) +define(<C7X>, <44>) + +define(<CX>, <45>) + +define(<C0>, <6>) +define(<C1>, <7>) +define(<C2>, <8>) +define(<C3>, <12>) +define(<C4>, <6>) +define(<C5>, <7>) +define(<C6>, <8>) +define(<C7>, <12>) + +define(<C>, <13>) + +define(<Ch>, <14>) +define(<Cl>, <15>) +define(<Cm>, <16>) + +define(<C01h>, <14>) +define(<C01l>, <15>) +define(<C01>, <16>) +define(<C23h>, <17>) +define(<C23l>, <18>) +define(<C23>, <19>) +define(<C45h>, <20>) +define(<C45l>, <21>) +define(<C45>, <22>) +define(<C67h>, <6>) +define(<C67l>, <7>) +define(<C67>, <8>) + +define(<H21>, <9>) +define(<H21h>, <10>) +define(<H21l>, <11>) +define(<H43>, <23>) +define(<H43h>, <24>) +define(<H43l>, <25>) +define(<H65>, <26>) +define(<H65h>, <27>) +define(<H65l>, <28>) +define(<H87>, <29>) +define(<H87h>, <30>) +define(<H87l>, <31>) + +define(<H21X>, <41>) +define(<H21hX>, <42>) +define(<H21lX>, <43>) +define(<H43X>, <55>) +define(<H43hX>, <56>) +define(<H43lX>, <57>) +define(<H65X>, <58>) +define(<H65hX>, <59>) +define(<H65lX>, <60>) +define(<H87X>, <61>) +define(<H87hX>, <62>) +define(<H87lX>, <63>) + +# gcm_fill registers: + +define(<CTR>, <3>) +define(<BLOCKS>, <4>) +define(<BUFFER>, <5>) + +define(<CTR0>, <2>) +define(<CTR0S>, <3>) +define(<CTR1>, <4>) +define(<CTR2>, <5>) +define(<CTR3>, <6>) +define(<CTR4>, <7>) +define(<CTR5>, <8>) +define(<CTR6>, <9>) +define(<CTR7>, <10>) + +define(<CTR0X>, <34>) +define(<CTR0SX>, <35>) +define(<CTR1X>, <36>) +define(<CTR2X>, <37>) +define(<CTR3X>, <38>) +define(<CTR4X>, <39>) +define(<CTR5X>, <40>) +define(<CTR6X>, <41>) +define(<CTR7X>, <42>) + +define(<I1>, <11>) +define(<I2>, <12>) +define(<I3>, <13>) +define(<I4>, <14>) +define(<I5>, <15>) +define(<I6>, <16>) +define(<I7>, <17>) +define(<I8>, <18>) + +.file "gcm-hash8.asm" + +.machine "any" + +IF_LE(<.abiversion 2>) +.text + + # void gcm_init_key (union gcm_block *table) + +IF_LE(<.align 5>) +PROLOGUE(_nettle_gcm_init_key8) +IF_LE(<.localentry _nettle_gcm_init_key8,0>) + + ld 7,.polynomial@got(TOCP) + lvx poly,0,7 +IF_LE(<ld 7,.swap_mask@got(TOCP) + lvx swap_mask,0,7>) + ld 7,.hidw_mask@got(TOCP) + lvx hidw_mask,0,7 + ld 7,.lodw_mask@got(TOCP) + lvx lodw_mask,0,7 + + li 10,0x800 + lxvd2x HX,10,TABLE # load H +IF_LE(<vperm H,H,H,swap_mask>) + + # --- calculate H = H shift left 1 modulo polynomial --- + + vupkhsw msb,H # most significant bit word-extend + vspltisb sl1,1 # splat 1 for shift left + vspltw msb,msb,0 # most significant bit extend + vsl H,H,sl1 # H shift left 1 + vand msb,msb,poly + vxor zero,zero,zero + vxor H_t,H,msb + + vsldoi H,H_t,H_t,8 # doubleword swap + vsldoi Hh,H,zero,8 + vsldoi Hl,zero,H,8 + + # --- calculate H^2 = H*H --- + + # reduction pre-processing + vsldoi poly_h,zero,poly,8 + vsldoi poly_l,poly_h,poly_h,8 + + # polynomial multiplication "classical" + vpmsumd H_h,H_t,Hh # H^1h*H^1h + vpmsumd H_l,H_t,Hl # H^1l*H^1l + vpmsumd H_m,H_t,H # H^1h*H^1l⊕H^1l*H^1h + + # reduction first phase # [1] + vpmsumd RP,H_l,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,H_m,8 # [2] + vsldoi Ml,H_m,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor H_h,H_h,Mh # [2] + vxor H_l,H_l,Ml # [2] + vxor H_l,H_l,RP # [1] + + # reduction second phase + vpmsumd RP,H_l,poly_l + vxor H_h,H_l,H_h + vxor H2_t,H_h,RP + + vsldoi H2,H2_t,H2_t,8 + vsldoi H2h,H2,zero,8 + vsldoi H2l,zero,H2,8 + + # --- calculate [H^2.Hi⊕H^2.Lo:H^1.Hi⊕H^1.Lo] --- + + vperm H_Hh,H2,H,lodw_mask + vperm H_Hl,H2,H,hidw_mask + vxor H_H,H_Hh,H_Hl + + # --- store H,[H^2.Hi⊕H^2.Lo:H^1.Hi⊕H^1.Lo] --- + + li 8,0x00 + li 9,0x100 + li 10,0x200 + stxvd2x HlX,8,TABLE + stxvd2x HX,9,TABLE + stxvd2x HhX,10,TABLE + + li 8,0x300 + li 9,0x400 + li 10,0x500 + stxvd2x H_HhX,8,TABLE + stxvd2x H_HX,9,TABLE + stxvd2x H_HlX,10,TABLE + + # --- calculate H^3,H^4 --- + + # polynomial multiplication "classical" + vpmsumd H_l,H_t,H2l # H^1l*H^2l + vpmsumd H_m,H_t,H2 # H^1h*H^2l⊕H^1l*H^2h + vpmsumd H_h,H_t,H2h # H^1h*H^2h + vpmsumd H2_l,H2_t,H2l # H^2l*H^2l + vpmsumd H2_m,H2_t,H2 # H^2h*H^2l⊕H^2l*H^2h + vpmsumd H2_h,H2_t,H2h # H^2h*H^2h + + # reduction first phase # [1] + vpmsumd RP,H_l,poly_h # [1] H^3 + vpmsumd RP2,H2_l,poly_h # [1] H^4 + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,H_m,8 # [2] H^3 + vsldoi M2h,zero,H2_m,8 # [2] H^4 + vsldoi Ml,H_m,zero,8 # [2] H^3 + vsldoi M2l,H2_m,zero,8 # [2] H^4 + vsldoi RP,RP,RP,8 # [1] H^3 + vsldoi RP2,RP2,RP2,8 # [1] H^4 + vxor H_h,H_h,Mh # [2] H^3 + vxor H2_h,H2_h,M2h # [2] H^4 + vxor H_l,H_l,Ml # [2] H^3 + vxor H2_l,H2_l,M2l # [2] H^4 + vxor H_l,H_l,RP # [1] H^3 + vxor H2_l,H2_l,RP2 # [1] H^4 + + # reduction second phase + vpmsumd RP,H_l,poly_l # H^3 + vpmsumd RP2,H2_l,poly_l # H^4 + vxor H_h,H_l,H_h # H^3 + vxor H2_h,H2_l,H2_h # H^4 + vxor H_h,H_h,RP # H^3 + vxor H2_h,H2_h,RP2 # H^4 + + vsldoi H2,H2_h,H2_h,8 # H^4 + vsldoi H,H_h,H_h,8 # H^3 + vsldoi H2l,zero,H2,8 # H^4 + vsldoi H2h,H2,zero,8 # H^4 + + # --- calculate [H^4.Hi⊕H^4.Lo:H^3.Hi⊕H^3.Lo] --- + + vperm H_Hh,H2,H,lodw_mask + vperm H_Hl,H2,H,hidw_mask + vxor H_H,H_Hh,H_Hl + + # --- store [H^4.Hi⊕H^4.Lo:H^3.Hi⊕H^3.Lo] --- + + li 8,0x600 + li 9,0x700 + li 10,0x800 + stxvd2x H_HhX,8,TABLE + stxvd2x H_HX,9,TABLE + stxvd2x H_HlX,10,TABLE + + # --- calculate H^5,H^6 --- + + # polynomial multiplication "classical" + vpmsumd H_l,H_t,H2l # H^1l*H^4l + vpmsumd H_m,H_t,H2 # H^1h*H^4l⊕H^1l*H^4h + vpmsumd H_h,H_t,H2h # H^1h*H^4h + vpmsumd H2_l,H2_t,H2l # H^2l*H^4l + vpmsumd H2_m,H2_t,H2 # H^2h*H^4l⊕H^2l*H^4h + vpmsumd H2_h,H2_t,H2h # H^2h*H^4h + + # reduction first phase # [1] + vpmsumd RP,H_l,poly_h # [1] H^5 + vpmsumd RP2,H2_l,poly_h # [1] H^6 + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,H_m,8 # [2] H^5 + vsldoi M2h,zero,H2_m,8 # [2] H^6 + vsldoi Ml,H_m,zero,8 # [2] H^5 + vsldoi M2l,H2_m,zero,8 # [2] H^6 + vsldoi RP,RP,RP,8 # [1] H^5 + vsldoi RP2,RP2,RP2,8 # [1] H^6 + vxor H_h,H_h,Mh # [2] H^5 + vxor H2_h,H2_h,M2h # [2] H^6 + vxor H_l,H_l,Ml # [2] H^5 + vxor H2_l,H2_l,M2l # [2] H^6 + vxor H_l,H_l,RP # [1] H^5 + vxor H2_l,H2_l,RP2 # [1] H^6 + + # reduction second phase + vpmsumd RP,H_l,poly_l # H^5 + vpmsumd RP2,H2_l,poly_l # H^6 + vxor H_h,H_l,H_h # H^5 + vxor H2_h,H2_l,H2_h # H^6 + vxor H_h,H_h,RP # H^5 + vxor H2_h,H2_h,RP2 # H^6 + + vsldoi H2,H2_h,H2_h,8 # H^6 + vsldoi H,H_h,H_h,8 # H^5 + vsldoi H2l,zero,H2,8 # H^6 + vsldoi H2h,H2,zero,8 # H^6 + + # --- calculate [H^6.Hi⊕H^6.Lo:H^5.Hi⊕H^5.Lo] --- + + vperm H_Hh,H2,H,lodw_mask + vperm H_Hl,H2,H,hidw_mask + vxor H_H,H_Hh,H_Hl + + # --- store [H^6.Hi⊕H^6.Lo:H^5.Hi⊕H^5.Lo] --- + + li 8,0x900 + li 9,0xA00 + li 10,0xB00 + stxvd2x H_HhX,8,TABLE + stxvd2x H_HX,9,TABLE + stxvd2x H_HlX,10,TABLE + + # --- calculate H^7,H^8 --- + + # polynomial multiplication "classical" + vpmsumd H_l,H_t,H2l # H^1l*H^6l + vpmsumd H_m,H_t,H2 # H^1h*H^6l⊕H^1l*H^6h + vpmsumd H_h,H_t,H2h # H^1h*H^6h + vpmsumd H2_l,H2_t,H2l # H^2l*H^6l + vpmsumd H2_m,H2_t,H2 # H^2h*H^6l⊕H^2l*H^6h + vpmsumd H2_h,H2_t,H2h # H^2h*H^6h + + # reduction first phase # [1] + vpmsumd RP,H_l,poly_h # [1] H^7 + vpmsumd RP2,H2_l,poly_h # [1] H^8 + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,H_m,8 # [2] H^7 + vsldoi M2h,zero,H2_m,8 # [2] H^8 + vsldoi Ml,H_m,zero,8 # [2] H^7 + vsldoi M2l,H2_m,zero,8 # [2] H^8 + vsldoi RP,RP,RP,8 # [1] H^7 + vsldoi RP2,RP2,RP2,8 # [1] H^8 + vxor H_h,H_h,Mh # [2] H^7 + vxor H2_h,H2_h,M2h # [2] H^8 + vxor H_l,H_l,Ml # [2] H^7 + vxor H2_l,H2_l,M2l # [2] H^8 + vxor H_l,H_l,RP # [1] H^7 + vxor H2_l,H2_l,RP2 # [1] H^8 + + # reduction second phase + vpmsumd RP,H_l,poly_l # H^7 + vpmsumd RP2,H2_l,poly_l # H^8 + vxor H_h,H_l,H_h # H^7 + vxor H2_h,H2_l,H2_h # H^8 + vxor H_h,H_h,RP # H^7 + vxor H2_h,H2_h,RP2 # H^8 + + vsldoi H,H_h,H_h,8 # H^7 + vsldoi H2,H2_h,H2_h,8 # H^8 + + # --- calculate [H^8.Hi⊕H^8.Lo:H^7.Hi⊕H^7.Lo] --- + + vperm H_Hh,H2,H,lodw_mask + vperm H_Hl,H2,H,hidw_mask + vxor H_H,H_Hh,H_Hl + + # --- store [H^8.Hi⊕H^8.Lo:H^7.Hi⊕H^7.Lo] --- + + li 8,0xC00 + li 9,0xD00 + li 10,0xE00 + stxvd2x H_HhX,8,TABLE + stxvd2x H_HX,9,TABLE + stxvd2x H_HlX,10,TABLE + + blr +EPILOGUE(_nettle_gcm_init_key8) + + # void gcm_hash (const struct gcm_key *key, union gcm_block *x, + # size_t length, const uint8_t *data) + +IF_LE(<.align 5>) +PROLOGUE(_nettle_gcm_hash8) +IF_LE(<.localentry _nettle_gcm_hash8,0>) + + vxor zero,zero,zero + + ld 7,.polynomial@got(TOCP) + lvx poly,0,7 +IF_LE(<ld 7,.swap_mask@got(TOCP) + lvx swap_mask,0,7>) + ld 7,.hidw_mask@got(TOCP) + lvx hidw_mask,0,7 + ld 7,.lodw_mask@got(TOCP) + lvx lodw_mask,0,7 + + vsldoi poly_h,zero,poly,8 + vsldoi poly_l,poly_h,poly_h,8 + + lxvd2x CX,0,X # load X +IF_LE(<vperm C,C,C,swap_mask>) + + srdi 7,LENGTH,7 # 8x loop count + cmpldi 7,0 + beq L2x + + # backup registers + stdu SP,-224(SP) + std 28,216(SP) + std 29,208(SP) + std 30,200(SP) + std 31,192(SP) + li 8,176 + stvx 20,8,SP + subi 8,8,16 + stvx 21,8,SP + subi 8,8,16 + stvx 22,8,SP + subi 8,8,16 + stvx 23,8,SP + subi 8,8,16 + stvx 24,8,SP + subi 8,8,16 + stvx 25,8,SP + subi 8,8,16 + stvx 26,8,SP + subi 8,8,16 + stvx 27,8,SP + subi 8,8,16 + stvx 28,8,SP + subi 8,8,16 + stvx 29,8,SP + subi 8,8,16 + stvx 30,8,SP + subi 8,8,16 + stvx 31,8,SP + + # table loading + li 8,0x300 + li 9,0x400 + li 10,0x500 + lxvd2x H21hX,8,TABLE + lxvd2x H21X,9,TABLE + lxvd2x H21lX,10,TABLE + li 8,0x600 + li 9,0x700 + li 10,0x800 + lxvd2x H43hX,8,TABLE + lxvd2x H43X,9,TABLE + lxvd2x H43lX,10,TABLE + li 8,0x900 + li 9,0xA00 + li 10,0xB00 + lxvd2x H65hX,8,TABLE + lxvd2x H65X,9,TABLE + lxvd2x H65lX,10,TABLE + li 8,0xC00 + li 9,0xD00 + li 10,0xE00 + lxvd2x H87hX,8,TABLE + lxvd2x H87X,9,TABLE + lxvd2x H87lX,10,TABLE + + li 8,0x10 + li 9,0x20 + li 10,0x30 + li 28,0x40 + li 29,0x50 + li 30,0x60 + li 31,0x70 + + mtctr 7 +.align 5 +L8x_loop: + # input loading + lxvd2x C0X,0,DATA # load C0 + lxvd2x C1X,8,DATA # load C1 + lxvd2x C2X,9,DATA # load C2 + lxvd2x C3X,10,DATA # load C3 + + # swap permuting +IF_LE(<vperm C0,C0,C0,swap_mask + vperm C1,C1,C1,swap_mask + vperm C2,C2,C2,swap_mask + vperm C3,C3,C3,swap_mask>) + + # previous digest combining + vxor C0,C0,C + + # polynomial multiplication "karatsuba" pre-processing + vperm C23h,C2,C3,hidw_mask + vperm C23l,C2,C3,lodw_mask + vperm C01h,C0,C1,hidw_mask + vperm C01l,C0,C1,lodw_mask + + # input loading + lxvd2x C4X,28,DATA # load C4 + lxvd2x C5X,29,DATA # load C5 + lxvd2x C6X,30,DATA # load C6 + lxvd2x C7X,31,DATA # load C7 + + # swap permuting +IF_LE(<vperm C4,C4,C4,swap_mask + vperm C5,C5,C5,swap_mask + vperm C6,C6,C6,swap_mask + vperm C7,C7,C7,swap_mask>) + + # polynomial multiplication "karatsuba" pre-processing + vperm C45h,C4,C5,hidw_mask + vperm C45l,C4,C5,lodw_mask + vperm C67h,C6,C7,hidw_mask + vperm C67l,C6,C7,lodw_mask + vxor C23,C23h,C23l + vxor C01,C01h,C01l + vxor C45,C45h,C45l + vxor C67,C67h,C67l + + # polynomial multiplication "karatsuba" + vpmsumd C23h,C23h,H65h # H23 = H^6h*C2h⊕H^5h*C3h + vpmsumd C23l,C23l,H65l # L23 = H^6l*C2l⊕H^5l*C3l + vpmsumd C01h,C01h,H87h # H01 = H^8h*C0h⊕H^7h*C1h + vpmsumd C01l,C01l,H87l # L01 = H^8l*C0l⊕H^7l*C1l + vpmsumd C67h,C67h,H21h # H67 = H^2h*C6h⊕H^1h*C7h + vpmsumd C67l,C67l,H21l # L67 = H^2l*C6l⊕H^1l*C7l + vpmsumd C45h,C45h,H43h # H45 = H^4h*C4h⊕H^3h*C5h + vpmsumd C45l,C45l,H43l # L45 = H^4l*C4l⊕H^3l*C5l + vpmsumd C23,C23,H65 # M23 = (H^6h⊕H^5h)*(C2h⊕C3h)⊕(H^6l⊕H^5l)*(C2l⊕C3l) + vpmsumd C01,C01,H87 # M01 = (H^8h⊕H^7h)*(C0h⊕C1h)⊕(H^8l⊕H^7l)*(C0l⊕C1l) + vpmsumd C45,C45,H43 # M45 = (H^4h⊕H^3h)*(C4h⊕C5h)⊕(H^4l⊕H^3l)*(C4l⊕C5l) + vpmsumd C67,C67,H21 # M67 = (H^2h⊕H^1h)*(C6h⊕C7h)⊕(H^2l⊕H^1l)*(C6l⊕C7l) + + # polynomial multiplication "karatsuba" post-processing + vxor C23,C23,C23h + vxor C01,C01,C01h + vxor C45,C45,C45h + vxor C67,C67,C67h + vxor C23,C23,C23l + vxor C01,C01,C01l + vxor C45,C45,C45l + vxor C67,C67,C67l + + # deferred recombination of partial products + vxor C01h,C01h,C23h # H0 = H01⊕H23 + vxor C45h,C45h,C67h # H1 = H45⊕H67 + vxor C01l,C01l,C23l # L0 = L01⊕L23 + vxor C45l,C45l,C67l # L1 = L45⊕L45 + vxor C01,C01,C23 # M0 = M01⊕M23 + vxor C45,C45,C67 # M1 = M45⊕M45 + vxor C01h,C01h,C45h # H = H0⊕H1 + vxor C01l,C01l,C45l # L = L0⊕L1 + vxor C01,C01,C45 # M = M0⊕M1 + + # reduction first phase # [1] + vpmsumd RP,C01l,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,C01,8 # [2] + vsldoi Ml,C01,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor C01h,C01h,Mh # [2] + vxor C01l,C01l,Ml # [2] + vxor C01l,C01l,RP # [1] + + # reduction second phase + vpmsumd RP,C01l,poly_l + vxor C01h,C01l,C01h + vxor C,C01h,RP + + addi DATA,DATA,0x80 + bdnz L8x_loop + + # restore registers + li 8,0 + lvx 31,8,SP + addi 8,8,16 + lvx 30,8,SP + addi 8,8,16 + lvx 29,8,SP + addi 8,8,16 + lvx 28,8,SP + addi 8,8,16 + lvx 27,8,SP + addi 8,8,16 + lvx 26,8,SP + addi 8,8,16 + lvx 25,8,SP + addi 8,8,16 + lvx 24,8,SP + addi 8,8,16 + lvx 23,8,SP + addi 8,8,16 + lvx 22,8,SP + addi 8,8,16 + lvx 21,8,SP + addi 8,8,16 + lvx 20,8,SP + ld 31,192(SP) + ld 30,200(SP) + ld 29,208(SP) + ld 28,216(SP) + addi SP,SP,224 + + clrldi LENGTH,LENGTH,57 +L2x: + srdi 7,LENGTH,5 + cmpldi 7,0 + beq L1x + + # table loading + li 8,0x300 + li 9,0x400 + li 10,0x500 + lxvd2x H21hX,8,TABLE + lxvd2x H21X,9,TABLE + lxvd2x H21lX,10,TABLE + + li 10,0x10 + + mtctr 7 +.align 5 +L2x_loop: + # input loading + lxvd2x C0X,0,DATA # load C0 + lxvd2x C1X,10,DATA # load C1 + + # swap permuting +IF_LE(<vperm C0,C0,C0,swap_mask + vperm C1,C1,C1,swap_mask>) + + # previous digest combining + vxor C0,C0,C + + # polynomial multiplication "karatsuba" pre-processing + vperm C01h,C0,C1,hidw_mask + vperm C01l,C0,C1,lodw_mask + vxor C01,C01h,C01l + + # polynomial multiplication "karatsuba" + vpmsumd C01h,C01h,H21h # H01 = H^2h*C0h⊕H^1h*C1h + vpmsumd C01l,C01l,H21l # L01 = H^2l*C0l⊕H^1l*C1l + vpmsumd C01,C01,H21 # M01 = (H^2h⊕H^1h)*(C0h⊕C1h)⊕(H^2l⊕H^1l)*(C0l⊕C1l) + + # polynomial multiplication "karatsuba" post-processing + vxor C01,C01,C01h + vxor C01,C01,C01l + + # reduction first phase # [1] + vpmsumd RP,C01l,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,C01,8 # [2] + vsldoi Ml,C01,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor C01h,C01h,Mh # [2] + vxor C01l,C01l,Ml # [2] + vxor C01l,C01l,RP # [1] + + # reduction second phase + vpmsumd RP,C01l,poly_l + vxor C01h,C01l,C01h + vxor C,C01h,RP + + addi DATA,DATA,0x20 + bdnz L2x_loop + + clrldi LENGTH,LENGTH,59 +L1x: + srdi 7,LENGTH,4 + cmpldi 7,0 + beq Lrem + + # table loading + li 9,0x100 + li 10,0x200 + lxvd2x HlX,0,TABLE + lxvd2x HX, 9,TABLE + lxvd2x HhX,10,TABLE + + # input loading + lxvd2x C0X,0,DATA # load C0 + + # swap permuting +IF_LE(<vperm C0,C0,C0,swap_mask>) + + # previous digest combining + vxor C0,C0,C + + vpmsumd Cl,C0,Hl # L = Hl*Cl + vpmsumd Cm,C0,H # M = Hh*Cl⊕Hl*Ch + vpmsumd Ch,C0,Hh # H = Hh*Ch + + # reduction first phase # [1] + vpmsumd RP,Cl,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,Cm,8 # [2] + vsldoi Ml,Cm,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor Ch,Ch,Mh # [2] + vxor Cl,Cl,Ml # [2] + vxor Cl,Cl,RP # [1] + + # reduction second phase + vpmsumd RP,Cl,poly_l + vxor Ch,Cl,Ch + vxor C,Ch,RP + + addi DATA,DATA,0x10 + clrldi LENGTH,LENGTH,60 +Lrem: + cmpldi LENGTH,0 + beq Ldone + + # table loading + li 9,0x100 + li 10,0x200 + lxvd2x HlX,0,TABLE + lxvd2x HX, 9,TABLE + lxvd2x HhX,10,TABLE + + # input loading + stdu SP,-16(SP) + stvx zero,0,SP +Lst_loop: + subic. LENGTH,LENGTH,1 + lbzx 7,LENGTH,DATA + stbx 7,LENGTH,SP + bne Lst_loop + lxvd2x C0X,0,SP + addi SP,SP,16 + + # swap permuting +IF_LE(<vperm C0,C0,C0,swap_mask>) + + # previous digest combining + vxor C0,C0,C + + vpmsumd Cl,C0,Hl # L = Hl*Cl + vpmsumd Cm,C0,H # M = Hh*Cl⊕Hl*Ch + vpmsumd Ch,C0,Hh # H = Hh*Ch + + # reduction first phase # [1] + vpmsumd RP,Cl,poly_h # [1] + + # polynomial multiplication post-processing # [2] + vsldoi Mh,zero,Cm,8 # [2] + vsldoi Ml,Cm,zero,8 # [2] + vsldoi RP,RP,RP,8 # [1] + vxor Ch,Ch,Mh # [2] + vxor Cl,Cl,Ml # [2] + vxor Cl,Cl,RP # [1] + + # reduction second phase + vpmsumd RP,Cl,poly_l + vxor Ch,Cl,Ch + vxor C,Ch,RP + +Ldone: +IF_LE(<vperm C,C,C,swap_mask>) + stxvd2x CX,0,X # store C + blr +EPILOGUE(_nettle_gcm_hash8) + + # gcm_fill (uint8_t *ctr, size_t blocks, union gcm_block *buffer) + +IF_LE(<.align 5>) +PROLOGUE(_nettle_gcm_fill) +IF_LE(<.localentry _nettle_gcm_fill,0>) + +IF_LE(<ld 6,.swap_mask@got(TOCP) + lvx swap_mask,0,6>) + + vxor zero,zero,zero + vspltisb I1,1 + vspltisb I2,2 + vspltisb I3,3 + vspltisb I4,4 + vspltisb I5,5 + vspltisb I6,6 + vspltisb I7,7 + vspltisb I8,8 + vsldoi I1,zero,I1,1 + vsldoi I2,zero,I2,1 + vsldoi I3,zero,I3,1 + vsldoi I4,zero,I4,1 + vsldoi I5,zero,I5,1 + vsldoi I6,zero,I6,1 + vsldoi I7,zero,I7,1 + vsldoi I8,zero,I8,1 + + lxvd2x CTR0X,0,CTR + IF_LE(<vperm CTR0,CTR0,CTR0,swap_mask>) + + srdi 6,BLOCKS,3 # 8x loop count + cmpldi 6,0 + beq Lfill_4x + + std 25,-56(SP); + std 26,-48(SP); + std 27,-40(SP); + std 28,-32(SP); + std 29,-24(SP); + std 30,-16(SP); + std 31,-8(SP); + + li 25,0x10 + li 26,0x20 + li 27,0x30 + li 28,0x40 + li 29,0x50 + li 30,0x60 + li 31,0x70 + + mtctr 6 + L8x_fill_loop: + vadduwm CTR1,CTR0,I1 + vadduwm CTR2,CTR0,I2 + vadduwm CTR3,CTR0,I3 + vadduwm CTR4,CTR0,I4 + vadduwm CTR5,CTR0,I5 + vadduwm CTR6,CTR0,I6 + vadduwm CTR7,CTR0,I7 + + IF_LE(<vperm CTR0S,CTR0,CTR0,swap_mask + vperm CTR1,CTR1,CTR1,swap_mask + vperm CTR2,CTR2,CTR2,swap_mask + vperm CTR3,CTR3,CTR3,swap_mask + vperm CTR4,CTR4,CTR4,swap_mask + vperm CTR5,CTR5,CTR5,swap_mask + vperm CTR6,CTR6,CTR6,swap_mask + vperm CTR7,CTR7,CTR7,swap_mask>) + + IF_LE(<stxvd2x CTR0SX,0,BUFFER>) + IF_BE(<stxvd2x CTR0X,0,BUFFER>) + stxvd2x CTR1X,25,BUFFER + stxvd2x CTR2X,26,BUFFER + stxvd2x CTR3X,27,BUFFER + stxvd2x CTR4X,28,BUFFER + stxvd2x CTR5X,29,BUFFER + stxvd2x CTR6X,30,BUFFER + stxvd2x CTR7X,31,BUFFER + + vadduwm CTR0,CTR0,I8 + addi BUFFER,BUFFER,0x80 + bdnz L8x_fill_loop + + ld 25,-56(SP); + ld 26,-48(SP); + ld 27,-40(SP); + ld 28,-32(SP); + ld 29,-24(SP); + ld 30,-16(SP); + ld 31,-8(SP); + + clrldi BLOCKS,BLOCKS,61 + + Lfill_4x: + srdi 6,BLOCKS,2 + cmpldi 6,0 + beq Lfill_2x + + li 8,0x10 + li 9,0x20 + li 10,0x30 + + vadduwm CTR1,CTR0,I1 + vadduwm CTR2,CTR0,I2 + vadduwm CTR3,CTR0,I3 + + IF_LE(<vperm CTR0S,CTR0,CTR0,swap_mask + vperm CTR1,CTR1,CTR1,swap_mask + vperm CTR2,CTR2,CTR2,swap_mask + vperm CTR3,CTR3,CTR3,swap_mask>) + + IF_LE(<stxvd2x CTR0SX,0,BUFFER>) + IF_BE(<stxvd2x CTR0X,0,BUFFER>) + stxvd2x CTR1X,8,BUFFER + stxvd2x CTR2X,9,BUFFER + stxvd2x CTR3X,10,BUFFER + + vadduwm CTR0,CTR0,I4 + addi BUFFER,BUFFER,0x40 + + clrldi BLOCKS,BLOCKS,62 + + Lfill_2x: + srdi 6,BLOCKS,1 + cmpldi 6,0 + beq Lfill_1x + + li 10,0x10 + + vadduwm CTR1,CTR0,I1 + + IF_LE(<vperm CTR0S,CTR0,CTR0,swap_mask + vperm CTR1,CTR1,CTR1,swap_mask>) + + IF_LE(<stxvd2x CTR0SX,0,BUFFER>) + IF_BE(<stxvd2x CTR0X,0,BUFFER>) + stxvd2x CTR1X,10,BUFFER + + vadduwm CTR0,CTR0,I2 + addi BUFFER,BUFFER,0x20 + + clrldi BLOCKS,BLOCKS,63 + + Lfill_1x: + cmpldi BLOCKS,0 + beq Lfill_done + + IF_LE(<vperm CTR0S,CTR0,CTR0,swap_mask>) + + IF_LE(<stxvd2x CTR0SX,0,BUFFER>) + IF_BE(<stxvd2x CTR0X,0,BUFFER>) + + vadduwm CTR0,CTR0,I1 + + Lfill_done: + IF_LE(<vperm CTR0,CTR0,CTR0,swap_mask>) + stxvd2x CTR0X,0,CTR + + blr +EPILOGUE(_nettle_gcm_fill) + + .data +IF_LE(<.align 4 +.polynomial: + .byte 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2 + .align 4 +.swap_mask: + .byte 8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7 + .align 4 +.hidw_mask: + .byte 23,22,21,20,19,18,17,16,7,6,5,4,3,2,1,0 + .align 4 +.lodw_mask: + .byte 31,30,29,28,27,26,25,24,15,14,13,12,11,10,9,8>) +IF_BE(<.align 4 +.polynomial: + .byte 0xc2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 + .align 4 +.hidw_mask: + .byte 0,1,2,3,4,5,6,7,16,17,18,19,20,21,22,23 + .align 4 +.lodw_mask: + .byte 8,9,10,11,12,13,14,15,24,25,26,27,28,29,30,31>) diff --git a/powerpc64/machine.m4 b/powerpc64/machine.m4 new file mode 100644 index 00000000..e69de29b diff --git a/testsuite/gcm-test.c b/testsuite/gcm-test.c index c8174019..df1fc94a 100644 --- a/testsuite/gcm-test.c +++ b/testsuite/gcm-test.c @@ -170,6 +170,29 @@ test_main(void) "16aedbf5a0de6a57a637b39b"), SHEX("619cc5aefffe0bfa462af43c1699d050")); + /* Test 128 bytes */ + test_aead(&nettle_gcm_aes128, NULL, + SHEX("feffe9928665731c6d6a8f9467308308"), + SHEX(""), + SHEX("d9313225f88406e5a55909c5aff5269a" + "86a7a9531534f7da2e4c303d8a318a72" + "1c3c0c95956809532fcf0e2449a6b525" + "b16aedf5aa0de657ba637b391aafd255" + "5ae376bc5e9f6a1b08e34db7a6ee0736" + "9ba662ea12f6f197e6bc3ed69d2480f3" + "ea5691347f2ba69113eb37910ebc18c8" + "0f697234582016fa956ca8f63ae6b473"), + SHEX("42831ec2217774244b7221b784d0d49c" + "e3aa212f2c02a4e035c17e2329aca12e" + "21d514b25466931c7d8f6a5aac84aa05" + "1ba30b396a0aac973d58e091473f5985" + "874b1178906ddbeab04ab2fe6cce8c57" + "8d7e961bd13fd6a8c56b66ca5e576492" + "1a48cd8bda04e66343e73055118b69b9" + "ced486813846958a11e602c03cfc232b"), + SHEX("cafebabefacedbaddecaf888"), + SHEX("796836f1246c9d735c5e1be0a715ccc3")); + /* Test case 7 */ test_aead(&nettle_gcm_aes192, NULL, SHEX("00000000000000000000000000000000" -- 2.17.1

3 8

[PATCH] Add ppc64 and ppc64el to Gitlab CI
by Maamoun TK 30 Jun '20

30 Jun '20

To run tests on ppc64 and ppc64el, this patch install cross building packages for both architectures on the Debian image, these packages will be install every time the CI triggered. A proper fix would be to install these packages to the image directly. This patch follows a different approach to get access LD shared library for qemu. Other architectures install native libc6 packages while both ppc64 and ppc64el install cross libc6 packages and export QEMU_LD_PREFIX to point to LD path which is more reasonable. .gitlab-ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9f3b5c63..7f12ecfa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -139,6 +139,11 @@ Debian.cross.x86: before_script: # remove any previously installed nettle headers to avoid conflicts - for arch in armhf mips arm64;do apt-get remove -y nettle-dev:$arch;done + - host="${CI_JOB_NAME#*.cross.}" + - if [ "$host" == "powerpc64-linux-gnu" ];then apt-get update && apt-get install -y gcc-$host && + apt-get install -y g++-$host && apt-get install -y libstdc++6-ppc64-cross && export QEMU_LD_PREFIX=/usr/$host;fi + - if [ "$host" == "powerpc64le-linux-gnu" ];then apt-get update && apt-get install -y gcc-$host && + apt-get install -y g++-$host && apt-get install -y libstdc++6-ppc64el-cross && export QEMU_LD_PREFIX=/usr/$host;fi script: - build=$(dpkg-architecture -qDEB_HOST_GNU_TYPE) - host="${CI_JOB_NAME#*.cross.}" @@ -162,3 +167,7 @@ Debian.cross.mips-linux-gnu: <<: *Debian_cross_template Debian.cross.aarch64-linux-gnu: <<: *Debian_cross_template +Debian.cross.powerpc64-linux-gnu: + <<: *Debian_cross_template +Debian.cross.powerpc64le-linux-gnu: + <<: *Debian_cross_template

3 8

Re: [Patch] Optimize AES and GHASH for PowerPC64 (support little-endian and big-endian)
by Maamoun TK 30 Jun '20

30 Jun '20

On Tue, Jun 30, 2020 at 12:29 PM Jeffrey Walton <noloader(a)gmail.com> wrote: One small comment for aes_encrypt and aes_decrypt... src and dst are > usually user supplied buffers. Using lxvd2x to load a vector may > produce incorrect results if the user is feeding a stream to an > encryptor or decryptor that is not naturally aligned to that of an > unsigned int. (On the other hand, Nettle controls the round keys array > so lxvd2x should be fine.) > > Instead of lxvd2x and friends for the user's buffers you should > consider using lvx and doing the lvsl thing to fix the data in the > registers. > > Jeff > I considered using lvx and lvsl to load the user's buffers but lxvd2x loads unaligned data properly as well as making the loading process simpler. I tested loading data that was not aligned at 1 byte and so lxvd2x handled it properly. Let me know if I miss something. >

1 0

[PATCH v2 1/8] Implement GOST R 34.11-2012 (Streebog) hash function
by Dmitry Baryshkov 30 Jun '20

30 Jun '20

Signed-off-by: Dmitry Baryshkov <dbaryshkov(a)gmail.com> --- Makefile.in | 3 +- streebog-meta.c | 44 ++ streebog.c | 1334 +++++++++++++++++++++++++++++++++++++++++++++++ streebog.h | 99 ++++ 4 files changed, 1479 insertions(+), 1 deletion(-) create mode 100644 streebog-meta.c create mode 100644 streebog.c create mode 100644 streebog.h diff --git a/Makefile.in b/Makefile.in index e5ccfc76b901..c36764dc4c45 100644 --- a/Makefile.in +++ b/Makefile.in @@ -137,6 +137,7 @@ nettle_SOURCES = aes-decrypt-internal.c aes-decrypt.c \ shake256.c \ serpent-set-key.c serpent-encrypt.c serpent-decrypt.c \ serpent-meta.c \ + streebog.c streebog-meta.c \ twofish.c twofish-meta.c \ umac-nh.c umac-nh-n.c umac-l2.c umac-l3.c \ umac-poly64.c umac-poly128.c umac-set-key.c \ @@ -222,7 +223,7 @@ HEADERS = aes.h arcfour.h arctwo.h asn1.h blowfish.h \ pbkdf2.h \ pgp.h pkcs1.h pss.h pss-mgf1.h realloc.h ripemd160.h rsa.h \ salsa20.h sexp.h \ - serpent.h sha.h sha1.h sha2.h sha3.h twofish.h \ + serpent.h sha.h sha1.h sha2.h sha3.h streebog.h twofish.h \ umac.h yarrow.h xts.h poly1305.h INSTALL_HEADERS = $(HEADERS) version.h @IF_MINI_GMP@ mini-gmp.h diff --git a/streebog-meta.c b/streebog-meta.c new file mode 100644 index 000000000000..de88651b8810 --- /dev/null +++ b/streebog-meta.c @@ -0,0 +1,44 @@ +/* streebog-meta.c + + Copyright (C) 2020 Dmitry Baryshkov + + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +*/ + +#if HAVE_CONFIG_H +# include "config.h" +#endif + +#include "nettle-meta.h" + +#include "streebog.h" + +const struct nettle_hash nettle_streebog512 += _NETTLE_HASH(streebog512, STREEBOG512); + +const struct nettle_hash nettle_streebog256 += _NETTLE_HASH(streebog256, STREEBOG256); diff --git a/streebog.c b/streebog.c new file mode 100644 index 000000000000..3f2fc2f40dfb --- /dev/null +++ b/streebog.c @@ -0,0 +1,1334 @@ +/* streebog.c - GOST R 34.11-2012 (Streebog) hash function + + Copyright (C) 2020 Dmitry Baryshkov + + Based on my code in libgcrypt. + + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. + */ + +#if HAVE_CONFIG_H +# include "config.h" +#endif + +#include <assert.h> +#include <stdlib.h> +#include <string.h> + +#include "streebog.h" + +#include "macros.h" +#include "nettle-write.h" + + +/* Pre-computed results of multiplication of bytes on A and reordered with + Pi[]. */ +static const uint64_t streebog_table[8][256] = +{ + /* 0 */ + { 0xd01f715b5c7ef8e6ULL, 0x16fa240980778325ULL, + 0xa8a42e857ee049c8ULL, 0x6ac1068fa186465bULL, + 0x6e417bd7a2e9320bULL, 0x665c8167a437daabULL, + 0x7666681aa89617f6ULL, 0x4b959163700bdcf5ULL, + 0xf14be6b78df36248ULL, 0xc585bd689a625cffULL, + 0x9557d7fca67d82cbULL, 0x89f0b969af6dd366ULL, + 0xb0833d48749f6c35ULL, 0xa1998c23b1ecbc7cULL, + 0x8d70c431ac02a736ULL, 0xd6dfbc2fd0a8b69eULL, + 0x37aeb3e551fa198bULL, 0x0b7d128a40b5cf9cULL, + 0x5a8f2008b5780cbcULL, 0xedec882284e333e5ULL, + 0xd25fc177d3c7c2ceULL, 0x5e0f5d50b61778ecULL, + 0x1d873683c0c24cb9ULL, 0xad040bcbb45d208cULL, + 0x2f89a0285b853c76ULL, 0x5732fff6791b8d58ULL, + 0x3e9311439ef6ec3fULL, 0xc9183a809fd3c00fULL, + 0x83adf3f5260a01eeULL, 0xa6791941f4e8ef10ULL, + 0x103ae97d0ca1cd5dULL, 0x2ce948121dee1b4aULL, + 0x39738421dbf2bf53ULL, 0x093da2a6cf0cf5b4ULL, + 0xcd9847d89cbcb45fULL, 0xf9561c078b2d8ae8ULL, + 0x9c6a755a6971777fULL, 0xbc1ebaa0712ef0c5ULL, + 0x72e61542abf963a6ULL, 0x78bb5fde229eb12eULL, + 0x14ba94250fceb90dULL, 0x844d6697630e5282ULL, + 0x98ea08026a1e032fULL, 0xf06bbea144217f5cULL, + 0xdb6263d11ccb377aULL, 0x641c314b2b8ee083ULL, + 0x320e96ab9b4770cfULL, 0x1ee7deb986a96b85ULL, + 0xe96cf57a878c47b5ULL, 0xfdd6615f8842feb8ULL, + 0xc83862965601dd1bULL, 0x2ea9f83e92572162ULL, + 0xf876441142ff97fcULL, 0xeb2c455608357d9dULL, + 0x5612a7e0b0c9904cULL, 0x6c01cbfb2d500823ULL, + 0x4548a6a7fa037a2dULL, 0xabc4c6bf388b6ef4ULL, + 0xbade77d4fdf8bebdULL, 0x799b07c8eb4cac3aULL, + 0x0c9d87e805b19cf0ULL, 0xcb588aac106afa27ULL, + 0xea0c1d40c1e76089ULL, 0x2869354a1e816f1aULL, + 0xff96d17307fbc490ULL, 0x9f0a9d602f1a5043ULL, + 0x96373fc6e016a5f7ULL, 0x5292dab8b3a6e41cULL, + 0x9b8ae0382c752413ULL, 0x4f15ec3b7364a8a5ULL, + 0x3fb349555724f12bULL, 0xc7c50d4415db66d7ULL, + 0x92b7429ee379d1a7ULL, 0xd37f99611a15dfdaULL, + 0x231427c05e34a086ULL, 0xa439a96d7b51d538ULL, + 0xb403401077f01865ULL, 0xdda2aea5901d7902ULL, + 0x0a5d4a9c8967d288ULL, 0xc265280adf660f93ULL, + 0x8bb0094520d4e94eULL, 0x2a29856691385532ULL, + 0x42a833c5bf072941ULL, 0x73c64d54622b7eb2ULL, + 0x07e095624504536cULL, 0x8a905153e906f45aULL, + 0x6f6123c16b3b2f1fULL, 0xc6e55552dc097bc3ULL, + 0x4468feb133d16739ULL, 0xe211e7f0c7398829ULL, + 0xa2f96419f7879b40ULL, 0x19074bdbc3ad38e9ULL, + 0xf4ebc3f9474e0b0cULL, 0x43886bd376d53455ULL, + 0xd8028beb5aa01046ULL, 0x51f23282f5cdc320ULL, + 0xe7b1c2be0d84e16dULL, 0x081dfab006dee8a0ULL, + 0x3b33340d544b857bULL, 0x7f5bcabc679ae242ULL, + 0x0edd37c48a08a6d8ULL, 0x81ed43d9a9b33bc6ULL, + 0xb1a3655ebd4d7121ULL, 0x69a1eeb5e7ed6167ULL, + 0xf6ab73d5c8f73124ULL, 0x1a67a3e185c61fd5ULL, + 0x2dc91004d43c065eULL, 0x0240b02c8fb93a28ULL, + 0x90f7f2b26cc0eb8fULL, 0x3cd3a16f114fd617ULL, + 0xaae49ea9f15973e0ULL, 0x06c0cd748cd64e78ULL, + 0xda423bc7d5192a6eULL, 0xc345701c16b41287ULL, + 0x6d2193ede4821537ULL, 0xfcf639494190e3acULL, + 0x7c3b228621f1c57eULL, 0xfb16ac2b0494b0c0ULL, + 0xbf7e529a3745d7f9ULL, 0x6881b6a32e3f7c73ULL, + 0xca78d2bad9b8e733ULL, 0xbbfe2fc2342aa3a9ULL, + 0x0dbddffecc6381e4ULL, 0x70a6a56e2440598eULL, + 0xe4d12a844befc651ULL, 0x8c509c2765d0ba22ULL, + 0xee8c6018c28814d9ULL, 0x17da7c1f49a59e31ULL, + 0x609c4c1328e194d3ULL, 0xb3e3d57232f44b09ULL, + 0x91d7aaa4a512f69bULL, 0x0ffd6fd243dabbccULL, + 0x50d26a943c1fde34ULL, 0x6be15e9968545b4fULL, + 0x94778fea6faf9fdfULL, 0x2b09dd7058ea4826ULL, + 0x677cd9716de5c7bfULL, 0x49d5214fffb2e6ddULL, + 0x0360e83a466b273cULL, 0x1fc786af4f7b7691ULL, + 0xa0b9d435783ea168ULL, 0xd49f0c035f118cb6ULL, + 0x01205816c9d21d14ULL, 0xac2453dd7d8f3d98ULL, + 0x545217cc3f70aa64ULL, 0x26b4028e9489c9c2ULL, + 0xdec2469fd6765e3eULL, 0x04807d58036f7450ULL, + 0xe5f17292823ddb45ULL, 0xf30b569b024a5860ULL, + 0x62dcfc3fa758aefbULL, 0xe84cad6c4e5e5aa1ULL, + 0xccb81fce556ea94bULL, 0x53b282ae7a74f908ULL, + 0x1b47fbf74c1402c1ULL, 0x368eebf39828049fULL, + 0x7afbeff2ad278b06ULL, 0xbe5e0a8cfe97caedULL, + 0xcfd8f7f413058e77ULL, 0xf78b2bc301252c30ULL, + 0x4d555c17fcdd928dULL, 0x5f2f05467fc565f8ULL, + 0x24f4b2a21b30f3eaULL, 0x860dd6bbecb768aaULL, + 0x4c750401350f8f99ULL, 0x0000000000000000ULL, + 0xecccd0344d312ef1ULL, 0xb5231806be220571ULL, + 0xc105c030990d28afULL, 0x653c695de25cfd97ULL, + 0x159acc33c61ca419ULL, 0xb89ec7f872418495ULL, + 0xa9847693b73254dcULL, 0x58cf90243ac13694ULL, + 0x59efc832f3132b80ULL, 0x5c4fed7c39ae42c4ULL, + 0x828dabe3efd81cfaULL, 0xd13f294d95ace5f2ULL, + 0x7d1b7a90e823d86aULL, 0xb643f03cf849224dULL, + 0x3df3f979d89dcb03ULL, 0x7426d836272f2ddeULL, + 0xdfe21e891fa4432aULL, 0x3a136c1b9d99986fULL, + 0xfa36f43dcd46add4ULL, 0xc025982650df35bbULL, + 0x856d3e81aadc4f96ULL, 0xc4a5e57e53b041ebULL, + 0x4708168b75ba4005ULL, 0xaf44bbe73be41aa4ULL, + 0x971767d029c4b8e3ULL, 0xb9be9feebb939981ULL, + 0x215497ecd18d9aaeULL, 0x316e7e91dd2c57f3ULL, + 0xcef8afe2dad79363ULL, 0x3853dc371220a247ULL, + 0x35ee03c9de4323a3ULL, 0xe6919aa8c456fc79ULL, + 0xe05157dc4880b201ULL, 0x7bdbb7e464f59612ULL, + 0x127a59518318f775ULL, 0x332ecebd52956ddbULL, + 0x8f30741d23bb9d1eULL, 0xd922d3fd93720d52ULL, + 0x7746300c61440ae2ULL, 0x25d4eab4d2e2eefeULL, + 0x75068020eefd30caULL, 0x135a01474acaea61ULL, + 0x304e268714fe4ae7ULL, 0xa519f17bb283c82cULL, + 0xdc82f6b359cf6416ULL, 0x5baf781e7caa11a8ULL, + 0xb2c38d64fb26561dULL, 0x34ce5bdf17913eb7ULL, + 0x5d6fb56af07c5fd0ULL, 0x182713cd0a7f25fdULL, + 0x9e2ac576e6c84d57ULL, 0x9aaab82ee5a73907ULL, + 0xa3d93c0f3e558654ULL, 0x7e7b92aaae48ff56ULL, + 0x872d8ead256575beULL, 0x41c8dbfff96c0e7dULL, + 0x99ca5014a3cc1e3bULL, 0x40e883e930be1369ULL, + 0x1ca76e95091051adULL, 0x4e35b42dbab6b5b1ULL, + 0x05a0254ecabd6944ULL, 0xe1710fca8152af15ULL, + 0xf22b0e8dcb984574ULL, 0xb763a82a319b3f59ULL, + 0x63fca4296e8ab3efULL, 0x9d4a2d4ca0a36a6bULL, + 0xe331bfe60eeb953dULL, 0xd5bf541596c391a2ULL, + 0xf5cb9bef8e9c1618ULL, 0x46284e9dbc685d11ULL, + 0x2074cffa185f87baULL, 0xbd3ee2b6b8fcedd1ULL, + 0xae64e3f1f23607b0ULL, 0xfeb68965ce29d984ULL, + 0x55724fdaf6a2b770ULL, 0x29496d5cd753720eULL, + 0xa75941573d3af204ULL, 0x8e102c0bea69800aULL, + 0x111ab16bc573d049ULL, 0xd7ffe439197aab8aULL, + 0xefac380e0b5a09cdULL, 0x48f579593660fbc9ULL, + 0x22347fd697e6bd92ULL, 0x61bc1405e13389c7ULL, + 0x4ab5c975b9d9c1e1ULL, 0x80cd1bcf606126d2ULL, + 0x7186fd78ed92449aULL, 0x93971a882aabccb3ULL, + 0x88d0e17f66bfce72ULL, 0x27945a985d5bd4d6ULL }, + /* 1 */ + { 0xde553f8c05a811c8ULL, 0x1906b59631b4f565ULL, + 0x436e70d6b1964ff7ULL, 0x36d343cb8b1e9d85ULL, + 0x843dfacc858aab5aULL, 0xfdfc95c299bfc7f9ULL, + 0x0f634bdea1d51fa2ULL, 0x6d458b3b76efb3cdULL, + 0x85c3f77cf8593f80ULL, 0x3c91315fbe737cb2ULL, + 0x2148b03366ace398ULL, 0x18f8b8264c6761bfULL, + 0xc830c1c495c9fb0fULL, 0x981a76102086a0aaULL, + 0xaa16012142f35760ULL, 0x35cc54060c763cf6ULL, + 0x42907d66cc45db2dULL, 0x8203d44b965af4bcULL, + 0x3d6f3cefc3a0e868ULL, 0xbc73ff69d292bda7ULL, + 0x8722ed0102e20a29ULL, 0x8f8185e8cd34deb7ULL, + 0x9b0561dda7ee01d9ULL, 0x5335a0193227fad6ULL, + 0xc9cecc74e81a6fd5ULL, 0x54f5832e5c2431eaULL, + 0x99e47ba05d553470ULL, 0xf7bee756acd226ceULL, + 0x384e05a5571816fdULL, 0xd1367452a47d0e6aULL, + 0xf29fde1c386ad85bULL, 0x320c77316275f7caULL, + 0xd0c879e2d9ae9ab0ULL, 0xdb7406c69110ef5dULL, + 0x45505e51a2461011ULL, 0xfc029872e46c5323ULL, + 0xfa3cb6f5f7bc0cc5ULL, 0x031f17cd8768a173ULL, + 0xbd8df2d9af41297dULL, 0x9d3b4f5ab43e5e3fULL, + 0x4071671b36feee84ULL, 0x716207e7d3e3b83dULL, + 0x48d20ff2f9283a1aULL, 0x27769eb4757cbc7eULL, + 0x5c56ebc793f2e574ULL, 0xa48b474f9ef5dc18ULL, + 0x52cbada94ff46e0cULL, 0x60c7da982d8199c6ULL, + 0x0e9d466edc068b78ULL, 0x4eec2175eaf865fcULL, + 0x550b8e9e21f7a530ULL, 0x6b7ba5bc653fec2bULL, + 0x5eb7f1ba6949d0ddULL, 0x57ea94e3db4c9099ULL, + 0xf640eae6d101b214ULL, 0xdd4a284182c0b0bbULL, + 0xff1d8fbf6304f250ULL, 0xb8accb933bf9d7e8ULL, + 0xe8867c478eb68c4dULL, 0x3f8e2692391bddc1ULL, + 0xcb2fd60912a15a7cULL, 0xaec935dbab983d2fULL, + 0xf55ffd2b56691367ULL, 0x80e2ce366ce1c115ULL, + 0x179bf3f8edb27e1dULL, 0x01fe0db07dd394daULL, + 0xda8a0b76ecc37b87ULL, 0x44ae53e1df9584cbULL, + 0xb310b4b77347a205ULL, 0xdfab323c787b8512ULL, + 0x3b511268d070b78eULL, 0x65e6e3d2b9396753ULL, + 0x6864b271e2574d58ULL, 0x259784c98fc789d7ULL, + 0x02e11a7dfabb35a9ULL, 0x8841a6dfa337158bULL, + 0x7ade78c39b5dcdd0ULL, 0xb7cf804d9a2cc84aULL, + 0x20b6bd831b7f7742ULL, 0x75bd331d3a88d272ULL, + 0x418f6aab4b2d7a5eULL, 0xd9951cbb6babdaf4ULL, + 0xb6318dfde7ff5c90ULL, 0x1f389b112264aa83ULL, + 0x492c024284fbaec0ULL, 0xe33a0363c608f9a0ULL, + 0x2688930408af28a4ULL, 0xc7538a1a341ce4adULL, + 0x5da8e677ee2171aeULL, 0x8c9e92254a5c7fc4ULL, + 0x63d8cd55aae938b5ULL, 0x29ebd8daa97a3706ULL, + 0x959827b37be88aa1ULL, 0x1484e4356adadf6eULL, + 0xa7945082199d7d6bULL, 0xbf6ce8a455fa1cd4ULL, + 0x9cc542eac9edcae5ULL, 0x79c16f0e1c356ca3ULL, + 0x89bfab6fdee48151ULL, 0xd4174d1830c5f0ffULL, + 0x9258048415eb419dULL, 0x6139d72850520d1cULL, + 0x6a85a80c18ec78f1ULL, 0xcd11f88e0171059aULL, + 0xcceff53e7ca29140ULL, 0xd229639f2315af19ULL, + 0x90b91ef9ef507434ULL, 0x5977d28d074a1be1ULL, + 0x311360fce51d56b9ULL, 0xc093a92d5a1f2f91ULL, + 0x1a19a25bb6dc5416ULL, 0xeb996b8a09de2d3eULL, + 0xfee3820f1ed7668aULL, 0xd7085ad5b7ad518cULL, + 0x7fff41890fe53345ULL, 0xec5948bd67dde602ULL, + 0x2fd5f65dbaaa68e0ULL, 0xa5754affe32648c2ULL, + 0xf8ddac880d07396cULL, 0x6fa491468c548664ULL, + 0x0c7c5c1326bdbed1ULL, 0x4a33158f03930fb3ULL, + 0x699abfc19f84d982ULL, 0xe4fa2054a80b329cULL, + 0x6707f9af438252faULL, 0x08a368e9cfd6d49eULL, + 0x47b1442c58fd25b8ULL, 0xbbb3dc5ebc91769bULL, + 0x1665fe489061eac7ULL, 0x33f27a811fa66310ULL, + 0x93a609346838d547ULL, 0x30ed6d4c98cec263ULL, + 0x1dd9816cd8df9f2aULL, 0x94662a03063b1e7bULL, + 0x83fdd9fbeb896066ULL, 0x7b207573e68e590aULL, + 0x5f49fc0a149a4407ULL, 0x343259b671a5a82cULL, + 0xfbc2bb458a6f981fULL, 0xc272b350a0a41a38ULL, + 0x3aaf1fd8ada32354ULL, 0x6cbb868b0b3c2717ULL, + 0xa2b569c88d2583feULL, 0xf180c9d1bf027928ULL, + 0xaf37386bd64ba9f5ULL, 0x12bacab2790a8088ULL, + 0x4c0d3b0810435055ULL, 0xb2eeb9070e9436dfULL, + 0xc5b29067cea7d104ULL, 0xdcb425f1ff132461ULL, + 0x4f122cc5972bf126ULL, 0xac282fa651230886ULL, + 0xe7e537992f6393efULL, 0xe61b3a2952b00735ULL, + 0x709c0a57ae302ce7ULL, 0xe02514ae416058d3ULL, + 0xc44c9dd7b37445deULL, 0x5a68c5408022ba92ULL, + 0x1c278cdca50c0bf0ULL, 0x6e5a9cf6f18712beULL, + 0x86dce0b17f319ef3ULL, 0x2d34ec2040115d49ULL, + 0x4bcd183f7e409b69ULL, 0x2815d56ad4a9a3dcULL, + 0x24698979f2141d0dULL, 0x0000000000000000ULL, + 0x1ec696a15fb73e59ULL, 0xd86b110b16784e2eULL, + 0x8e7f8858b0e74a6dULL, 0x063e2e8713d05fe6ULL, + 0xe2c40ed3bbdb6d7aULL, 0xb1f1aeca89fc97acULL, + 0xe1db191e3cb3cc09ULL, 0x6418ee62c4eaf389ULL, + 0xc6ad87aa49cf7077ULL, 0xd6f65765ca7ec556ULL, + 0x9afb6c6dda3d9503ULL, 0x7ce05644888d9236ULL, + 0x8d609f95378feb1eULL, 0x23a9aa4e9c17d631ULL, + 0x6226c0e5d73aac6fULL, 0x56149953a69f0443ULL, + 0xeeb852c09d66d3abULL, 0x2b0ac2a753c102afULL, + 0x07c023376e03cb3cULL, 0x2ccae1903dc2c993ULL, + 0xd3d76e2f5ec63bc3ULL, 0x9e2458973356ff4cULL, + 0xa66a5d32644ee9b1ULL, 0x0a427294356de137ULL, + 0x783f62be61e6f879ULL, 0x1344c70204d91452ULL, + 0x5b96c8f0fdf12e48ULL, 0xa90916ecc59bf613ULL, + 0xbe92e5142829880eULL, 0x727d102a548b194eULL, + 0x1be7afebcb0fc0ccULL, 0x3e702b2244c8491bULL, + 0xd5e940a84d166425ULL, 0x66f9f41f3e51c620ULL, + 0xabe80c913f20c3baULL, 0xf07ec461c2d1edf2ULL, + 0xf361d3ac45b94c81ULL, 0x0521394a94b8fe95ULL, + 0xadd622162cf09c5cULL, 0xe97871f7f3651897ULL, + 0xf4a1f09b2bba87bdULL, 0x095d6559b2054044ULL, + 0x0bbc7f2448be75edULL, 0x2af4cf172e129675ULL, + 0x157ae98517094bb4ULL, 0x9fda55274e856b96ULL, + 0x914713499283e0eeULL, 0xb952c623462a4332ULL, + 0x74433ead475b46a8ULL, 0x8b5eb112245fb4f8ULL, + 0xa34b6478f0f61724ULL, 0x11a5dd7ffe6221fbULL, + 0xc16da49d27ccbb4bULL, 0x76a224d0bde07301ULL, + 0x8aa0bca2598c2022ULL, 0x4df336b86d90c48fULL, + 0xea67663a740db9e4ULL, 0xef465f70e0b54771ULL, + 0x39b008152acb8227ULL, 0x7d1e5bf4f55e06ecULL, + 0x105bd0cf83b1b521ULL, 0x775c2960c033e7dbULL, + 0x7e014c397236a79fULL, 0x811cc386113255cfULL, + 0xeda7450d1a0e72d8ULL, 0x5889df3d7a998f3bULL, + 0x2e2bfbedc779fc3aULL, 0xce0eef438619a4e9ULL, + 0x372d4e7bf6cd095fULL, 0x04df34fae96b6a4fULL, + 0xf923a13870d4adb6ULL, 0xa1aa7e050a4d228dULL, + 0xa8f71b5cb84862c9ULL, 0xb52e9a306097fde3ULL, + 0x0d8251a35b6e2a0bULL, 0x2257a7fee1c442ebULL, + 0x73831d9a29588d94ULL, 0x51d4ba64c89ccf7fULL, + 0x502ab7d4b54f5ba5ULL, 0x97793dce8153bf08ULL, + 0xe5042de4d5d8a646ULL, 0x9687307efc802bd2ULL, + 0xa05473b5779eb657ULL, 0xb4d097801d446939ULL, + 0xcff0e2f3fbca3033ULL, 0xc38cbee0dd778ee2ULL, + 0x464f499c252eb162ULL, 0xcad1dbb96f72cea6ULL, + 0xba4dd1eec142e241ULL, 0xb00fa37af42f0376ULL }, + /* 2 */ + { 0xcce4cd3aa968b245ULL, 0x089d5484e80b7fafULL, + 0x638246c1b3548304ULL, 0xd2fe0ec8c2355492ULL, + 0xa7fbdf7ff2374eeeULL, 0x4df1600c92337a16ULL, + 0x84e503ea523b12fbULL, 0x0790bbfd53ab0c4aULL, + 0x198a780f38f6ea9dULL, 0x2ab30c8f55ec48cbULL, + 0xe0f7fed6b2c49db5ULL, 0xb6ecf3f422cadbdcULL, + 0x409c9a541358df11ULL, 0xd3ce8a56dfde3fe3ULL, + 0xc3e9224312c8c1a0ULL, 0x0d6dfa58816ba507ULL, + 0xddf3e1b179952777ULL, 0x04c02a42748bb1d9ULL, + 0x94c2abff9f2decb8ULL, 0x4f91752da8f8acf4ULL, + 0x78682befb169bf7bULL, 0xe1c77a48af2ff6c4ULL, + 0x0c5d7ec69c80ce76ULL, 0x4cc1e4928fd81167ULL, + 0xfeed3d24d9997b62ULL, 0x518bb6dfc3a54a23ULL, + 0x6dbf2d26151f9b90ULL, 0xb5bc624b05ea664fULL, + 0xe86aaa525acfe21aULL, 0x4801ced0fb53a0beULL, + 0xc91463e6c00868edULL, 0x1027a815cd16fe43ULL, + 0xf67069a0319204cdULL, 0xb04ccc976c8abce7ULL, + 0xc0b9b3fc35e87c33ULL, 0xf380c77c58f2de65ULL, + 0x50bb3241de4e2152ULL, 0xdf93f490435ef195ULL, + 0xf1e0d25d62390887ULL, 0xaf668bfb1a3c3141ULL, + 0xbc11b251f00a7291ULL, 0x73a5eed47e427d47ULL, + 0x25bee3f6ee4c3b2eULL, 0x43cc0beb34786282ULL, + 0xc824e778dde3039cULL, 0xf97d86d98a327728ULL, + 0xf2b043e24519b514ULL, 0xe297ebf7880f4b57ULL, + 0x3a94a49a98fab688ULL, 0x868516cb68f0c419ULL, + 0xeffa11af0964ee50ULL, 0xa4ab4ec0d517f37dULL, + 0xa9c6b498547c567aULL, 0x8e18424f80fbbbb6ULL, + 0x0bcdc53bcf2bc23cULL, 0x137739aaea3643d0ULL, + 0x2c1333ec1bac2ff0ULL, 0x8d48d3f0a7db0625ULL, + 0x1e1ac3f26b5de6d7ULL, 0xf520f81f16b2b95eULL, + 0x9f0f6ec450062e84ULL, 0x0130849e1deb6b71ULL, + 0xd45e31ab8c7533a9ULL, 0x652279a2fd14e43fULL, + 0x3209f01e70f1c927ULL, 0xbe71a770cac1a473ULL, + 0x0e3d6be7a64b1894ULL, 0x7ec8148cff29d840ULL, + 0xcb7476c7fac3be0fULL, 0x72956a4a63a91636ULL, + 0x37f95ec21991138fULL, 0x9e3fea5a4ded45f5ULL, + 0x7b38ba50964902e8ULL, 0x222e580bbde73764ULL, + 0x61e253e0899f55e6ULL, 0xfc8d2805e352ad80ULL, + 0x35994be3235ac56dULL, 0x09add01af5e014deULL, + 0x5e8659a6780539c6ULL, 0xb17c48097161d796ULL, + 0x026015213acbd6e2ULL, 0xd1ae9f77e515e901ULL, + 0xb7dc776a3f21b0adULL, 0xaba6a1b96eb78098ULL, + 0x9bcf4486248d9f5dULL, 0x582666c536455efdULL, + 0xfdbdac9bfeb9c6f1ULL, 0xc47999be4163cdeaULL, + 0x765540081722a7efULL, 0x3e548ed8ec710751ULL, + 0x3d041f67cb51bac2ULL, 0x7958af71ac82d40aULL, + 0x36c9da5c047a78feULL, 0xed9a048e33af38b2ULL, + 0x26ee7249c96c86bdULL, 0x900281bdeba65d61ULL, + 0x11172c8bd0fd9532ULL, 0xea0abf73600434f8ULL, + 0x42fc8f75299309f3ULL, 0x34a9cf7d3eb1ae1cULL, + 0x2b838811480723baULL, 0x5ce64c8742ceef24ULL, + 0x1adae9b01fd6570eULL, 0x3c349bf9d6bad1b3ULL, + 0x82453c891c7b75c0ULL, 0x97923a40b80d512bULL, + 0x4a61dbf1c198765cULL, 0xb48ce6d518010d3eULL, + 0xcfb45c858e480fd6ULL, 0xd933cbf30d1e96aeULL, + 0xd70ea014ab558e3aULL, 0xc189376228031742ULL, + 0x9262949cd16d8b83ULL, 0xeb3a3bed7def5f89ULL, + 0x49314a4ee6b8cbcfULL, 0xdcc3652f647e4c06ULL, + 0xda635a4c2a3e2b3dULL, 0x470c21a940f3d35bULL, + 0x315961a157d174b4ULL, 0x6672e81dda3459acULL, + 0x5b76f77a1165e36eULL, 0x445cb01667d36ec8ULL, + 0xc5491d205c88a69bULL, 0x456c34887a3805b9ULL, + 0xffddb9bac4721013ULL, 0x99af51a71e4649bfULL, + 0xa15be01cbc7729d5ULL, 0x52db2760e485f7b0ULL, + 0x8c78576eba306d54ULL, 0xae560f6507d75a30ULL, + 0x95f22f6182c687c9ULL, 0x71c5fbf54489aba5ULL, + 0xca44f259e728d57eULL, 0x88b87d2ccebbdc8dULL, + 0xbab18d32be4a15aaULL, 0x8be8ec93e99b611eULL, + 0x17b713e89ebdf209ULL, 0xb31c5d284baa0174ULL, + 0xeeca9531148f8521ULL, 0xb8d198138481c348ULL, + 0x8988f9b2d350b7fcULL, 0xb9e11c8d996aa839ULL, + 0x5a4673e40c8e881fULL, 0x1687977683569978ULL, + 0xbf4123eed72acf02ULL, 0x4ea1f1b3b513c785ULL, + 0xe767452be16f91ffULL, 0x7505d1b730021a7cULL, + 0xa59bca5ec8fc980cULL, 0xad069eda20f7e7a3ULL, + 0x38f4b1bba231606aULL, 0x60d2d77e94743e97ULL, + 0x9affc0183966f42cULL, 0x248e6768f3a7505fULL, + 0xcdd449a4b483d934ULL, 0x87b59255751baf68ULL, + 0x1bea6d2e023d3c7fULL, 0x6b1f12455b5ffcabULL, + 0x743555292de9710dULL, 0xd8034f6d10f5fddfULL, + 0xc6198c9f7ba81b08ULL, 0xbb8109aca3a17edbULL, + 0xfa2d1766ad12cabbULL, 0xc729080166437079ULL, + 0x9c5fff7b77269317ULL, 0x0000000000000000ULL, + 0x15d706c9a47624ebULL, 0x6fdf38072fd44d72ULL, + 0x5fb6dd3865ee52b7ULL, 0xa33bf53d86bcff37ULL, + 0xe657c1b5fc84fa8eULL, 0xaa962527735cebe9ULL, + 0x39c43525bfda0b1bULL, 0x204e4d2a872ce186ULL, + 0x7a083ece8ba26999ULL, 0x554b9c9db72efbfaULL, + 0xb22cd9b656416a05ULL, 0x96a2bedea5e63a5aULL, + 0x802529a826b0a322ULL, 0x8115ad363b5bc853ULL, + 0x8375b81701901eb1ULL, 0x3069e53f4a3a1fc5ULL, + 0xbd2136cfede119e0ULL, 0x18bafc91251d81ecULL, + 0x1d4a524d4c7d5b44ULL, 0x05f0aedc6960daa8ULL, + 0x29e39d3072ccf558ULL, 0x70f57f6b5962c0d4ULL, + 0x989fd53903ad22ceULL, 0xf84d024797d91c59ULL, + 0x547b1803aac5908bULL, 0xf0d056c37fd263f6ULL, + 0xd56eb535919e58d8ULL, 0x1c7ad6d351963035ULL, + 0x2e7326cd2167f912ULL, 0xac361a443d1c8cd2ULL, + 0x697f076461942a49ULL, 0x4b515f6fdc731d2dULL, + 0x8ad8680df4700a6fULL, 0x41ac1eca0eb3b460ULL, + 0x7d988533d80965d3ULL, 0xa8f6300649973d0bULL, + 0x7765c4960ac9cc9eULL, 0x7ca801adc5e20ea2ULL, + 0xdea3700e5eb59ae4ULL, 0xa06b6482a19c42a4ULL, + 0x6a2f96db46b497daULL, 0x27def6d7d487edccULL, + 0x463ca5375d18b82aULL, 0xa6cb5be1efdc259fULL, + 0x53eba3fef96e9cc1ULL, 0xce84d81b93a364a7ULL, + 0xf4107c810b59d22fULL, 0x333974806d1aa256ULL, + 0x0f0def79bba073e5ULL, 0x231edc95a00c5c15ULL, + 0xe437d494c64f2c6cULL, 0x91320523f64d3610ULL, + 0x67426c83c7df32ddULL, 0x6eefbc99323f2603ULL, + 0x9d6f7be56acdf866ULL, 0x5916e25b2bae358cULL, + 0x7ff89012e2c2b331ULL, 0x035091bf2720bd93ULL, + 0x561b0d22900e4669ULL, 0x28d319ae6f279e29ULL, + 0x2f43a2533c8c9263ULL, 0xd09e1be9f8fe8270ULL, + 0xf740ed3e2c796fbcULL, 0xdb53ded237d5404cULL, + 0x62b2c25faebfe875ULL, 0x0afd41a5d2c0a94dULL, + 0x6412fd3ce0ff8f4eULL, 0xe3a76f6995e42026ULL, + 0x6c8fa9b808f4f0e1ULL, 0xc2d9a6dd0f23aad1ULL, + 0x8f28c6d19d10d0c7ULL, 0x85d587744fd0798aULL, + 0xa20b71a39b579446ULL, 0x684f83fa7c7f4138ULL, + 0xe507500adba4471dULL, 0x3f640a46f19a6c20ULL, + 0x1247bd34f7dd28a1ULL, 0x2d23b77206474481ULL, + 0x93521002cc86e0f2ULL, 0x572b89bc8de52d18ULL, + 0xfb1d93f8b0f9a1caULL, 0xe95a2ecc4724896bULL, + 0x3ba420048511ddf9ULL, 0xd63e248ab6bee54bULL, + 0x5dd6c8195f258455ULL, 0x06a03f634e40673bULL, + 0x1f2a476c76b68da6ULL, 0x217ec9b49ac78af7ULL, + 0xecaa80102e4453c3ULL, 0x14e78257b99d4f9aULL }, + /* 3 */ + { 0x20329b2cc87bba05ULL, 0x4f5eb6f86546a531ULL, + 0xd4f44775f751b6b1ULL, 0x8266a47b850dfa8bULL, + 0xbb986aa15a6ca985ULL, 0xc979eb08f9ae0f99ULL, + 0x2da6f447a2375ea1ULL, 0x1e74275dcd7d8576ULL, + 0xbc20180a800bc5f8ULL, 0xb4a2f701b2dc65beULL, + 0xe726946f981b6d66ULL, 0x48e6c453bf21c94cULL, + 0x42cad9930f0a4195ULL, 0xefa47b64aacccd20ULL, + 0x71180a8960409a42ULL, 0x8bb3329bf6a44e0cULL, + 0xd34c35de2d36daccULL, 0xa92f5b7cbc23dc96ULL, + 0xb31a85aa68bb09c3ULL, 0x13e04836a73161d2ULL, + 0xb24dfc4129c51d02ULL, 0x8ae44b70b7da5acdULL, + 0xe671ed84d96579a7ULL, 0xa4bb3417d66f3832ULL, + 0x4572ab38d56d2de8ULL, 0xb1b47761ea47215cULL, + 0xe81c09cf70aba15dULL, 0xffbdb872ce7f90acULL, + 0xa8782297fd5dc857ULL, 0x0d946f6b6a4ce4a4ULL, + 0xe4df1f4f5b995138ULL, 0x9ebc71edca8c5762ULL, + 0x0a2c1dc0b02b88d9ULL, 0x3b503c115d9d7b91ULL, + 0xc64376a8111ec3a2ULL, 0xcec199a323c963e4ULL, + 0xdc76a87ec58616f7ULL, 0x09d596e073a9b487ULL, + 0x14583a9d7d560dafULL, 0xf4c6dc593f2a0cb4ULL, + 0xdd21d19584f80236ULL, 0x4a4836983ddde1d3ULL, + 0xe58866a41ae745f9ULL, 0xf591a5b27e541875ULL, + 0x891dc05074586693ULL, 0x5b068c651810a89eULL, + 0xa30346bc0c08544fULL, 0x3dbf3751c684032dULL, + 0x2a1e86ec785032dcULL, 0xf73f5779fca830eaULL, + 0xb60c05ca30204d21ULL, 0x0cc316802b32f065ULL, + 0x8770241bdd96be69ULL, 0xb861e18199ee95dbULL, + 0xf805cad91418fcd1ULL, 0x29e70dccbbd20e82ULL, + 0xc7140f435060d763ULL, 0x0f3a9da0e8b0cc3bULL, + 0xa2543f574d76408eULL, 0xbd7761e1c175d139ULL, + 0x4b1f4f737ca3f512ULL, 0x6dc2df1f2fc137abULL, + 0xf1d05c3967b14856ULL, 0xa742bf3715ed046cULL, + 0x654030141d1697edULL, 0x07b872abda676c7dULL, + 0x3ce84eba87fa17ecULL, 0xc1fb0403cb79afdfULL, + 0x3e46bc7105063f73ULL, 0x278ae987121cd678ULL, + 0xa1adb4778ef47cd0ULL, 0x26dd906c5362c2b9ULL, + 0x05168060589b44e2ULL, 0xfbfc41f9d79ac08fULL, + 0x0e6de44ba9ced8faULL, 0x9feb08068bf243a3ULL, + 0x7b341749d06b129bULL, 0x229c69e74a87929aULL, + 0xe09ee6c4427c011bULL, 0x5692e30e725c4c3aULL, + 0xda99a33e5e9f6e4bULL, 0x353dd85af453a36bULL, + 0x25241b4c90e0fee7ULL, 0x5de987258309d022ULL, + 0xe230140fc0802984ULL, 0x93281e86a0c0b3c6ULL, + 0xf229d719a4337408ULL, 0x6f6c2dd4ad3d1f34ULL, + 0x8ea5b2fbae3f0aeeULL, 0x8331dd90c473ee4aULL, + 0x346aa1b1b52db7aaULL, 0xdf8f235e06042aa9ULL, + 0xcc6f6b68a1354b7bULL, 0x6c95a6f46ebf236aULL, + 0x52d31a856bb91c19ULL, 0x1a35ded6d498d555ULL, + 0xf37eaef2e54d60c9ULL, 0x72e181a9a3c2a61cULL, + 0x98537aad51952fdeULL, 0x16f6c856ffaa2530ULL, + 0xd960281e9d1d5215ULL, 0x3a0745fa1ce36f50ULL, + 0x0b7b642bf1559c18ULL, 0x59a87eae9aec8001ULL, + 0x5e100c05408bec7cULL, 0x0441f98b19e55023ULL, + 0xd70dcc5534d38aefULL, 0x927f676de1bea707ULL, + 0x9769e70db925e3e5ULL, 0x7a636ea29115065aULL, + 0x468b201816ef11b6ULL, 0xab81a9b73edff409ULL, + 0xc0ac7de88a07bb1eULL, 0x1f235eb68c0391b7ULL, + 0x6056b074458dd30fULL, 0xbe8eeac102f7ed67ULL, + 0xcd381283e04b5fbaULL, 0x5cbefecec277c4e3ULL, + 0xd21b4c356c48ce0dULL, 0x1019c31664b35d8cULL, + 0x247362a7d19eea26ULL, 0xebe582efb3299d03ULL, + 0x02aef2cb82fc289fULL, 0x86275df09ce8aaa8ULL, + 0x28b07427faac1a43ULL, 0x38a9b7319e1f47cfULL, + 0xc82e92e3b8d01b58ULL, 0x06ef0b409b1978bcULL, + 0x62f842bfc771fb90ULL, 0x9904034610eb3b1fULL, + 0xded85ab5477a3e68ULL, 0x90d195a663428f98ULL, + 0x5384636e2ac708d8ULL, 0xcbd719c37b522706ULL, + 0xae9729d76644b0ebULL, 0x7c8c65e20a0c7ee6ULL, + 0x80c856b007f1d214ULL, 0x8c0b40302cc32271ULL, + 0xdbcedad51fe17a8aULL, 0x740e8ae938dbdea0ULL, + 0xa615c6dc549310adULL, 0x19cc55f6171ae90bULL, + 0x49b1bdb8fe5fdd8dULL, 0xed0a89af2830e5bfULL, + 0x6a7aadb4f5a65bd6ULL, 0x7e22972988f05679ULL, + 0xf952b3325566e810ULL, 0x39fecedadf61530eULL, + 0x6101c99f04f3c7ceULL, 0x2e5f7f6761b562ffULL, + 0xf08725d226cf5c97ULL, 0x63af3b54860fef51ULL, + 0x8ff2cb10ef411e2fULL, 0x884ab9bb35267252ULL, + 0x4df04433e7ba8daeULL, 0x9afd8866d3690741ULL, + 0x66b9bb34de94abb3ULL, 0x9baaf18d92171380ULL, + 0x543c11c5f0a064a5ULL, 0x17a1b1bdbed431f1ULL, + 0xb5f58eeaf3a2717fULL, 0xc355f6c849858740ULL, + 0xec5df044694ef17eULL, 0xd83751f5dc6346d4ULL, + 0xfc4433520dfdacf2ULL, 0x0000000000000000ULL, + 0x5a51f58e596ebc5fULL, 0x3285aaf12e34cf16ULL, + 0x8d5c39db6dbd36b0ULL, 0x12b731dde64f7513ULL, + 0x94906c2d7aa7dfbbULL, 0x302b583aacc8e789ULL, + 0x9d45facd090e6b3cULL, 0x2165e2c78905aec4ULL, + 0x68d45f7f775a7349ULL, 0x189b2c1d5664fdcaULL, + 0xe1c99f2f030215daULL, 0x6983269436246788ULL, + 0x8489af3b1e148237ULL, 0xe94b702431d5b59cULL, + 0x33d2d31a6f4adbd7ULL, 0xbfd9932a4389f9a6ULL, + 0xb0e30e8aab39359dULL, 0xd1e2c715afcaf253ULL, + 0x150f43763c28196eULL, 0xc4ed846393e2eb3dULL, + 0x03f98b20c3823c5eULL, 0xfd134ab94c83b833ULL, + 0x556b682eb1de7064ULL, 0x36c4537a37d19f35ULL, + 0x7559f30279a5ca61ULL, 0x799ae58252973a04ULL, + 0x9c12832648707ffdULL, 0x78cd9c6913e92ec5ULL, + 0x1d8dac7d0effb928ULL, 0x439da0784e745554ULL, + 0x413352b3cc887dcbULL, 0xbacf134a1b12bd44ULL, + 0x114ebafd25cd494dULL, 0x2f08068c20cb763eULL, + 0x76a07822ba27f63fULL, 0xeab2fb04f25789c2ULL, + 0xe3676de481fe3d45ULL, 0x1b62a73d95e6c194ULL, + 0x641749ff5c68832cULL, 0xa5ec4dfc97112cf3ULL, + 0xf6682e92bdd6242bULL, 0x3f11c59a44782bb2ULL, + 0x317c21d1edb6f348ULL, 0xd65ab5be75ad9e2eULL, + 0x6b2dd45fb4d84f17ULL, 0xfaab381296e4d44eULL, + 0xd0b5befeeeb4e692ULL, 0x0882ef0b32d7a046ULL, + 0x512a91a5a83b2047ULL, 0x963e9ee6f85bf724ULL, + 0x4e09cf132438b1f0ULL, 0x77f701c9fb59e2feULL, + 0x7ddb1c094b726a27ULL, 0x5f4775ee01f5f8bdULL, + 0x9186ec4d223c9b59ULL, 0xfeeac1998f01846dULL, + 0xac39db1ce4b89874ULL, 0xb75b7c21715e59e0ULL, + 0xafc0503c273aa42aULL, 0x6e3b543fec430bf5ULL, + 0x704f7362213e8e83ULL, 0x58ff0745db9294c0ULL, + 0x67eec2df9feabf72ULL, 0xa0facd9ccf8a6811ULL, + 0xb936986ad890811aULL, 0x95c715c63bd9cb7aULL, + 0xca8060283a2c33c7ULL, 0x507de84ee9453486ULL, + 0x85ded6d05f6a96f6ULL, 0x1cdad5964f81ade9ULL, + 0xd5a33e9eb62fa270ULL, 0x40642b588df6690aULL, + 0x7f75eec2c98e42b8ULL, 0x2cf18dace3494a60ULL, + 0x23cb100c0bf9865bULL, 0xeef3028febb2d9e1ULL, + 0x4425d2d394133929ULL, 0xaad6d05c7fa1e0c8ULL, + 0xad6ea2f7a5c68cb5ULL, 0xc2028f2308fb9381ULL, + 0x819f2f5b468fc6d5ULL, 0xc5bafd88d29cfffcULL, + 0x47dc59f357910577ULL, 0x2b49ff07392e261dULL, + 0x57c59ae5332258fbULL, 0x73b6f842e2bcb2ddULL, + 0xcf96e04862b77725ULL, 0x4ca73dd8a6c4996fULL, + 0x015779eb417e14c1ULL, 0x37932a9176af8bf4ULL }, + /* 4 */ + { 0x190a2c9b249df23eULL, 0x2f62f8b62263e1e9ULL, + 0x7a7f754740993655ULL, 0x330b7ba4d5564d9fULL, + 0x4c17a16a46672582ULL, 0xb22f08eb7d05f5b8ULL, + 0x535f47f40bc148ccULL, 0x3aec5d27d4883037ULL, + 0x10ed0a1825438f96ULL, 0x516101f72c233d17ULL, + 0x13cc6f949fd04eaeULL, 0x739853c441474bfdULL, + 0x653793d90d3f5b1bULL, 0x5240647b96b0fc2fULL, + 0x0c84890ad27623e0ULL, 0xd7189b32703aaea3ULL, + 0x2685de3523bd9c41ULL, 0x99317c5b11bffefaULL, + 0x0d9baa854f079703ULL, 0x70b93648fbd48ac5ULL, + 0xa80441fce30bc6beULL, 0x7287704bdc36ff1eULL, + 0xb65384ed33dc1f13ULL, 0xd36417343ee34408ULL, + 0x39cd38ab6e1bf10fULL, 0x5ab861770a1f3564ULL, + 0x0ebacf09f594563bULL, 0xd04572b884708530ULL, + 0x3cae9722bdb3af47ULL, 0x4a556b6f2f5cbaf2ULL, + 0xe1704f1f76c4bd74ULL, 0x5ec4ed7144c6dfcfULL, + 0x16afc01d4c7810e6ULL, 0x283f113cd629ca7aULL, + 0xaf59a8761741ed2dULL, 0xeed5a3991e215facULL, + 0x3bf37ea849f984d4ULL, 0xe413e096a56ce33cULL, + 0x2c439d3a98f020d1ULL, 0x637559dc6404c46bULL, + 0x9e6c95d1e5f5d569ULL, 0x24bb9836045fe99aULL, + 0x44efa466dac8ecc9ULL, 0xc6eab2a5c80895d6ULL, + 0x803b50c035220cc4ULL, 0x0321658cba93c138ULL, + 0x8f9ebc465dc7ee1cULL, 0xd15a5137190131d3ULL, + 0x0fa5ec8668e5e2d8ULL, 0x91c979578d1037b1ULL, + 0x0642ca05693b9f70ULL, 0xefca80168350eb4fULL, + 0x38d21b24f36a45ecULL, 0xbeab81e1af73d658ULL, + 0x8cbfd9cae7542f24ULL, 0xfd19cc0d81f11102ULL, + 0x0ac6430fbb4dbc90ULL, 0x1d76a09d6a441895ULL, + 0x2a01573ff1cbbfa1ULL, 0xb572e161894fde2bULL, + 0x8124734fa853b827ULL, 0x614b1fdf43e6b1b0ULL, + 0x68ac395c4238cc18ULL, 0x21d837bfd7f7b7d2ULL, + 0x20c714304a860331ULL, 0x5cfaab726324aa14ULL, + 0x74c5ba4eb50d606eULL, 0xf3a3030474654739ULL, + 0x23e671bcf015c209ULL, 0x45f087e947b9582aULL, + 0xd8bd77b418df4c7bULL, 0xe06f6c90ebb50997ULL, + 0x0bd96080263c0873ULL, 0x7e03f9410e40dcfeULL, + 0xb8e94be4c6484928ULL, 0xfb5b0608e8ca8e72ULL, + 0x1a2b49179e0e3306ULL, 0x4e29e76961855059ULL, + 0x4f36c4e6fcf4e4baULL, 0x49740ee395cf7bcaULL, + 0xc2963ea386d17f7dULL, 0x90d65ad810618352ULL, + 0x12d34c1b02a1fa4dULL, 0xfa44258775bb3a91ULL, + 0x18150f14b9ec46ddULL, 0x1491861e6b9a653dULL, + 0x9a1019d7ab2c3fc2ULL, 0x3668d42d06fe13d7ULL, + 0xdcc1fbb25606a6d0ULL, 0x969490dd795a1c22ULL, + 0x3549b1a1bc6dd2efULL, 0xc94f5e23a0ed770eULL, + 0xb9f6686b5b39fdcbULL, 0xc4d4f4a6efeae00dULL, + 0xe732851a1fff2204ULL, 0x94aad6de5eb869f9ULL, + 0x3f8ff2ae07206e7fULL, 0xfe38a9813b62d03aULL, + 0xa7a1ad7a8bee2466ULL, 0x7b6056c8dde882b6ULL, + 0x302a1e286fc58ca7ULL, 0x8da0fa457a259bc7ULL, + 0xb3302b64e074415bULL, 0x5402ae7eff8b635fULL, + 0x08f8050c9cafc94bULL, 0xae468bf98a3059ceULL, + 0x88c355cca98dc58fULL, 0xb10e6d67c7963480ULL, + 0xbad70de7e1aa3cf3ULL, 0xbfb4a26e320262bbULL, + 0xcb711820870f02d5ULL, 0xce12b7a954a75c9dULL, + 0x563ce87dd8691684ULL, 0x9f73b65e7884618aULL, + 0x2b1e74b06cba0b42ULL, 0x47cec1ea605b2df1ULL, + 0x1c698312f735ac76ULL, 0x5fdbcefed9b76b2cULL, + 0x831a354c8fb1cdfcULL, 0x820516c312c0791fULL, + 0xb74ca762aeadabf0ULL, 0xfc06ef821c80a5e1ULL, + 0x5723cbf24518a267ULL, 0x9d4df05d5f661451ULL, + 0x588627742dfd40bfULL, 0xda8331b73f3d39a0ULL, + 0x17b0e392d109a405ULL, 0xf965400bcf28fba9ULL, + 0x7c3dbf4229a2a925ULL, 0x023e460327e275dbULL, + 0x6cd0b55a0ce126b3ULL, 0xe62da695828e96e7ULL, + 0x42ad6e63b3f373b9ULL, 0xe50cc319381d57dfULL, + 0xc5cbd729729b54eeULL, 0x46d1e265fd2a9912ULL, + 0x6428b056904eeff8ULL, 0x8be23040131e04b7ULL, + 0x6709d5da2add2ec0ULL, 0x075de98af44a2b93ULL, + 0x8447dcc67bfbe66fULL, 0x6616f655b7ac9a23ULL, + 0xd607b8bded4b1a40ULL, 0x0563af89d3a85e48ULL, + 0x3db1b4ad20c21ba4ULL, 0x11f22997b8323b75ULL, + 0x292032b34b587e99ULL, 0x7f1cdace9331681dULL, + 0x8e819fc9c0b65affULL, 0xa1e3677fe2d5bb16ULL, + 0xcd33d225ee349da5ULL, 0xd9a2543b85aef898ULL, + 0x795e10cbfa0af76dULL, 0x25a4bbb9992e5d79ULL, + 0x78413344677b438eULL, 0xf0826688cef68601ULL, + 0xd27b34bba392f0ebULL, 0x551d8df162fad7bcULL, + 0x1e57c511d0d7d9adULL, 0xdeffbdb171e4d30bULL, + 0xf4feea8e802f6caaULL, 0xa480c8f6317de55eULL, + 0xa0fc44f07fa40ff5ULL, 0x95b5f551c3c9dd1aULL, + 0x22f952336d6476eaULL, 0x0000000000000000ULL, + 0xa6be8ef5169f9085ULL, 0xcc2cf1aa73452946ULL, + 0x2e7ddb39bf12550aULL, 0xd526dd3157d8db78ULL, + 0x486b2d6c08becf29ULL, 0x9b0f3a58365d8b21ULL, + 0xac78cdfaadd22c15ULL, 0xbc95c7e28891a383ULL, + 0x6a927f5f65dab9c3ULL, 0xc3891d2c1ba0cb9eULL, + 0xeaa92f9f50f8b507ULL, 0xcf0d9426c9d6e87eULL, + 0xca6e3baf1a7eb636ULL, 0xab25247059980786ULL, + 0x69b31ad3df4978fbULL, 0xe2512a93cc577c4cULL, + 0xff278a0ea61364d9ULL, 0x71a615c766a53e26ULL, + 0x89dc764334fc716cULL, 0xf87a638452594f4aULL, + 0xf2bc208be914f3daULL, 0x8766b94ac1682757ULL, + 0xbbc82e687cdb8810ULL, 0x626a7a53f9757088ULL, + 0xa2c202f358467a2eULL, 0x4d0882e5db169161ULL, + 0x09e7268301de7da8ULL, 0xe897699c771ac0dcULL, + 0xc8507dac3d9cc3edULL, 0xc0a878a0a1330aa6ULL, + 0x978bb352e42ba8c1ULL, 0xe9884a13ea6b743fULL, + 0x279afdbabecc28a2ULL, 0x047c8c064ed9eaabULL, + 0x507e2278b15289f4ULL, 0x599904fbb08cf45cULL, + 0xbd8ae46d15e01760ULL, 0x31353da7f2b43844ULL, + 0x8558ff49e68a528cULL, 0x76fbfc4d92ef15b5ULL, + 0x3456922e211c660cULL, 0x86799ac55c1993b4ULL, + 0x3e90d1219a51da9cULL, 0x2d5cbeb505819432ULL, + 0x982e5fd48cce4a19ULL, 0xdb9c1238a24c8d43ULL, + 0xd439febecaa96f9bULL, 0x418c0bef0960b281ULL, + 0x158ea591f6ebd1deULL, 0x1f48e69e4da66d4eULL, + 0x8afd13cf8e6fb054ULL, 0xf5e1c9011d5ed849ULL, + 0xe34e091c5126c8afULL, 0xad67ee7530a398f6ULL, + 0x43b24dec2e82c75aULL, 0x75da99c1287cd48dULL, + 0x92e81cdb3783f689ULL, 0xa3dd217cc537cecdULL, + 0x60543c50de970553ULL, 0x93f73f54aaf2426aULL, + 0xa91b62737e7a725dULL, 0xf19d4507538732e2ULL, + 0x77e4dfc20f9ea156ULL, 0x7d229ccdb4d31dc6ULL, + 0x1b346a98037f87e5ULL, 0xedf4c615a4b29e94ULL, + 0x4093286094110662ULL, 0xb0114ee85ae78063ULL, + 0x6ff1d0d6b672e78bULL, 0x6dcf96d591909250ULL, + 0xdfe09e3eec9567e8ULL, 0x3214582b4827f97cULL, + 0xb46dc2ee143e6ac8ULL, 0xf6c0ac8da7cd1971ULL, + 0xebb60c10cd8901e4ULL, 0xf7df8f023abcad92ULL, + 0x9c52d3d2c217a0b2ULL, 0x6b8d5cd0f8ab0d20ULL, + 0x3777f7a29b8fa734ULL, 0x011f238f9d71b4e3ULL, + 0xc1b75b2f3c42be45ULL, 0x5de588fdfe551ef7ULL, + 0x6eeef3592b035368ULL, 0xaa3a07ffc4e9b365ULL, + 0xecebe59a39c32a77ULL, 0x5ba742f8976e8187ULL, + 0x4b4a48e0b22d0e11ULL, 0xddded83dcb771233ULL, + 0xa59feb79ac0c51bdULL, 0xc7f5912a55792135ULL }, + /* 5 */ + { 0x6d6ae04668a9b08aULL, 0x3ab3f04b0be8c743ULL, + 0xe51e166b54b3c908ULL, 0xbe90a9eb35c2f139ULL, + 0xb2c7066637f2bec1ULL, 0xaa6945613392202cULL, + 0x9a28c36f3b5201ebULL, 0xddce5a93ab536994ULL, + 0x0e34133ef6382827ULL, 0x52a02ba1ec55048bULL, + 0xa2f88f97c4b2a177ULL, 0x8640e513ca2251a5ULL, + 0xcdf1d36258137622ULL, 0xfe6cb708dedf8ddbULL, + 0x8a174a9ec8121e5dULL, 0x679896036b81560eULL, + 0x59ed033395795feeULL, 0x1dd778ab8b74edafULL, + 0xee533ef92d9f926dULL, 0x2a8c79baf8a8d8f5ULL, + 0x6bcf398e69b119f6ULL, 0xe20491742fafdd95ULL, + 0x276488e0809c2aecULL, 0xea955b82d88f5cceULL, + 0x7102c63a99d9e0c4ULL, 0xf9763017a5c39946ULL, + 0x429fa2501f151b3dULL, 0x4659c72bea05d59eULL, + 0x984b7fdccf5a6634ULL, 0xf742232953fbb161ULL, + 0x3041860e08c021c7ULL, 0x747bfd9616cd9386ULL, + 0x4bb1367192312787ULL, 0x1b72a1638a6c44d3ULL, + 0x4a0e68a6e8359a66ULL, 0x169a5039f258b6caULL, + 0xb98a2ef44edee5a4ULL, 0xd9083fe85e43a737ULL, + 0x967f6ce239624e13ULL, 0x8874f62d3c1a7982ULL, + 0x3c1629830af06e3fULL, 0x9165ebfd427e5a8eULL, + 0xb5dd81794ceeaa5cULL, 0x0de8f15a7834f219ULL, + 0x70bd98ede3dd5d25ULL, 0xaccc9ca9328a8950ULL, + 0x56664eda1945ca28ULL, 0x221db34c0f8859aeULL, + 0x26dbd637fa98970dULL, 0x1acdffb4f068f932ULL, + 0x4585254f64090fa0ULL, 0x72de245e17d53afaULL, + 0x1546b25d7c546cf4ULL, 0x207e0ffffb803e71ULL, + 0xfaaad2732bcf4378ULL, 0xb462dfae36ea17bdULL, + 0xcf926fd1ac1b11fdULL, 0xe0672dc7dba7ba4aULL, + 0xd3fa49ad5d6b41b3ULL, 0x8ba81449b216a3bcULL, + 0x14f9ec8a0650d115ULL, 0x40fc1ee3eb1d7ce2ULL, + 0x23a2ed9b758ce44fULL, 0x782c521b14fddc7eULL, + 0x1c68267cf170504eULL, 0xbcf31558c1ca96e6ULL, + 0xa781b43b4ba6d235ULL, 0xf6fd7dfe29ff0c80ULL, + 0xb0a4bad5c3fad91eULL, 0xd199f51ea963266cULL, + 0x414340349119c103ULL, 0x5405f269ed4dadf7ULL, + 0xabd61bb649969dcdULL, 0x6813dbeae7bdc3c8ULL, + 0x65fb2ab09f8931d1ULL, 0xf1e7fae152e3181dULL, + 0xc1a67cef5a2339daULL, 0x7a4feea8e0f5bba1ULL, + 0x1e0b9acf05783791ULL, 0x5b8ebf8061713831ULL, + 0x80e53cdbcb3af8d9ULL, 0x7e898bd315e57502ULL, + 0xc6bcfbf0213f2d47ULL, 0x95a38e86b76e942dULL, + 0x092e94218d243cbaULL, 0x8339debf453622e7ULL, + 0xb11be402b9fe64ffULL, 0x57d9100d634177c9ULL, + 0xcc4e8db52217cbc3ULL, 0x3b0cae9c71ec7aa2ULL, + 0xfb158ca451cbfe99ULL, 0x2b33276d82ac6514ULL, + 0x01bf5ed77a04bde1ULL, 0xc5601994af33f779ULL, + 0x75c4a3416cc92e67ULL, 0xf3844652a6eb7fc2ULL, + 0x3487e375fdd0ef64ULL, 0x18ae430704609eedULL, + 0x4d14efb993298efbULL, 0x815a620cb13e4538ULL, + 0x125c354207487869ULL, 0x9eeea614ce42cf48ULL, + 0xce2d3106d61fac1cULL, 0xbbe99247bad6827bULL, + 0x071a871f7b1c149dULL, 0x2e4a1cc10db81656ULL, + 0x77a71ff298c149b8ULL, 0x06a5d9c80118a97cULL, + 0xad73c27e488e34b1ULL, 0x443a7b981e0db241ULL, + 0xe3bbcfa355ab6074ULL, 0x0af276450328e684ULL, + 0x73617a896dd1871bULL, 0x58525de4ef7de20fULL, + 0xb7be3dcab8e6cd83ULL, 0x19111dd07e64230cULL, + 0x842359a03e2a367aULL, 0x103f89f1f3401fb6ULL, + 0xdc710444d157d475ULL, 0xb835702334da5845ULL, + 0x4320fc876511a6dcULL, 0xd026abc9d3679b8dULL, + 0x17250eee885c0b2bULL, 0x90dab52a387ae76fULL, + 0x31fed8d972c49c26ULL, 0x89cba8fa461ec463ULL, + 0x2ff5421677bcabb7ULL, 0x396f122f85e41d7dULL, + 0xa09b332430bac6a8ULL, 0xc888e8ced7070560ULL, + 0xaeaf201ac682ee8fULL, 0x1180d7268944a257ULL, + 0xf058a43628e7a5fcULL, 0xbd4c4b8fbbce2b07ULL, + 0xa1246df34abe7b49ULL, 0x7d5569b79be9af3cULL, + 0xa9b5a705bd9efa12ULL, 0xdb6b835baa4bc0e8ULL, + 0x05793bac8f147342ULL, 0x21c1512881848390ULL, + 0xfdb0556c50d357e5ULL, 0x613d4fcb6a99ff72ULL, + 0x03dce2648e0cda3eULL, 0xe949b9e6568386f0ULL, + 0xfc0f0bbb2ad7ea04ULL, 0x6a70675913b5a417ULL, + 0x7f36d5046fe1c8e3ULL, 0x0c57af8d02304ff8ULL, + 0x32223abdfcc84618ULL, 0x0891caf6f720815bULL, + 0xa63eeaec31a26fd4ULL, 0x2507345374944d33ULL, + 0x49d28ac266394058ULL, 0xf5219f9aa7f3d6beULL, + 0x2d96fea583b4cc68ULL, 0x5a31e1571b7585d0ULL, + 0x8ed12fe53d02d0feULL, 0xdfade6205f5b0e4bULL, + 0x4cabb16ee92d331aULL, 0x04c6657bf510cea3ULL, + 0xd73c2cd6a87b8f10ULL, 0xe1d87310a1a307abULL, + 0x6cd5be9112ad0d6bULL, 0x97c032354366f3f2ULL, + 0xd4e0ceb22677552eULL, 0x0000000000000000ULL, + 0x29509bde76a402cbULL, 0xc27a9e8bd42fe3e4ULL, + 0x5ef7842cee654b73ULL, 0xaf107ecdbc86536eULL, + 0x3fcacbe784fcb401ULL, 0xd55f90655c73e8cfULL, + 0xe6c2f40fdabf1336ULL, 0xe8f6e7312c873b11ULL, + 0xeb2a0555a28be12fULL, 0xe4a148bc2eb774e9ULL, + 0x9b979db84156bc0aULL, 0x6eb60222e6a56ab4ULL, + 0x87ffbbc4b026ec44ULL, 0xc703a5275b3b90a6ULL, + 0x47e699fc9001687fULL, 0x9c8d1aa73a4aa897ULL, + 0x7cea3760e1ed12ddULL, 0x4ec80ddd1d2554c5ULL, + 0x13e36b957d4cc588ULL, 0x5d2b66486069914dULL, + 0x92b90999cc7280b0ULL, 0x517cc9c56259deb5ULL, + 0xc937b619ad03b881ULL, 0xec30824ad997f5b2ULL, + 0xa45d565fc5aa080bULL, 0xd6837201d27f32f1ULL, + 0x635ef3789e9198adULL, 0x531f75769651b96aULL, + 0x4f77530a6721e924ULL, 0x486dd4151c3dfdb9ULL, + 0x5f48dafb9461f692ULL, 0x375b011173dc355aULL, + 0x3da9775470f4d3deULL, 0x8d0dcd81b30e0ac0ULL, + 0x36e45fc609d888bbULL, 0x55baacbe97491016ULL, + 0x8cb29356c90ab721ULL, 0x76184125e2c5f459ULL, + 0x99f4210bb55edbd5ULL, 0x6f095cf59ca1d755ULL, + 0x9f51f8c3b44672a9ULL, 0x3538bda287d45285ULL, + 0x50c39712185d6354ULL, 0xf23b1885dcefc223ULL, + 0x79930ccc6ef9619fULL, 0xed8fdc9da3934853ULL, + 0xcb540aaa590bdf5eULL, 0x5c94389f1a6d2cacULL, + 0xe77daad8a0bbaed7ULL, 0x28efc5090ca0bf2aULL, + 0xbf2ff73c4fc64cd8ULL, 0xb37858b14df60320ULL, + 0xf8c96ec0dfc724a7ULL, 0x828680683f329f06ULL, + 0x941cd051cd6a29ccULL, 0xc3c5c05cae2b5e05ULL, + 0xb601631dc2e27062ULL, 0xc01922382027843bULL, + 0x24b86a840e90f0d2ULL, 0xd245177a276ffc52ULL, + 0x0f8b4de98c3c95c6ULL, 0x3e759530fef809e0ULL, + 0x0b4d2892792c5b65ULL, 0xc4df4743d5374a98ULL, + 0xa5e20888bfaeb5eaULL, 0xba56cc90c0d23f9aULL, + 0x38d04cf8ffe0a09cULL, 0x62e1adafe495254cULL, + 0x0263bcb3f40867dfULL, 0xcaeb547d230f62bfULL, + 0x6082111c109d4293ULL, 0xdad4dd8cd04f7d09ULL, + 0xefec602e579b2f8cULL, 0x1fb4c4187f7c8a70ULL, + 0xffd3e9dfa4db303aULL, 0x7bf0b07f9af10640ULL, + 0xf49ec14dddf76b5fULL, 0x8f6e713247066d1fULL, + 0x339d646a86ccfbf9ULL, 0x64447467e58d8c30ULL, + 0x2c29a072f9b07189ULL, 0xd8b7613f24471ad6ULL, + 0x6627c8d41185ebefULL, 0xa347d140beb61c96ULL, + 0xde12b8f7255fb3aaULL, 0x9d324470404e1576ULL, + 0x9306574eb6763d51ULL, 0xa80af9d2c79a47f3ULL, + 0x859c0777442e8b9bULL, 0x69ac853d9db97e29ULL }, + /* 6 */ + { 0xc3407dfc2de6377eULL, 0x5b9e93eea4256f77ULL, + 0xadb58fdd50c845e0ULL, 0x5219ff11a75bed86ULL, + 0x356b61cfd90b1de9ULL, 0xfb8f406e25abe037ULL, + 0x7a5a0231c0f60796ULL, 0x9d3cd216e1f5020bULL, + 0x0c6550fb6b48d8f3ULL, 0xf57508c427ff1c62ULL, + 0x4ad35ffa71cb407dULL, 0x6290a2da1666aa6dULL, + 0xe284ec2349355f9fULL, 0xb3c307c53d7c84ecULL, + 0x05e23c0468365a02ULL, 0x190bac4d6c9ebfa8ULL, + 0x94bbbee9e28b80faULL, 0xa34fc777529cb9b5ULL, + 0xcc7b39f095bcd978ULL, 0x2426addb0ce532e3ULL, + 0x7e79329312ce4fc7ULL, 0xab09a72eebec2917ULL, + 0xf8d15499f6b9d6c2ULL, 0x1a55b8babf8c895dULL, + 0xdb8add17fb769a85ULL, 0xb57f2f368658e81bULL, + 0x8acd36f18f3f41f6ULL, 0x5ce3b7bba50f11d3ULL, + 0x114dcc14d5ee2f0aULL, 0xb91a7fcded1030e8ULL, + 0x81d5425fe55de7a1ULL, 0xb6213bc1554adeeeULL, + 0x80144ef95f53f5f2ULL, 0x1e7688186db4c10cULL, + 0x3b912965db5fe1bcULL, 0xc281715a97e8252dULL, + 0x54a5d7e21c7f8171ULL, 0x4b12535ccbc5522eULL, + 0x1d289cefbea6f7f9ULL, 0x6ef5f2217d2e729eULL, + 0xe6a7dc819b0d17ceULL, 0x1b94b41c05829b0eULL, + 0x33d7493c622f711eULL, 0xdcf7f942fa5ce421ULL, + 0x600fba8b7f7a8ecbULL, 0x46b60f011a83988eULL, + 0x235b898e0dcf4c47ULL, 0x957ab24f588592a9ULL, + 0x4354330572b5c28cULL, 0xa5f3ef84e9b8d542ULL, + 0x8c711e02341b2d01ULL, 0x0b1874ae6a62a657ULL, + 0x1213d8e306fc19ffULL, 0xfe6d7c6a4d9dba35ULL, + 0x65ed868f174cd4c9ULL, 0x88522ea0e6236550ULL, + 0x899322065c2d7703ULL, 0xc01e690bfef4018bULL, + 0x915982ed8abddaf8ULL, 0xbe675b98ec3a4e4cULL, + 0xa996bf7f82f00db1ULL, 0xe1daf8d49a27696aULL, + 0x2effd5d3dc8986e7ULL, 0xd153a51f2b1a2e81ULL, + 0x18caa0ebd690adfbULL, 0x390e3134b243c51aULL, + 0x2778b92cdff70416ULL, 0x029f1851691c24a6ULL, + 0x5e7cafeacc133575ULL, 0xfa4e4cc89fa5f264ULL, + 0x5a5f9f481e2b7d24ULL, 0x484c47ab18d764dbULL, + 0x400a27f2a1a7f479ULL, 0xaeeb9b2a83da7315ULL, + 0x721c626879869734ULL, 0x042330a2d2384851ULL, + 0x85f672fd3765aff0ULL, 0xba446b3a3e02061dULL, + 0x73dd6ecec3888567ULL, 0xffac70ccf793a866ULL, + 0xdfa9edb5294ed2d4ULL, 0x6c6aea7014325638ULL, + 0x834a5a0e8c41c307ULL, 0xcdba35562fb2cb2bULL, + 0x0ad97808d06cb404ULL, 0x0f3b440cb85aee06ULL, + 0xe5f9c876481f213bULL, 0x98deee1289c35809ULL, + 0x59018bbfcd394bd1ULL, 0xe01bf47220297b39ULL, + 0xde68e1139340c087ULL, 0x9fa3ca4788e926adULL, + 0xbb85679c840c144eULL, 0x53d8f3b71d55ffd5ULL, + 0x0da45c5dd146caa0ULL, 0x6f34fe87c72060cdULL, + 0x57fbc315cf6db784ULL, 0xcee421a1fca0fddeULL, + 0x3d2d0196607b8d4bULL, 0x642c8a29ad42c69aULL, + 0x14aff010bdd87508ULL, 0xac74837beac657b3ULL, + 0x3216459ad821634dULL, 0x3fb219c70967a9edULL, + 0x06bc28f3bb246cf7ULL, 0xf2082c9126d562c6ULL, + 0x66b39278c45ee23cULL, 0xbd394f6f3f2878b9ULL, + 0xfd33689d9e8f8cc0ULL, 0x37f4799eb017394fULL, + 0x108cc0b26fe03d59ULL, 0xda4bd1b1417888d6ULL, + 0xb09d1332ee6eb219ULL, 0x2f3ed975668794b4ULL, + 0x58c0871977375982ULL, 0x7561463d78ace990ULL, + 0x09876cff037e82f1ULL, 0x7fb83e35a8c05d94ULL, + 0x26b9b58a65f91645ULL, 0xef20b07e9873953fULL, + 0x3148516d0b3355b8ULL, 0x41cb2b541ba9e62aULL, + 0x790416c613e43163ULL, 0xa011d380818e8f40ULL, + 0x3a5025c36151f3efULL, 0xd57095bdf92266d0ULL, + 0x498d4b0da2d97688ULL, 0x8b0c3a57353153a5ULL, + 0x21c491df64d368e1ULL, 0x8f2f0af5e7091bf4ULL, + 0x2da1c1240f9bb012ULL, 0xc43d59a92ccc49daULL, + 0xbfa6573e56345c1fULL, 0x828b56a8364fd154ULL, + 0x9a41f643e0df7cafULL, 0xbcf843c985266aeaULL, + 0x2b1de9d7b4bfdce5ULL, 0x20059d79dedd7ab2ULL, + 0x6dabe6d6ae3c446bULL, 0x45e81bf6c991ae7bULL, + 0x6351ae7cac68b83eULL, 0xa432e32253b6c711ULL, + 0xd092a9b991143cd2ULL, 0xcac711032e98b58fULL, + 0xd8d4c9e02864ac70ULL, 0xc5fc550f96c25b89ULL, + 0xd7ef8dec903e4276ULL, 0x67729ede7e50f06fULL, + 0xeac28c7af045cf3dULL, 0xb15c1f945460a04aULL, + 0x9cfddeb05bfb1058ULL, 0x93c69abce3a1fe5eULL, + 0xeb0380dc4a4bdd6eULL, 0xd20db1e8f8081874ULL, + 0x229a8528b7c15e14ULL, 0x44291750739fbc28ULL, + 0xd3ccbd4e42060a27ULL, 0xf62b1c33f4ed2a97ULL, + 0x86a8660ae4779905ULL, 0xd62e814a2a305025ULL, + 0x477703a7a08d8addULL, 0x7b9b0e977af815c5ULL, + 0x78c51a60a9ea2330ULL, 0xa6adfb733aaae3b7ULL, + 0x97e5aa1e3199b60fULL, 0x0000000000000000ULL, + 0xf4b404629df10e31ULL, 0x5564db44a6719322ULL, + 0x9207961a59afec0dULL, 0x9624a6b88b97a45cULL, + 0x363575380a192b1cULL, 0x2c60cd82b595a241ULL, + 0x7d272664c1dc7932ULL, 0x7142769faa94a1c1ULL, + 0xa1d0df263b809d13ULL, 0x1630e841d4c451aeULL, + 0xc1df65ad44fa13d8ULL, 0x13d2d445bcf20bacULL, + 0xd915c546926abe23ULL, 0x38cf3d92084dd749ULL, + 0xe766d0272103059dULL, 0xc7634d5effde7f2fULL, + 0x077d2455012a7ea4ULL, 0xedbfa82ff16fb199ULL, + 0xaf2a978c39d46146ULL, 0x42953fa3c8bbd0dfULL, + 0xcb061da59496a7dcULL, 0x25e7a17db6eb20b0ULL, + 0x34aa6d6963050fbaULL, 0xa76cf7d580a4f1e4ULL, + 0xf7ea10954ee338c4ULL, 0xfcf2643b24819e93ULL, + 0xcf252d0746aeef8dULL, 0x4ef06f58a3f3082cULL, + 0x563acfb37563a5d7ULL, 0x5086e740ce47c920ULL, + 0x2982f186dda3f843ULL, 0x87696aac5e798b56ULL, + 0x5d22bb1d1f010380ULL, 0x035e14f7d31236f5ULL, + 0x3cec0d30da759f18ULL, 0xf3c920379cdb7095ULL, + 0xb8db736b571e22bbULL, 0xdd36f5e44052f672ULL, + 0xaac8ab8851e23b44ULL, 0xa857b3d938fe1fe2ULL, + 0x17f1e4e76eca43fdULL, 0xec7ea4894b61a3caULL, + 0x9e62c6e132e734feULL, 0xd4b1991b432c7483ULL, + 0x6ad6c283af163acfULL, 0x1ce9904904a8e5aaULL, + 0x5fbda34c761d2726ULL, 0xf910583f4cb7c491ULL, + 0xc6a241f845d06d7cULL, 0x4f3163fe19fd1a7fULL, + 0xe99c988d2357f9c8ULL, 0x8eee06535d0709a7ULL, + 0x0efa48aa0254fc55ULL, 0xb4be23903c56fa48ULL, + 0x763f52caabbedf65ULL, 0xeee1bcd8227d876cULL, + 0xe345e085f33b4dccULL, 0x3e731561b369bbbeULL, + 0x2843fd2067adea10ULL, 0x2adce5710eb1ceb6ULL, + 0xb7e03767ef44ccbdULL, 0x8db012a48e153f52ULL, + 0x61ceb62dc5749c98ULL, 0xe85d942b9959eb9bULL, + 0x4c6f7709caef2c8aULL, 0x84377e5b8d6bbda3ULL, + 0x30895dcbb13d47ebULL, 0x74a04a9bc2a2fbc3ULL, + 0x6b17ce251518289cULL, 0xe438c4d0f2113368ULL, + 0x1fb784bed7bad35fULL, 0x9b80fae55ad16efcULL, + 0x77fe5e6c11b0cd36ULL, 0xc858095247849129ULL, + 0x08466059b97090a2ULL, 0x01c10ca6ba0e1253ULL, + 0x6988d6747c040c3aULL, 0x6849dad2c60a1e69ULL, + 0x5147ebe67449db73ULL, 0xc99905f4fd8a837aULL, + 0x991fe2b433cd4a5aULL, 0xf09734c04fc94660ULL, + 0xa28ecbd1e892abe6ULL, 0xf1563866f5c75433ULL, + 0x4dae7baf70e13ed9ULL, 0x7ce62ac27bd26b61ULL, + 0x70837a39109ab392ULL, 0x90988e4b30b3c8abULL, + 0xb2020b63877296bfULL, 0x156efcb607d6675bULL }, + /* 7 */ + { 0xe63f55ce97c331d0ULL, 0x25b506b0015bba16ULL, + 0xc8706e29e6ad9ba8ULL, 0x5b43d3775d521f6aULL, + 0x0bfa3d577035106eULL, 0xab95fc172afb0e66ULL, + 0xf64b63979e7a3276ULL, 0xf58b4562649dad4bULL, + 0x48f7c3dbae0c83f1ULL, 0xff31916642f5c8c5ULL, + 0xcbb048dc1c4a0495ULL, 0x66b8f83cdf622989ULL, + 0x35c130e908e2b9b0ULL, 0x7c761a61f0b34fa1ULL, + 0x3601161cf205268dULL, 0x9e54ccfe2219b7d6ULL, + 0x8b7d90a538940837ULL, 0x9cd403588ea35d0bULL, + 0xbc3c6fea9ccc5b5aULL, 0xe5ff733b6d24aeedULL, + 0xceed22de0f7eb8d2ULL, 0xec8581cab1ab545eULL, + 0xb96105e88ff8e71dULL, 0x8ca03501871a5eadULL, + 0x76ccce65d6db2a2fULL, 0x5883f582a7b58057ULL, + 0x3f7be4ed2e8adc3eULL, 0x0fe7be06355cd9c9ULL, + 0xee054e6c1d11be83ULL, 0x1074365909b903a6ULL, + 0x5dde9f80b4813c10ULL, 0x4a770c7d02b6692cULL, + 0x5379c8d5d7809039ULL, 0xb4067448161ed409ULL, + 0x5f5e5026183bd6cdULL, 0xe898029bf4c29df9ULL, + 0x7fb63c940a54d09cULL, 0xc5171f897f4ba8bcULL, + 0xa6f28db7b31d3d72ULL, 0x2e4f3be7716eaa78ULL, + 0x0d6771a099e63314ULL, 0x82076254e41bf284ULL, + 0x2f0fd2b42733df98ULL, 0x5c9e76d3e2dc49f0ULL, + 0x7aeb569619606cdbULL, 0x83478b07b2468764ULL, + 0xcfadcb8d5923cd32ULL, 0x85dac7f05b95a41eULL, + 0xb5469d1b4043a1e9ULL, 0xb821ecbbd9a592fdULL, + 0x1b8e0b0e798c13c8ULL, 0x62a57b6d9a0be02eULL, + 0xfcf1b793b81257f8ULL, 0x9d94ea0bd8fe28ebULL, + 0x4cea408aeb654a56ULL, 0x23284a47e888996cULL, + 0x2d8f1d128b893545ULL, 0xf4cbac3132c0d8abULL, + 0xbd7c86b9ca912ebaULL, 0x3a268eef3dbe6079ULL, + 0xf0d62f6077a9110cULL, 0x2735c916ade150cbULL, + 0x89fd5f03942ee2eaULL, 0x1acee25d2fd16628ULL, + 0x90f39bab41181bffULL, 0x430dfe8cde39939fULL, + 0xf70b8ac4c8274796ULL, 0x1c53aeaac6024552ULL, + 0x13b410acf35e9c9bULL, 0xa532ab4249faa24fULL, + 0x2b1251e5625a163fULL, 0xd7e3e676da4841c7ULL, + 0xa7b264e4e5404892ULL, 0xda8497d643ae72d3ULL, + 0x861ae105a1723b23ULL, 0x38a6414991048aa4ULL, + 0x6578dec92585b6b4ULL, 0x0280cfa6acbaeaddULL, + 0x88bdb650c273970aULL, 0x9333bd5ebbff84c2ULL, + 0x4e6a8f2c47dfa08bULL, 0x321c954db76cef2aULL, + 0x418d312a72837942ULL, 0xb29b38bfffcdf773ULL, + 0x6c022c38f90a4c07ULL, 0x5a033a240b0f6a8aULL, + 0x1f93885f3ce5da6fULL, 0xc38a537e96988bc6ULL, + 0x39e6a81ac759ff44ULL, 0x29929e43cee0fce2ULL, + 0x40cdd87924de0ca2ULL, 0xe9d8ebc8a29fe819ULL, + 0x0c2798f3cfbb46f4ULL, 0x55e484223e53b343ULL, + 0x4650948ecd0d2fd8ULL, 0x20e86cb2126f0651ULL, + 0x6d42c56baf5739e7ULL, 0xa06fc1405ace1e08ULL, + 0x7babbfc54f3d193bULL, 0x424d17df8864e67fULL, + 0xd8045870ef14980eULL, 0xc6d7397c85ac3781ULL, + 0x21a885e1443273b1ULL, 0x67f8116f893f5c69ULL, + 0x24f5efe35706cff6ULL, 0xd56329d076f2ab1aULL, + 0x5e1eb9754e66a32dULL, 0x28d2771098bd8902ULL, + 0x8f6013f47dfdc190ULL, 0x17a993fdb637553cULL, + 0xe0a219397e1012aaULL, 0x786b9930b5da8606ULL, + 0x6e82e39e55b0a6daULL, 0x875a0856f72f4ec3ULL, + 0x3741ff4fa458536dULL, 0xac4859b3957558fcULL, + 0x7ef6d5c75c09a57cULL, 0xc04a758b6c7f14fbULL, + 0xf9acdd91ab26ebbfULL, 0x7391a467c5ef9668ULL, + 0x335c7c1ee1319acaULL, 0xa91533b18641e4bbULL, + 0xe4bf9a683b79db0dULL, 0x8e20faa72ba0b470ULL, + 0x51f907737b3a7ae4ULL, 0x2268a314bed5ec8cULL, + 0xd944b123b949edeeULL, 0x31dcb3b84d8b7017ULL, + 0xd3fe65279f218860ULL, 0x097af2f1dc8ffab3ULL, + 0x9b09a6fc312d0b91ULL, 0xcc6ded78a3c4520fULL, + 0x3481d9ba5ebfcc50ULL, 0x4f2a667f1182d56bULL, + 0xdfd9fdd4509ace94ULL, 0x26752045fbbc252bULL, + 0xbffc491f662bc467ULL, 0xdd593272fc202449ULL, + 0x3cbbc218d46d4303ULL, 0x91b372f817456e1fULL, + 0x681faf69bc6385a0ULL, 0xb686bbeebaa43ed4ULL, + 0x1469b5084cd0ca01ULL, 0x98c98009cbca94acULL, + 0x6438379a73d8c354ULL, 0xc2caba2dc0c5fe26ULL, + 0x3e3b0dbe78d7a9deULL, 0x50b9ee202d670f04ULL, + 0x4590b27b37eab0e5ULL, 0x6025b4cb36b10af3ULL, + 0xfb2c1237079c0162ULL, 0xa12f28130c936be8ULL, + 0x4b37e52e54eb1cccULL, 0x083a1ba28ad28f53ULL, + 0xc10a9cd83a22611bULL, 0x9f1425ad7444c236ULL, + 0x069d4cf7e9d3237aULL, 0xedc56899e7f621beULL, + 0x778c273680865fcfULL, 0x309c5aeb1bd605f7ULL, + 0x8de0dc52d1472b4dULL, 0xf8ec34c2fd7b9e5fULL, + 0xea18cd3d58787724ULL, 0xaad515447ca67b86ULL, + 0x9989695a9d97e14cULL, 0x0000000000000000ULL, + 0xf196c63321f464ecULL, 0x71116bc169557cb5ULL, + 0xaf887f466f92c7c1ULL, 0x972e3e0ffe964d65ULL, + 0x190ec4a8d536f915ULL, 0x95aef1a9522ca7b8ULL, + 0xdc19db21aa7d51a9ULL, 0x94ee18fa0471d258ULL, + 0x8087adf248a11859ULL, 0xc457f6da2916dd5cULL, + 0xfa6cfb6451c17482ULL, 0xf256e0c6db13fbd1ULL, + 0x6a9f60cf10d96f7dULL, 0x4daaa9d9bd383fb6ULL, + 0x03c026f5fae79f3dULL, 0xde99148706c7bb74ULL, + 0x2a52b8b6340763dfULL, 0x6fc20acd03edd33aULL, + 0xd423c08320afdefaULL, 0xbbe1ca4e23420dc0ULL, + 0x966ed75ca8cb3885ULL, 0xeb58246e0e2502c4ULL, + 0x055d6a021334bc47ULL, 0xa47242111fa7d7afULL, + 0xe3623fcc84f78d97ULL, 0x81c744a11efc6db9ULL, + 0xaec8961539cfb221ULL, 0xf31609958d4e8e31ULL, + 0x63e5923ecc5695ceULL, 0x47107ddd9b505a38ULL, + 0xa3afe7b5a0298135ULL, 0x792b7063e387f3e6ULL, + 0x0140e953565d75e0ULL, 0x12f4f9ffa503e97bULL, + 0x750ce8902c3cb512ULL, 0xdbc47e8515f30733ULL, + 0x1ed3610c6ab8af8fULL, 0x5239218681dde5d9ULL, + 0xe222d69fd2aaf877ULL, 0xfe71783514a8bd25ULL, + 0xcaf0a18f4a177175ULL, 0x61655d9860ec7f13ULL, + 0xe77fbc9dc19e4430ULL, 0x2ccff441ddd440a5ULL, + 0x16e97aaee06a20dcULL, 0xa855dae2d01c915bULL, + 0x1d1347f9905f30b2ULL, 0xb7c652bdecf94b34ULL, + 0xd03e43d265c6175dULL, 0xfdb15ec0ee4f2218ULL, + 0x57644b8492e9599eULL, 0x07dda5a4bf8e569aULL, + 0x54a46d71680ec6a3ULL, 0x5624a2d7c4b42c7eULL, + 0xbebca04c3076b187ULL, 0x7d36f332a6ee3a41ULL, + 0x3b6667bc6be31599ULL, 0x695f463aea3ef040ULL, + 0xad08b0e0c3282d1cULL, 0xb15b1e4a052a684eULL, + 0x44d05b2861b7c505ULL, 0x15295c5b1a8dbfe1ULL, + 0x744c01c37a61c0f2ULL, 0x59c31cd1f1e8f5b7ULL, + 0xef45a73f4b4ccb63ULL, 0x6bdf899c46841a9dULL, + 0x3dfb2b4b823036e3ULL, 0xa2ef0ee6f674f4d5ULL, + 0x184e2dfb836b8cf5ULL, 0x1134df0a5fe47646ULL, + 0xbaa1231d751f7820ULL, 0xd17eaa81339b62bdULL, + 0xb01bf71953771daeULL, 0x849a2ea30dc8d1feULL, + 0x705182923f080955ULL, 0x0ea757556301ac29ULL, + 0x041d83514569c9a7ULL, 0x0abad4042668658eULL, + 0x49b72a88f851f611ULL, 0x8a3d79f66ec97dd7ULL, + 0xcd2d042bf59927efULL, 0xc930877ab0f0ee48ULL, + 0x9273540deda2f122ULL, 0xc797d02fd3f14261ULL, + 0xe1e2f06a284d674aULL, 0xd2be8c74c97cfd80ULL, + 0x9a494faf67707e71ULL, 0xb3dbd1eca9908293ULL, + 0x72d14d3493b2e388ULL, 0xd6a30f258c153427ULL }, +}; + +static const uint64_t C16[12][8] = +{ + { 0xdd806559f2a64507ULL, 0x05767436cc744d23ULL, + 0xa2422a08a460d315ULL, 0x4b7ce09192676901ULL, + 0x714eb88d7585c4fcULL, 0x2f6a76432e45d016ULL, + 0xebcb2f81c0657c1fULL, 0xb1085bda1ecadae9ULL }, + { 0xe679047021b19bb7ULL, 0x55dda21bd7cbcd56ULL, + 0x5cb561c2db0aa7caULL, 0x9ab5176b12d69958ULL, + 0x61d55e0f16b50131ULL, 0xf3feea720a232b98ULL, + 0x4fe39d460f70b5d7ULL, 0x6fa3b58aa99d2f1aULL }, + { 0x991e96f50aba0ab2ULL, 0xc2b6f443867adb31ULL, + 0xc1c93a376062db09ULL, 0xd3e20fe490359eb1ULL, + 0xf2ea7514b1297b7bULL, 0x06f15e5f529c1f8bULL, + 0x0a39fc286a3d8435ULL, 0xf574dcac2bce2fc7ULL }, + { 0x220cbebc84e3d12eULL, 0x3453eaa193e837f1ULL, + 0xd8b71333935203beULL, 0xa9d72c82ed03d675ULL, + 0x9d721cad685e353fULL, 0x488e857e335c3c7dULL, + 0xf948e1a05d71e4ddULL, 0xef1fdfb3e81566d2ULL }, + { 0x601758fd7c6cfe57ULL, 0x7a56a27ea9ea63f5ULL, + 0xdfff00b723271a16ULL, 0xbfcd1747253af5a3ULL, + 0x359e35d7800fffbdULL, 0x7f151c1f1686104aULL, + 0x9a3f410c6ca92363ULL, 0x4bea6bacad474799ULL }, + { 0xfa68407a46647d6eULL, 0xbf71c57236904f35ULL, + 0x0af21f66c2bec6b6ULL, 0xcffaa6b71c9ab7b4ULL, + 0x187f9ab49af08ec6ULL, 0x2d66c4f95142a46cULL, + 0x6fa4c33b7a3039c0ULL, 0xae4faeae1d3ad3d9ULL }, + { 0x8886564d3a14d493ULL, 0x3517454ca23c4af3ULL, + 0x06476983284a0504ULL, 0x0992abc52d822c37ULL, + 0xd3473e33197a93c9ULL, 0x399ec6c7e6bf87c9ULL, + 0x51ac86febf240954ULL, 0xf4c70e16eeaac5ecULL }, + { 0xa47f0dd4bf02e71eULL, 0x36acc2355951a8d9ULL, + 0x69d18d2bd1a5c42fULL, 0xf4892bcb929b0690ULL, + 0x89b4443b4ddbc49aULL, 0x4eb7f8719c36de1eULL, + 0x03e7aa020c6e4141ULL, 0x9b1f5b424d93c9a7ULL }, + { 0x7261445183235adbULL, 0x0e38dc92cb1f2a60ULL, + 0x7b2b8a9aa6079c54ULL, 0x800a440bdbb2ceb1ULL, + 0x3cd955b7e00d0984ULL, 0x3a7d3a1b25894224ULL, + 0x944c9ad8ec165fdeULL, 0x378f5a541631229bULL }, + { 0x74b4c7fb98459cedULL, 0x3698fad1153bb6c3ULL, + 0x7a1e6c303b7652f4ULL, 0x9fe76702af69334bULL, + 0x1fffe18a1b336103ULL, 0x8941e71cff8a78dbULL, + 0x382ae548b2e4f3f3ULL, 0xabbedea680056f52ULL }, + { 0x6bcaa4cd81f32d1bULL, 0xdea2594ac06fd85dULL, + 0xefbacd1d7d476e98ULL, 0x8a1d71efea48b9caULL, + 0x2001802114846679ULL, 0xd8fa6bbbebab0761ULL, + 0x3002c6cd635afe94ULL, 0x7bcd9ed0efc889fbULL }, + { 0x48bc924af11bd720ULL, 0xfaf417d5d9b21b99ULL, + 0xe71da4aa88e12852ULL, 0x5d80ef9d1891cc86ULL, + 0xf82012d430219f9bULL, 0xcda43c32bcdf1d77ULL, + 0xd21380b00449b17aULL, 0x378ee767f11631baULL }, +}; + +#define strido(out, temp, i) do { \ + uint64_t t; \ + t = streebog_table[0][(temp[0] >> (i * 8)) & 0xff]; \ + t ^= streebog_table[1][(temp[1] >> (i * 8)) & 0xff]; \ + t ^= streebog_table[2][(temp[2] >> (i * 8)) & 0xff]; \ + t ^= streebog_table[3][(temp[3] >> (i * 8)) & 0xff]; \ + t ^= streebog_table[4][(temp[4] >> (i * 8)) & 0xff]; \ + t ^= streebog_table[5][(temp[5] >> (i * 8)) & 0xff]; \ + t ^= streebog_table[6][(temp[6] >> (i * 8)) & 0xff]; \ + t ^= streebog_table[7][(temp[7] >> (i * 8)) & 0xff]; \ + out[i] = t; } while(0) + +static void LPSX (uint64_t *out, const uint64_t *a, const uint64_t *b) +{ + uint64_t temp[8]; + temp[0] = a[0] ^ b[0]; + temp[1] = a[1] ^ b[1]; + temp[2] = a[2] ^ b[2]; + temp[3] = a[3] ^ b[3]; + temp[4] = a[4] ^ b[4]; + temp[5] = a[5] ^ b[5]; + temp[6] = a[6] ^ b[6]; + temp[7] = a[7] ^ b[7]; + strido (out, temp, 0); + strido (out, temp, 1); + strido (out, temp, 2); + strido (out, temp, 3); + strido (out, temp, 4); + strido (out, temp, 5); + strido (out, temp, 6); + strido (out, temp, 7); +} + +static inline void g (uint64_t *h, uint64_t *m, uint64_t *N) +{ + uint64_t K[8]; + uint64_t T[8]; + int i; + + LPSX (K, h, N); + + LPSX (T, K, m); + LPSX (K, K, C16[0]); + for (i = 1; i < 12; i++) + { + LPSX (T, K, T); + LPSX (K, K, C16[i]); + } + + h[0] ^= T[0] ^ K[0] ^ m[0]; + h[1] ^= T[1] ^ K[1] ^ m[1]; + h[2] ^= T[2] ^ K[2] ^ m[2]; + h[3] ^= T[3] ^ K[3] ^ m[3]; + h[4] ^= T[4] ^ K[4] ^ m[4]; + h[5] ^= T[5] ^ K[5] ^ m[5]; + h[6] ^= T[6] ^ K[6] ^ m[6]; + h[7] ^= T[7] ^ K[7] ^ m[7]; +} + + +static void +streebog512_compress (struct streebog512_ctx *ctx, const uint8_t *input, size_t count) +{ + uint64_t M[8]; + uint64_t l, cf; + int i; + + for (i = 0; i < 8; i++, input += 8) + M[i] = LE_READ_UINT64(input); + + g (ctx->state, M, ctx->count); + l = ctx->count[0]; + ctx->count[0] += count; + if (ctx->count[0] < l) + { /* overflow */ + for (i = 1; i < 8; i++) + { + ctx->count[i]++; + if (ctx->count[i] != 0) + break; + } + } + + cf = 0; + ctx->sigma[0] += M[0]; + for (i = 1; i < 8; i++) + { + if (ctx->sigma[i-1] != M[i-1]) + cf = (ctx->sigma[i-1] < M[i-1]); + ctx->sigma[i] += M[i] + cf; + } +} + +static void +streebog_final (struct streebog512_ctx *ctx) +{ + uint64_t Z[8] = {}; + unsigned int i; + + /* PAD. It does not count towards message length */ + i = ctx->index; + /* We have at least one byte free) */ + ctx->block[i++] = 1; + while (i < 64) + ctx->block[i++] = 0; + streebog512_compress (ctx, ctx->block, ctx->index * 8); + + g (ctx->state, ctx->count, Z); + g (ctx->state, ctx->sigma, Z); +} + +#define COMPRESS(ctx, data) (streebog512_compress((ctx), (data), 64 * 8)) + +void +streebog512_init(struct streebog512_ctx *ctx) +{ + memset(ctx->state, 0, sizeof(ctx->state)); + memset(ctx->count, 0, sizeof(ctx->count)); + memset(ctx->sigma, 0, sizeof(ctx->sigma)); + + /* Initialize buffer */ + ctx->index = 0; +} + +void +streebog512_update(struct streebog512_ctx *ctx, + size_t length, const uint8_t *data) +{ + MD_UPDATE (ctx, length, data, COMPRESS, (void)0); +} + +static void +streebog512_write_digest(struct streebog512_ctx *ctx, + size_t offset, size_t length, + uint8_t *digest) +{ + unsigned i; + unsigned words; + unsigned leftover; + + assert(offset + length <= STREEBOG512_DIGEST_SIZE); + + streebog_final(ctx); + + words = length / 8; + leftover = length % 8; + + for (i = 0; i < words; i++, digest += 8) + LE_WRITE_UINT64(digest, ctx->state[offset + i]); + + if (leftover) + { + /* Truncate to the right size */ + uint64_t word = ctx->state[offset + i] << (8*(8 - leftover)); + + do { + digest[--leftover] = (word >> 56) & 0xff; + word <<= 8; + } while (leftover); + } +} + +void +streebog512_digest(struct streebog512_ctx *ctx, + size_t length, + uint8_t *digest) +{ + assert(length <= STREEBOG512_DIGEST_SIZE); + + streebog512_write_digest(ctx, 0, length, digest); + streebog512_init(ctx); +} + +void +streebog256_init(struct streebog256_ctx *ctx) +{ + memset(ctx->state, 1, sizeof(ctx->state)); + memset(ctx->count, 0, sizeof(ctx->count)); + memset(ctx->sigma, 0, sizeof(ctx->sigma)); + + /* Initialize buffer */ + ctx->index = 0; +} + +void +streebog256_digest(struct streebog256_ctx *ctx, + size_t length, + uint8_t *digest) +{ + assert(length <= STREEBOG256_DIGEST_SIZE); + + streebog512_write_digest(ctx, + 4, + length, + digest); + streebog256_init(ctx); +} diff --git a/streebog.h b/streebog.h new file mode 100644 index 000000000000..69ac0d5c2746 --- /dev/null +++ b/streebog.h @@ -0,0 +1,99 @@ +/* streebog.h + + The Streebog family of hash functions. + + Copyright (C) 2020 Dmitry Baryshkov + + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +*/ + +#ifndef NETTLE_STREEBOG_H_INCLUDED +#define NETTLE_STREEBOG_H_INCLUDED + +#include "nettle-types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Name mangling */ +#define streebog256_init nettle_streebog256_init +#define streebog256_digest nettle_streebog256_digest +#define streebog512_init nettle_streebog512_init +#define streebog512_update nettle_streebog512_update +#define streebog512_digest nettle_streebog512_digest + +/* STREEBOG512 */ + +#define STREEBOG512_DIGEST_SIZE 64 +#define STREEBOG512_BLOCK_SIZE 64 + +/* Digest is kept internally as 8 64-bit words. */ +#define _STREEBOG512_DIGEST_LENGTH 8 + +struct streebog512_ctx +{ + uint64_t state[_STREEBOG512_DIGEST_LENGTH]; /* State variables */ + uint64_t count[_STREEBOG512_DIGEST_LENGTH]; + uint64_t sigma[_STREEBOG512_DIGEST_LENGTH]; + unsigned int index; /* index into buffer */ + uint8_t block[STREEBOG512_BLOCK_SIZE]; /* STREEBOG512 data buffer */ +}; + +void +streebog512_init(struct streebog512_ctx *ctx); + +void +streebog512_update(struct streebog512_ctx *ctx, + size_t length, + const uint8_t *data); + +void +streebog512_digest(struct streebog512_ctx *ctx, + size_t length, + uint8_t *digest); + + +#define STREEBOG256_DIGEST_SIZE 32 +#define STREEBOG256_BLOCK_SIZE STREEBOG512_BLOCK_SIZE +#define streebog256_ctx streebog512_ctx + +void +streebog256_init(struct streebog256_ctx *ctx); + +#define streebog256_update nettle_streebog512_update + +void +streebog256_digest(struct streebog256_ctx *ctx, + size_t length, + uint8_t *digest); + +#ifdef __cplusplus +} +#endif + +#endif /* NETTLE_STREEBOG_H_INCLUDED */ -- 2.26.2

2 17

FYI: Forced update of master-updates
by nisse＠lysator.liu.se 27 Jun '20

27 Jun '20

Hi, I tried out a microoptimization of the arm neon implementation of chacha and salsa20. Gave a 10% speedup on the older Cortex-A5 core, but unclear if it's an improvement overall, so I don't want to push it to master, and I've removed that commit from master-updates (now on its own branch arm-salsa20-chacha-vsra instead, in case anyone is curious). I'm considering changing the internal _salsa20_core and _chacha_core to do more than one block at a time, since processing a few blocks in parallel has a great potential for performance improvements. Regards, /Niels -- Niels Möller. PGP-encrypted email is preferred. Keyid 368C6677. Internet email is subject to wholesale government surveillance.

1 0

Re: [PATCH] (revision 3.2) Added bcrypt() support.
by Stephen R. van den Berg 23 Jun '20

23 Jun '20

Changes since 3.1: Took out the tablesize optimisations. --- Makefile.in | 2 +- blowfish-bcrypt.c | 519 ++++++++++++++++++++++++++++++ blowfish.h => blowfish-internal.h | 51 +-- blowfish.c | 23 +- blowfish.h | 15 + nettle.texinfo | 67 ++++ 6 files changed, 621 insertions(+), 56 deletions(-) create mode 100644 blowfish-bcrypt.c copy blowfish.h => blowfish-internal.h (50%) diff --git a/Makefile.in b/Makefile.in index e5ccfc76..4ddbb32e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -73,7 +73,7 @@ nettle_SOURCES = aes-decrypt-internal.c aes-decrypt.c \ aes256-set-encrypt-key.c aes256-set-decrypt-key.c \ aes256-meta.c \ arcfour.c arcfour-crypt.c \ - arctwo.c arctwo-meta.c blowfish.c \ + arctwo.c arctwo-meta.c blowfish.c blowfish-bcrypt.c \ base16-encode.c base16-decode.c base16-meta.c \ base64-encode.c base64-decode.c base64-meta.c \ base64url-encode.c base64url-decode.c base64url-meta.c \ diff --git a/blowfish-bcrypt.c b/blowfish-bcrypt.c new file mode 100644 index 00000000..0354dbe6 --- /dev/null +++ b/blowfish-bcrypt.c @@ -0,0 +1,519 @@ +/* blowfish-bcrypt.c + + The blowfish bcrypt implementation. + + Copyright (c) 2020 Stephen R. van den Berg + + This file is part of GNU Nettle. + + GNU Nettle is free software: you can redistribute it and/or + modify it under the terms of either: + + * the GNU Lesser General Public License as published by the Free + Software Foundation; either version 3 of the License, or (at your + option) any later version. + + or + + * the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + or both in parallel, as here. + + GNU Nettle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received copies of the GNU General Public License and + the GNU Lesser General Public License along with this program. If + not, see http://www.gnu.org/licenses/. +*/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif + +#include <assert.h> +#include <string.h> +#include <stdlib.h> + +#include "blowfish.h" +#include "blowfish-internal.h" +#include "base64.h" + +#include "macros.h" + +#define CRYPTPLEN 7 +#define SALTLEN ((BLOWFISH_BCRYPT_BINSALT_SIZE*8+5) / 6) + +#define HASHOFFSET (CRYPTPLEN + SALTLEN) + +static const signed char radix64_decode_table[0x100] = { + /* White space is HT, VT, FF, CR, LF and SPC */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -2, -2, -2, -2, -2, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, -1, -1, -1, -3, -1, -1, + -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, -1, -1, -1, -1, -1, + -1, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, +}; + +static const char radix64_encode_table[64] = + "./ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789"; + +int +blowfish_bcrypt_verify(const char *key, + const char *hashed) +{ + char newhash[BLOWFISH_BCRYPT_HASH_SIZE]; + + return blowfish_bcrypt_hash(sizeof newhash, + newhash, key, hashed, -1, (void*)0) + && !strcmp(newhash, hashed); +} + +static char *encode_radix64(char *dst, size_t len, const uint8_t *src) +{ + struct base64_encode_ctx ctx; + base64_encode_init(&ctx); + ctx.alphabet = radix64_encode_table; + dst += base64_encode_update(&ctx, dst, len, src); + dst += base64_encode_final(&ctx, dst); + *--dst = '\0'; /* Strip the trailing = */ + return dst; +} + +/* + * Large parts of the code below are based on public domain sources. + * The comments and copyright notices have been preserved. + * Any code added or modified by me is licensed under the + * licenses listed above. -- Stephen R. van den Berg + */ + +/* + * This code comes from John the Ripper password cracker, with reentrant + * and crypt(3) interfaces added, but optimizations specific to password + * cracking removed. + * + * Written by Solar Designer <solar at openwall.com> in 1998-2015. + * No copyright is claimed, and the software is hereby placed in the public + * domain. In case this attempt to disclaim copyright and place the software + * in the public domain is deemed null and void, then the software is + * Copyright (c) 1998-2015 Solar Designer and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * It is my intent that you should be able to use this on your system, + * as part of a software package, or anywhere else to improve security, + * ensure compatibility, or for any other purpose. I would appreciate + * it if you give credit where it is due and keep your modifications in + * the public domain as well, but I don't require that in order to let + * you place this code and any modifications you make under a license + * of your choice. + * + * This implementation is fully compatible with OpenBSD's bcrypt.c for prefix + * "$2b$", originally by Niels Provos <provos at citi.umich.edu>, and it uses + * some of his ideas. The password hashing algorithm was designed by David + * Mazieres <dm at lcs.mit.edu>. For information on the level of + * compatibility for bcrypt hash prefixes other than "$2b$", please refer to + * the comments in set_key() below and to the included crypt(3) man page. + */ + +typedef uint32_t bf_key[_BLOWFISH_ROUNDS + 2]; + +/* + * Magic IV for 64 Blowfish encryptions that we do at the end. + * The string is "OrpheanBeholderScryDoubt" on big-endian. + */ +static uint32_t magic_w[6] = { + 0x4F727068, 0x65616E42, 0x65686F6C, + 0x64657253, 0x63727944, 0x6F756274 +}; + +static void swap32(uint32_t *x, int count) +{ +#if !WORDS_BIGENDIAN + do { + uint32_t tmp = *x; + tmp = (tmp << 16) | (tmp >> 16); + *x++ = ((tmp & 0x00FF00FF) << 8) | ((tmp >> 8) & 0x00FF00FF); + } while (--count); +#endif +} + +static void set_xkey(const char *key, bf_key expanded, bf_key initial, + unsigned bug, uint32_t safety) +{ + const char *ptr = key; + unsigned i, j; + uint32_t sign, diff, tmp[2]; + +/* + * There was a sign extension bug in older revisions of this function. While + * we would have liked to simply fix the bug and move on, we have to provide + * a backwards compatibility feature (essentially the bug) for some systems and + * a safety measure for some others. The latter is needed because for certain + * multiple inputs to the buggy algorithm there exist easily found inputs to + * the correct algorithm that produce the same hash. Thus, we optionally + * deviate from the correct algorithm just enough to avoid such collisions. + * While the bug itself affected the majority of passwords containing + * characters with the 8th bit set (although only a percentage of those in a + * collision-producing way), the anti-collision safety measure affects + * only a subset of passwords containing the '\xff' character (not even all of + * those passwords, just some of them). This character is not found in valid + * UTF-8 sequences and is rarely used in popular 8-bit character encodings. + * Thus, the safety measure is unlikely to cause much annoyance, and is a + * reasonable tradeoff to use when authenticating against existing hashes that + * are not reliably known to have been computed with the correct algorithm. + * + * We use an approach that tries to minimize side-channel leaks of password + * information - that is, we mostly use fixed-cost bitwise operations instead + * of branches or table lookups. (One conditional branch based on password + * length remains. It is not part of the bug aftermath, though, and is + * difficult and possibly unreasonable to avoid given the use of C strings by + * the caller, which results in similar timing leaks anyway.) + * + * For actual implementation, we set an array index in the variable "bug" + * (0 means no bug, 1 means sign extension bug emulation) and a flag in the + * variable "safety" (bit 16 is set when the safety measure is requested). + * Valid combinations of settings are: + * + * Prefix "$2a$": bug = 0, safety = 0x10000 + * Prefix "$2b$": bug = 0, safety = 0 + * Prefix "$2x$": bug = 1, safety = 0 + * Prefix "$2y$": bug = 0, safety = 0 + */ + + sign = diff = 0; + + for (i = 0; i < _BLOWFISH_ROUNDS + 2; i++) { + tmp[0] = tmp[1] = 0; + for (j = 0; j < 4; j++) { + tmp[0] <<= 8; + tmp[0] |= (unsigned char)*ptr; /* correct */ + tmp[1] <<= 8; + tmp[1] |= (signed char)*ptr; /* bug */ +/* + * Sign extension in the first char has no effect - nothing to overwrite yet, + * and those extra 24 bits will be fully shifted out of the 32-bit word. For + * chars 2, 3, 4 in each four-char block, we set bit 7 of "sign" if sign + * extension in tmp[1] occurs. Once this flag is set, it remains set. + */ + if (j) + sign |= tmp[1] & 0x80; + if (!*ptr) + ptr = key; + else + ptr++; + } + diff |= tmp[0] ^ tmp[1]; /* Non-zero on any differences */ + + expanded[i] = tmp[bug]; + initial[i] = _nettle_blowfish_initial_ctx.p[i] ^ tmp[bug]; + } + +/* + * At this point, "diff" is zero if the correct and buggy algorithms produced + * exactly the same result. If so and if "sign" is non-zero, which indicates + * that there was a non-benign sign extension, this means that we have a + * collision between the correctly computed hash for this password and a set of + * passwords that could be supplied to the buggy algorithm. Our safety measure + * is meant to protect from such many-buggy to one-correct collisions, by + * deviating from the correct algorithm in such cases. Let's check for this. + */ + diff |= diff >> 16; /* still zero if exact match */ + diff &= 0xffff; /* ditto */ + diff += 0xffff; /* bit 16 set if "diff" was non-zero (on non-match) */ + sign <<= 9; /* move the non-benign sign extension flag to bit 16 */ + sign &= ~diff & safety; /* action needed? */ + +/* + * If we have determined that we need to deviate from the correct algorithm, + * flip bit 16 in initial expanded key. (The choice of 16 is arbitrary, but + * let's stick to it now. It came out of the approach we used above, and it's + * not any worse than any other choice we could make.) + * + * It is crucial that we don't do the same to the expanded key used in the main + * Eksblowfish loop. By doing it to only one of these two, we deviate from a + * state that could be directly specified by a password to the buggy algorithm + * (and to the fully correct one as well, but that's a side-effect). + */ + initial[0] ^= sign; +} + +static int ibcrypt(size_t length, char *dst, + const char *key, const char *scheme, + int minlog2rounds, + int log2rounds, const uint8_t *salt) +{ + struct { + struct blowfish_ctx ctx; + bf_key expanded_key; + union { + uint32_t salt[4]; + uint32_t output[6]; + } binary; + } data; + uint8_t psalt[BLOWFISH_BCRYPT_BINSALT_SIZE]; + uint32_t L, R; + uint32_t *ptr; + uint32_t count; + int i; + size_t schemelen = strlen(scheme); + unsigned cscheme; + unsigned bug = 0; + uint32_t safety = 0; + if (length < BLOWFISH_BCRYPT_HASH_SIZE || + schemelen < 2) + return 0; + + if (schemelen >= 3 && *scheme++ != '$') + return 0; + if (*scheme++ != '2') + return 0; + + switch (cscheme = *scheme++) { + default: + return 0; + case 'a': safety = 0x10000; + break; + case 'x': bug = 1; + break; + case 'b': case 'y': + break; + } + + if (schemelen >= 4) { + if (*scheme++ != '$') + return 0; + if (schemelen >= 6) { + if (log2rounds < 0) + log2rounds = atoi(scheme); + scheme += 2; + if (schemelen >= CRYPTPLEN && *scheme++ != '$') + return 0; + if (schemelen >= HASHOFFSET && !salt) { + struct base64_decode_ctx ctx; + size_t saltlen = BLOWFISH_BCRYPT_BINSALT_SIZE; + + base64_decode_init(&ctx); + ctx.table = radix64_decode_table; + + if (!base64_decode_update(&ctx, &saltlen, (uint8_t *) data.binary.salt, + SALTLEN, scheme) + || saltlen != BLOWFISH_BCRYPT_BINSALT_SIZE) + return 0; + } + } + } + + if (salt) + memcpy(data.binary.salt, salt, BLOWFISH_BCRYPT_BINSALT_SIZE); + else if (schemelen < HASHOFFSET) + return 0; + memcpy(psalt, data.binary.salt, BLOWFISH_BCRYPT_BINSALT_SIZE); + swap32(data.binary.salt, 4); + + if (log2rounds < minlog2rounds || log2rounds > 31) + return 0; + count = (uint32_t)1 << log2rounds; + + set_xkey(key, data.expanded_key, data.ctx.p, bug, safety); + memcpy(data.ctx.s, _nettle_blowfish_initial_ctx.s, sizeof(data.ctx.s)); + + L = R = 0; + for (i = 0; i < _BLOWFISH_ROUNDS + 2; i += 2) { + L ^= data.binary.salt[i & 2]; + R ^= data.binary.salt[(i & 2) + 1]; + _nettle_blowfish_encround(&data.ctx, &L, &R); + data.ctx.p[i] = L; + data.ctx.p[i + 1] = R; + } + + ptr = data.ctx.s[0]; + do { + ptr += 4; + L ^= data.binary.salt[(_BLOWFISH_ROUNDS + 2) & 3]; + R ^= data.binary.salt[(_BLOWFISH_ROUNDS + 3) & 3]; + _nettle_blowfish_encround(&data.ctx, &L, &R); + *(ptr - 4) = L; + *(ptr - 3) = R; + + L ^= data.binary.salt[(_BLOWFISH_ROUNDS + 4) & 3]; + R ^= data.binary.salt[(_BLOWFISH_ROUNDS + 5) & 3]; + _nettle_blowfish_encround(&data.ctx, &L, &R); + *(ptr - 2) = L; + *(ptr - 1) = R; + } while (ptr < &data.ctx.s[3][0xFF]); + + do { + int done; + + for (i = 0; i < _BLOWFISH_ROUNDS + 2; i += 2) { + data.ctx.p[i] ^= data.expanded_key[i]; + data.ctx.p[i + 1] ^= data.expanded_key[i + 1]; + } + + done = 0; + do { + uint32_t tmp1, tmp2, tmp3, tmp4; + + L = R = 0; + ptr = data.ctx.p; + do { + ptr += 2; + _nettle_blowfish_encround(&data.ctx, &L, &R); + *(ptr - 2) = L; + *(ptr - 1) = R; + } while (ptr < &data.ctx.p[_BLOWFISH_ROUNDS + 2]); + + ptr = data.ctx.s[0]; + do { + ptr += 2; + _nettle_blowfish_encround(&data.ctx, &L, &R); + *(ptr - 2) = L; + *(ptr - 1) = R; + } while (ptr < &data.ctx.s[3][0xFF]); + + if (done) + break; + done = 1; + + tmp1 = data.binary.salt[0]; + tmp2 = data.binary.salt[1]; + tmp3 = data.binary.salt[2]; + tmp4 = data.binary.salt[3]; + for (i = 0; i < _BLOWFISH_ROUNDS; i += 4) { + data.ctx.p[i] ^= tmp1; + data.ctx.p[i + 1] ^= tmp2; + data.ctx.p[i + 2] ^= tmp3; + data.ctx.p[i + 3] ^= tmp4; + } + data.ctx.p[16] ^= tmp1; + data.ctx.p[17] ^= tmp2; + } while (1); + } while (--count); + + for (i = 0; i < 6; i += 2) { + L = magic_w[i]; + R = magic_w[i + 1]; + + count = 64; + do + _nettle_blowfish_encround(&data.ctx, &L, &R); + while (--count); + + data.binary.output[i] = L; + data.binary.output[i + 1] = R; + } + + *dst++ = '$'; + *dst++ = '2'; + *dst++ = cscheme; + *dst++ = '$'; + *dst++ = '0' + log2rounds / 10; + *dst++ = '0' + log2rounds % 10; + *dst++ = '$'; + dst = encode_radix64(dst, BLOWFISH_BCRYPT_BINSALT_SIZE, psalt) - 1; + + swap32(data.binary.output, 6); +/* This has to be bug-compatible with the original implementation, so + only encode 23 of the 24 bytes. */ + encode_radix64(dst, 23, (uint8_t *) data.binary.output); + return cscheme; +} + +/* + * Please preserve the runtime self-test. It serves two purposes at once: + * + * 1. We really can't afford the risk of producing incompatible hashes e.g. + * when there's something like gcc bug 26587 again, whereas an application or + * library integrating this code might not also integrate our external tests or + * it might not run them after every build. Even if it does, the miscompile + * might only occur on the production build, but not on a testing build (such + * as because of different optimization settings). It is painful to recover + * from incorrectly-computed hashes - merely fixing whatever broke is not + * enough. Thus, a proactive measure like this self-test is needed. + * + * 2. We don't want to leave sensitive data from our actual password hash + * computation on the stack or in registers. Previous revisions of the code + * would do explicit cleanups, but simply running the self-test after hash + * computation is more reliable. + * + * The performance cost of this quick self-test is around 0.6% at the "$2a$08" + * setting. + */ +int blowfish_bcrypt_hash(size_t length, char *dst, + const char *key, const char *scheme, + int log2rounds, const uint8_t *salt) +{ + const char *test_pw = "8b \xd0\xc1\xd2\xcf\xcc\xd8"; + const char *test_scheme = "$2a$00$abcdefghijklmnopqrstuu"; + static const char * const test_hashes[2] = + {"i1D709vfamulimlGcq0qq3UvuUasvEa\0\x55", /* 'a', 'b', 'y' */ + "VUrPmXD6q/nVSSp7pNDhCR9071IfIRe\0\x55"}; /* 'x' */ + const char *test_hash = test_hashes[0]; + int cscheme; + int ok; + struct { + char s[HASHOFFSET + 1]; + char o[HASHOFFSET + 31 + 1 + 1 + 1]; + } buf; + + if (length) + *dst = '\0'; +/* Hash the supplied password */ + cscheme = ibcrypt(length, dst, key, scheme, 4, log2rounds, salt); + +/* + * Do a quick self-test. It is important that we make both calls to ibcrypt() + * from the same scope such that they likely use the same stack locations, + * which makes the second call overwrite the first call's sensitive data on the + * stack and makes it more likely that any alignment related issues would be + * detected by the self-test. + */ + memcpy(buf.s, test_scheme, sizeof(buf.s)); + + if (cscheme) + test_hash = test_hashes[(buf.s[2] = cscheme) == 'x']; + + memset(buf.o, 0x55, sizeof(buf.o)); + buf.o[sizeof(buf.o) - 1] = 0; + ok = ibcrypt(sizeof(buf.o) - (1 + 1), buf.o, test_pw, + buf.s, 0, -1, (void*)0); + + ok = (ok && + !memcmp(buf.o, buf.s, HASHOFFSET) && + !memcmp(buf.o + HASHOFFSET, test_hash, 31 + 1 + 1 + 1)); + + { + const char *k = "\xff\xa3" "34" "\xff\xff\xff\xa3" "345"; + bf_key ae, ai, ye, yi; + set_xkey(k, ae, ai, 0, 0x10000); /* $2a$ */ + set_xkey(k, ye, yi, 0, 0); /* $2y$ */ + ai[0] ^= 0x10000; /* undo the safety (for comparison) */ + ok = ok && ai[0] == 0xdb9c59bc && ye[17] == 0x33343500 && + !memcmp(ae, ye, sizeof(ae)) && + !memcmp(ai, yi, sizeof(ai)); + } + + return ok && !!cscheme; +} diff --git a/blowfish.h b/blowfish-internal.h similarity index 50% copy from blowfish.h copy to blowfish-internal.h index bcdc7cb6..da28fa34 100644 --- a/blowfish.h +++ b/blowfish-internal.h @@ -1,4 +1,4 @@ -/* blowfish.h +/* blowfish-internal.h Blowfish block cipher. @@ -32,8 +32,8 @@ not, see http://www.gnu.org/licenses/. */ -#ifndef NETTLE_BLOWFISH_H_INCLUDED -#define NETTLE_BLOWFISH_H_INCLUDED +#ifndef NETTLE_BLOWFISH_INTERNAL_H_INCLUDED +#define NETTLE_BLOWFISH_INTERNAL_H_INCLUDED #include "nettle-types.h" @@ -41,49 +41,12 @@ extern "C" { #endif -/* Name mangling */ -#define blowfish_set_key nettle_blowfish_set_key -#define blowfish128_set_key nettle_blowfish128_set_key -#define blowfish_encrypt nettle_blowfish_encrypt -#define blowfish_decrypt nettle_blowfish_decrypt - -#define BLOWFISH_BLOCK_SIZE 8 - -/* Variable key size between 64 and 448 bits. */ -#define BLOWFISH_MIN_KEY_SIZE 8 -#define BLOWFISH_MAX_KEY_SIZE 56 - -/* Default to 128 bits */ -#define BLOWFISH_KEY_SIZE 16 - -#define BLOWFISH128_KEY_SIZE 16 - -#define _BLOWFISH_ROUNDS 16 - -struct blowfish_ctx -{ - uint32_t s[4][256]; - uint32_t p[_BLOWFISH_ROUNDS+2]; -}; - -/* Returns 0 for weak keys, otherwise 1. */ -int -blowfish_set_key(struct blowfish_ctx *ctx, - size_t length, const uint8_t *key); -int -blowfish128_set_key(struct blowfish_ctx *ctx, const uint8_t *key); - -void -blowfish_encrypt(const struct blowfish_ctx *ctx, - size_t length, uint8_t *dst, - const uint8_t *src); -void -blowfish_decrypt(const struct blowfish_ctx *ctx, - size_t length, uint8_t *dst, - const uint8_t *src); +extern const struct blowfish_ctx _nettle_blowfish_initial_ctx; +extern void _nettle_blowfish_encround (const struct blowfish_ctx *ctx, + uint32_t * ret_xl, uint32_t * ret_xr); #ifdef __cplusplus } #endif -#endif /* NETTLE_BLOWFISH_H_INCLUDED */ +#endif /* NETTLE_BLOWFISH_INTERNAL_H_INCLUDED */ diff --git a/blowfish.c b/blowfish.c index 52040f13..e73caffe 100644 --- a/blowfish.c +++ b/blowfish.c @@ -54,12 +54,13 @@ #include <assert.h> #include "blowfish.h" +#include "blowfish-internal.h" #include "macros.h" /* precomputed S boxes */ -static const struct blowfish_ctx -initial_ctx = { +const struct blowfish_ctx +_nettle_blowfish_initial_ctx = { { { /* ks0 */ 0xD1310BA6, 0x98DFB5AC, 0x2FFD72DB, 0xD01ADFB7, 0xB8E1AFED, 0x6A267E96, @@ -261,8 +262,8 @@ initial_ctx = { #define R(c, l,r,i) do { l ^= c->p[i]; r ^= F(c,l); } while(0) -static void -encrypt (const struct blowfish_ctx *ctx, uint32_t * ret_xl, +void +_nettle_blowfish_encround (const struct blowfish_ctx *ctx, uint32_t * ret_xl, uint32_t * ret_xr) { uint32_t xl, xr; @@ -295,7 +296,7 @@ encrypt (const struct blowfish_ctx *ctx, uint32_t * ret_xl, } static void -decrypt (const struct blowfish_ctx *ctx, uint32_t * ret_xl, uint32_t * ret_xr) +decround (const struct blowfish_ctx *ctx, uint32_t * ret_xl, uint32_t * ret_xr) { uint32_t xl, xr; @@ -339,7 +340,7 @@ blowfish_encrypt (const struct blowfish_ctx *ctx, d1 = READ_UINT32(src); d2 = READ_UINT32(src+4); - encrypt (ctx, &d1, &d2); + _nettle_blowfish_encround (ctx, &d1, &d2); dst[0] = (d1 >> 24) & 0xff; dst[1] = (d1 >> 16) & 0xff; dst[2] = (d1 >> 8) & 0xff; @@ -361,7 +362,7 @@ blowfish_decrypt (const struct blowfish_ctx *ctx, d1 = READ_UINT32(src); d2 = READ_UINT32(src+4); - decrypt (ctx, &d1, &d2); + decround (ctx, &d1, &d2); dst[0] = (d1 >> 24) & 0xff; dst[1] = (d1 >> 16) & 0xff; dst[2] = (d1 >> 8) & 0xff; @@ -380,7 +381,7 @@ blowfish_set_key (struct blowfish_ctx *ctx, int i, j; uint32_t data, datal, datar; - *ctx = initial_ctx; + *ctx = _nettle_blowfish_initial_ctx; for (i = j = 0; i < _BLOWFISH_ROUNDS + 2; i++) { @@ -393,15 +394,15 @@ blowfish_set_key (struct blowfish_ctx *ctx, datal = datar = 0; for (i = 0; i < _BLOWFISH_ROUNDS + 2; i += 2) { - encrypt (ctx, &datal, &datar); + _nettle_blowfish_encround (ctx, &datal, &datar); ctx->p[i] = datal; ctx->p[i + 1] = datar; } - + for (j = 0; j < 4; j++) for (i = 0; i < 256; i += 2) { - encrypt (ctx, &datal, &datar); + _nettle_blowfish_encround (ctx, &datal, &datar); ctx->s[j][i] = datal; ctx->s[j][i + 1] = datar; } diff --git a/blowfish.h b/blowfish.h index bcdc7cb6..af48e20f 100644 --- a/blowfish.h +++ b/blowfish.h @@ -46,6 +46,8 @@ extern "C" { #define blowfish128_set_key nettle_blowfish128_set_key #define blowfish_encrypt nettle_blowfish_encrypt #define blowfish_decrypt nettle_blowfish_decrypt +#define blowfish_bcrypt_hash nettle_blowfish_bcrypt_hash +#define blowfish_bcrypt_verify nettle_blowfish_bcrypt_verify #define BLOWFISH_BLOCK_SIZE 8 @@ -60,6 +62,9 @@ extern "C" { #define _BLOWFISH_ROUNDS 16 +#define BLOWFISH_BCRYPT_HASH_SIZE (60 + 1) /* Including null-terminator */ +#define BLOWFISH_BCRYPT_BINSALT_SIZE 16 /* Binary string size */ + struct blowfish_ctx { uint32_t s[4][256]; @@ -81,6 +86,16 @@ void blowfish_decrypt(const struct blowfish_ctx *ctx, size_t length, uint8_t *dst, const uint8_t *src); +int +blowfish_bcrypt_hash(size_t length, + char *dst, + const char *key, + const char *scheme, + int log2rounds, + const uint8_t *salt); +int +blowfish_bcrypt_verify(const char *key, + const char *hashed); #ifdef __cplusplus } diff --git a/nettle.texinfo b/nettle.texinfo index 995d5de8..75e18b58 100644 --- a/nettle.texinfo +++ b/nettle.texinfo @@ -1513,6 +1513,73 @@ in any other way. Analogous to @code{blowfish_encrypt} @end deftypefun +@deftypefun int blowfish_bcrypt_hash (size_t @var{length}, char *@var{dst}, const char *@var{key}, const char *@var{scheme}, int @var{log2rounds}, const uint8_t *@var{salt}) +Compute the bcrypt password hash. +The function will return @code{0} if the hash cannot be computed +due to invalid input. +The function will return @code{1} and store the computed hash +in the array pointed to by @var{dst}. The hash is computed based +on the chosen @var{scheme}, number of rounds @var{log2rounds} and +specified @var{salt}. + +@var{length} must be at least @code{BLOWFISH_BCRYPT_HASH_SIZE}. + +@var{dst} must point to a character array of the specified @var{length}. + +@var{key} contains the zero terminated plaintext password string. + +@var{scheme} contains either just the chosen scheme (valid schemes +are: @code{2a}, @code{2b}, @code{2x} or @code{2y}), or +(the prefix of) an existing hashed password (typically @code{$2b$10$...}). + +@var{log2rounds} contains the log2 of the number of encryption rounds +that must be used to compute the hash. If it is @code{-1} the value +will be extracted from @var{scheme}. + +@var{salt} should point to an array of @code{BLOWFISH_BCRYPT_BINSALT_SIZE} +random bytes to be used to perturb the hash computation. If it is @code{NULL} +the salt will be extracted from @var{scheme}. + +Sample code to generate a bcrypt hash: +@example +char cleartxtpassword[] = "ExamplePassword"; +uint8_t salt[BLOWFISH_BCRYPT_BINSALT_SIZE]; +@dots{} +/* Make sure that salt is filled with random bytes */ +@dots{} +char hashedresult[BLOWFISH_BCRYPT_HASH_SIZE]; +int result = blowfish_bcrypt(sizeof(hashedresult), hashedresult, + cleartxtpassword, "2b", 10, salt); +if (result) + printf("%s\n", hashedresult); +@end example +@end deftypefun + +@deftypefun int blowfish_bcrypt_verify (const char *@var{key}, const char *@var{hashed}) +Verifies the bcrypt password hash against the supplied plaintext password. +The function will return @code{0} if the password does not match. +The function will return @code{1} if the password matches. + +@var{key} contains the zero terminated plaintext password string. + +@var{hashed} contains the zero terminated hashed string to compare with. + +Sample code to verify a bcrypt hash: +@example +char cleartxtpassword[] = "ExamplePassword"; +char existinghashed[] = + "$2y$" /* Hash algorithm version */ + "10" /* 2^10 hash rounds (strength) */ + "$" /* separator */ + "1b2lPgo4XumibnJGN3r3sO" /* base64 encoded 16-byte salt */ + "u7wE7xNfYDKlAxZffJDCJdVfFTAyevu"; /* Hashedpart */ +if (blowfish_bcrypt_verify(cleartxtpassword, existinghashed)) + printf("Password is correct."); +else + printf("Password is incorrect."); +@end example +@end deftypefun + @subsection Camellia @cindex Camellia -- 2.20.1

2 7

Fwd: PPC64LE optimizing AES and GHASH
by Maamoun TK 22 Jun '20

22 Jun '20

On Sat, Jun 20, 2020 at 11:54 AM Niels Möller <nisse(a)lysator.liu.se> wrote: > Have you measured speedup when going from 4 to 8 blocks? We shouldn't > add larger loops than needed. > The 8x loop has x~1.15 performance boost over 4x loop, if you think it's not worth it, I can add only 4x loop to make the code simpler. > Do you measure a speedup from this? Karatsuba usually pays off only for > a bit larger sizes (but I guess overhead is a little less here than for > standard multiplication). > Actually, I considered the Karatsuba algorithm not only for performance but to reduce the number of registers used. However, I believe that using the Karatsuba algorithm in my case performs better or similar to classical multiplication. > > - Since the functionality of gcm_set_key() is replaced with > > gcm_init_key() for PowerPC64LE, two warnings will pop up: > [‘gcm_gf_shift’ > > defined but not used] and [‘gcm_gf_add’ defined but not used] > When I applied the patch to the last upstream, these warnings did not appear, some changes have occurred to gcm.c, I will look at it. > To test PPC code, I wonder if it's easy to add a PPC build to > .gitlab-ci, in the same way as arm and mips tests. These are based on > Debian packaged cross compilers and qemu-user. I'm also not that familiar > with the variants within the Power and PowerPC family of processors. > I will see what I can do.

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

2005

2004

2003

2002

nettle-bugs