diff --git a/lib/Makefile b/lib/Makefile index 6b6ff9e4..7239fc40 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -340,7 +340,8 @@ c_lib_objs := \ quic_hp_chacha20.o \ quic_chacha20_poly1305.o \ hmac_ipad_opad.o \ - cipher_suite_id.o + cipher_suite_id.o \ + capabilities.o ifeq ($(AESNI_EMU), y) c_lib_objs := $(c_lib_objs) \ @@ -678,6 +679,7 @@ asm_avx2_t4_lib_objs := \ sm3_msg_avx2.o \ sm3_hmac_avx2.o \ sha512_x1_ni_avx2.o \ + sha512_x2_ni_avx2.o \ sha512_hmac_ni_avx2.o # diff --git a/lib/avx2_t1/mb_mgr_avx2_t1.c b/lib/avx2_t1/mb_mgr_avx2_t1.c index 1060696a..14795973 100644 --- a/lib/avx2_t1/mb_mgr_avx2_t1.c +++ b/lib/avx2_t1/mb_mgr_avx2_t1.c @@ -346,6 +346,9 @@ init_mb_mgr_avx2_t1_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_AVX2; + /* Set architecture type for future checks */ + state->used_arch_type = 1; + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/avx2_t1/sha_mb_avx2.c b/lib/avx2_t1/sha_mb_avx2.c index eee83390..5777ccbf 100644 --- a/lib/avx2_t1/sha_mb_avx2.c +++ b/lib/avx2_t1/sha_mb_avx2.c @@ -111,7 +111,7 @@ IMB_JOB * submit_job_sha384_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 4, 1, 384, IMB_SHA_512_BLOCK_SIZE, - SHA384_PAD_SIZE, call_sha512_x4_avx2_from_c); + SHA384_PAD_SIZE, call_sha512_x4_avx2_from_c, 0); } IMB_DLL_LOCAL @@ -119,7 +119,7 @@ IMB_JOB * flush_job_sha384_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 4, 0, 384, IMB_SHA_512_BLOCK_SIZE, - SHA384_PAD_SIZE, call_sha512_x4_avx2_from_c); + SHA384_PAD_SIZE, call_sha512_x4_avx2_from_c, 0); } /* ========================================================================== */ @@ -132,7 +132,7 @@ IMB_JOB * submit_job_sha512_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 4, 1, 512, IMB_SHA_512_BLOCK_SIZE, - SHA512_PAD_SIZE, call_sha512_x4_avx2_from_c); + SHA512_PAD_SIZE, call_sha512_x4_avx2_from_c, 0); } IMB_DLL_LOCAL @@ -140,5 +140,5 @@ IMB_JOB * flush_job_sha512_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 4, 0, 512, IMB_SHA_512_BLOCK_SIZE, - SHA512_PAD_SIZE, call_sha512_x4_avx2_from_c); + SHA512_PAD_SIZE, call_sha512_x4_avx2_from_c, 0); } diff --git a/lib/avx2_t2/mb_mgr_avx2_t2.c b/lib/avx2_t2/mb_mgr_avx2_t2.c index eeb3fa34..d7bb2cdd 100644 --- a/lib/avx2_t2/mb_mgr_avx2_t2.c +++ b/lib/avx2_t2/mb_mgr_avx2_t2.c @@ -349,6 +349,9 @@ init_mb_mgr_avx2_t2_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_AVX2; + /* Set architecture type for future checks */ + state->used_arch_type = 2; + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/avx2_t3/mb_mgr_avx2_t3.c b/lib/avx2_t3/mb_mgr_avx2_t3.c index 761569fd..ce719313 100644 --- a/lib/avx2_t3/mb_mgr_avx2_t3.c +++ b/lib/avx2_t3/mb_mgr_avx2_t3.c @@ -349,6 +349,9 @@ init_mb_mgr_avx2_t3_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_AVX2; + /* Set architecture type for future checks */ + state->used_arch_type = 3; + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/avx2_t4/mb_mgr_avx2_t4.c b/lib/avx2_t4/mb_mgr_avx2_t4.c index bd898293..fcbb4fd8 100644 --- a/lib/avx2_t4/mb_mgr_avx2_t4.c +++ b/lib/avx2_t4/mb_mgr_avx2_t4.c @@ -350,6 +350,9 @@ init_mb_mgr_avx2_t4_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_AVX2; + /* Set architecture type for future checks */ + state->used_arch_type = 4; + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/avx2_t4/sha512_x2_ni_avx2.asm b/lib/avx2_t4/sha512_x2_ni_avx2.asm new file mode 100644 index 00000000..6995a275 --- /dev/null +++ b/lib/avx2_t4/sha512_x2_ni_avx2.asm @@ -0,0 +1,412 @@ +;; +;; Copyright (c) 2024, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; =========================================================== +;; NOTE about comment format: +;; +;; xmm = a b c d +;; ^ ^ +;; | | +;; MSB--+ +--LSB +;; +;; a - most significant word in `ymm` +;; d - least significant word in `ymm` +;; =========================================================== + +%use smartalign + +%include "include/os.inc" +%include "include/clear_regs.inc" +%include "include/reg_sizes.inc" +%include "include/mb_mgr_datastruct.inc" + +; resdq = res0 => 16 bytes +struc frame +.ABEF_SAVE resy 1 +.CDGH_SAVE resy 1 +.ABEF_SAVEb resy 1 +.CDGH_SAVEb resy 1 +endstruc + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%define arg4 rcx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%define arg4 r9 +%endif + +%define args arg1 +%define NUM_BLKS arg2 + +%define INP arg3 +%define INPb arg4 + +%define SHA512_CONSTS rax + +%define MSG ymm0 +%define STATE0 ymm1 +%define STATE1 ymm2 +%define MSGTMP0 ymm3 +%define MSGTMP1 ymm4 +%define MSGTMP2 ymm5 + +%define YTMP0 ymm6 +%define YTMP1 ymm7 + +%define STATE0b ymm8 +%define STATE1b ymm9 +%define MSGb ymm10 + +%define YTMP2 ymm11 +%define YTMP3 ymm12 + +%define MSGTMP0b ymm13 +%define MSGTMP1b ymm14 +%define MSGTMP2b ymm15 + +%define GP_STORAGE 6*8 +%ifndef LINUX +%define XMM_STORAGE 10*16 +%else +%define XMM_STORAGE 0 +%endif + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~31 ; align rsp to 32 bytes + + mov [rsp + 0*8], rbx + mov [rsp + 1*8], rbp + mov [rsp + 2*8], r12 +%ifndef LINUX + mov [rsp + 3*8], rsi + mov [rsp + 4*8], rdi + vmovdqa [rsp + 3*16], xmm6 + vmovdqa [rsp + 4*16], xmm7 + vmovdqa [rsp + 5*16], xmm8 + vmovdqa [rsp + 6*16], xmm9 + vmovdqa [rsp + 7*16], xmm10 + vmovdqa [rsp + 8*16], xmm11 + vmovdqa [rsp + 9*16], xmm12 + vmovdqa [rsp + 10*16], xmm13 + vmovdqa [rsp + 11*16], xmm14 + vmovdqa [rsp + 12*16], xmm15 +%endif ; LINUX + mov [rsp + 5*8], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 + mov rbx, [rsp + 0*8] + mov rbp, [rsp + 1*8] + mov r12, [rsp + 2*8] +%ifndef LINUX + mov rsi, [rsp + 3*8] + mov rdi, [rsp + 4*8] + vmovdqa xmm6, [rsp + 3*16] + vmovdqa xmm7, [rsp + 4*16] + vmovdqa xmm8, [rsp + 5*16] + vmovdqa xmm9, [rsp + 6*16] + vmovdqa xmm10, [rsp + 7*16] + vmovdqa xmm11, [rsp + 8*16] + vmovdqa xmm12, [rsp + 9*16] + vmovdqa xmm13, [rsp + 10*16] + vmovdqa xmm14, [rsp + 11*16] + vmovdqa xmm15, [rsp + 12*16] +%endif ; LINUX + mov rsp, [rsp + 5*8] ;; rsp pointer +%endmacro + +%macro SHA512ROUNDS4 7 +%define %%Y0 %1 +%define %%Y1 %2 +%define %%Y2 %3 +%define %%Y3 %4 +%define %%Y4 %5 +%define %%Y6 %6 +%define %%I %7 + + vpaddq %%Y0, %%Y3, [SHA512_CONSTS+32*%%I] + vpermq YTMP3, %%Y3, 0x1b + vpermq YTMP1, %%Y6, 0x39 + vpblendd YTMP1, YTMP3, YTMP1, 0x3f + vpaddq %%Y4, %%Y4, YTMP1 + vsha512msg2 %%Y4, %%Y3 + vsha512rnds2 %%Y2, %%Y1, XWORD(%%Y0) + vperm2i128 %%Y0, %%Y0, %%Y0, 0x01 + vsha512rnds2 %%Y1, %%Y2, XWORD(%%Y0) + vsha512msg1 %%Y6, XWORD(%%Y3) +%endmacro + +%macro SHA512ROUNDS4_FINAL 7 +%define %%Y0 %1 +%define %%Y1 %2 +%define %%Y2 %3 +%define %%Y3 %4 +%define %%Y4 %5 +%define %%Y6 %6 +%define %%I %7 + + vpaddq %%Y0, %%Y3, [SHA512_CONSTS+32*%%I] + vpermq YTMP3, %%Y3, 0x1b + vpermq YTMP1, %%Y6, 0x39 + vpblendd YTMP1, YTMP3, YTMP1, 0x3f + vpaddq %%Y4, %%Y4, YTMP1 + vsha512msg2 %%Y4, %%Y3 + vsha512rnds2 %%Y2, %%Y1, XWORD(%%Y0) + vperm2i128 %%Y0, %%Y0, %%Y0, 0x01 + vsha512rnds2 %%Y1, %%Y2, XWORD(%%Y0) +%endmacro + +;; re-use symbols from AVX codebase +extern SHA512_K_AVX + +mksection .rodata +default rel + +align 32 +SHUF_MASK: + dq 0x0001020304050607, 0x08090a0b0c0d0e0f + dq 0x0001020304050607, 0x08090a0b0c0d0e0f + +mksection .text +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void sha512_ni_x2(SHA512_ARGS *args, UINT64 size_in_blocks) +;; arg1 : pointer to args +;; arg2 : size (in blocks) ;; assumed to be >= 1 +align 32 +MKGLOBAL(sha512_ni_x2,function,internal) +sha512_ni_x2: + mov r11, rsp + sub rsp, frame_size + and rsp, -32 + + or NUM_BLKS, NUM_BLKS + je .done_hash + + ;; load input pointers + mov INP, [args + _data_ptr_sha512 + 0*PTR_SZ] + mov INPb, [args + _data_ptr_sha512 + 1*PTR_SZ] + + ;; load constants pointer + lea SHA512_CONSTS, [rel SHA512_K_AVX] + + ;; load current hash value and transform + vmovdqu STATE0, [args + 0*SHA512NI_DIGEST_ROW_SIZE] + vmovdqu STATE1, [args + 0*SHA512NI_DIGEST_ROW_SIZE + 32] + vmovdqu STATE0b, [args + 1*SHA512NI_DIGEST_ROW_SIZE] + vmovdqu STATE1b, [args + 1*SHA512NI_DIGEST_ROW_SIZE + 32] + + vperm2i128 YTMP1, STATE0, STATE1, 0x20 + vperm2i128 YTMP0, STATE0b, STATE1b, 0x20 + vperm2i128 STATE1, STATE0, STATE1, 0x31 + vperm2i128 STATE1b, STATE0b, STATE1b, 0x31 + vpermq STATE0, YTMP1, 0x1b + vpermq STATE0b, YTMP0, 0x1b + vpermq STATE1, STATE1, 0x1b + vpermq STATE1b, STATE1b, 0x1b + +align 32 +.block_loop: + ;; Save digests + vmovdqa [rsp + frame.ABEF_SAVE], STATE0 + vmovdqa [rsp + frame.CDGH_SAVE], STATE1 + vmovdqa [rsp + frame.ABEF_SAVEb], STATE0b + vmovdqa [rsp + frame.CDGH_SAVEb], STATE1b + + ;; R0- R3 + vmovdqu MSG, [INP+32*0] + vmovdqu MSGb, [INPb+32*0] + vpshufb MSG, MSG, [SHUF_MASK] + vpshufb MSGb, MSGb, [SHUF_MASK] + vmovdqu MSGTMP0, MSG + vmovdqu MSGTMP0b, MSGb + vpaddq MSG, MSG, [SHA512_CONSTS+32*0] + vpaddq MSGb, MSGb, [SHA512_CONSTS+32*0] + vsha512rnds2 STATE1, STATE0, XWORD(MSG) + vsha512rnds2 STATE1b, STATE0b, XWORD(MSGb) + vperm2i128 MSG, MSG, MSG, 0x01 + vperm2i128 MSGb, MSGb, MSGb, 0x01 + vsha512rnds2 STATE0, STATE1, XWORD(MSG) + vsha512rnds2 STATE0b, STATE1b, XWORD(MSGb) + + ;; R4-7 + vmovdqu MSG, [INP+32*1] + vmovdqu MSGb, [INPb+32*1] + vpshufb MSG, MSG, [SHUF_MASK] + vpshufb MSGb, MSGb, [SHUF_MASK] + vmovdqu MSGTMP1, MSG + vmovdqu MSGTMP1b, MSGb + vpaddq MSG, MSG, [SHA512_CONSTS+32*1] + vpaddq MSGb, MSGb, [SHA512_CONSTS+32*1] + vsha512rnds2 STATE1, STATE0, XWORD(MSG) + vsha512rnds2 STATE1b, STATE0b, XWORD(MSGb) + vperm2i128 MSG, MSG, MSG, 0x01 + vperm2i128 MSGb, MSGb, MSGb, 0x01 + vsha512rnds2 STATE0, STATE1, XWORD(MSG) + vsha512rnds2 STATE0b, STATE1b, XWORD(MSGb) + vsha512msg1 MSGTMP0, XWORD(MSGTMP1) + vsha512msg1 MSGTMP0b, XWORD(MSGTMP1b) + + ;; R8-R11 + vmovdqu MSG, [INP+32*2] + vmovdqu MSGb, [INPb+32*2] + vpshufb MSG, MSG, [SHUF_MASK] + vpshufb MSGb, MSGb, [SHUF_MASK] + vmovdqu MSGTMP2, MSG + vmovdqu MSGTMP2b, MSGb + + + vpaddq MSG, MSG, [SHA512_CONSTS+32*2] + vpaddq MSGb, MSGb, [SHA512_CONSTS+32*2] + vsha512rnds2 STATE1, STATE0, XWORD(MSG) + vsha512rnds2 STATE1b, STATE0b, XWORD(MSGb) + vperm2i128 MSG, MSG, MSG, 0x01 + vperm2i128 MSGb, MSGb, MSGb, 0x01 + vsha512rnds2 STATE0, STATE1, XWORD(MSG) + vsha512rnds2 STATE0b, STATE1b, XWORD(MSGb) + vsha512msg1 MSGTMP1, XWORD(MSGTMP2) + vsha512msg1 MSGTMP1b, XWORD(MSGTMP2b) + + ;; R12-15 + vmovdqu MSG, [INP+32*3] + vmovdqu MSGb, [INPb+32*3] + vpshufb MSG, MSG, [SHUF_MASK] + vpshufb MSGb, MSGb, [SHUF_MASK] + vmovdqu YTMP0, MSG + vmovdqu YTMP2, MSGb + + ;; R16-75 + SHA512ROUNDS4 MSG, STATE0, STATE1, YTMP0, MSGTMP0, MSGTMP2, 3 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, YTMP2, MSGTMP0b, MSGTMP2b, 3 + SHA512ROUNDS4 MSG, STATE0, STATE1, MSGTMP0, MSGTMP1, YTMP0, 4 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, MSGTMP0b, MSGTMP1b, YTMP2, 4 + + SHA512ROUNDS4 MSG, STATE0, STATE1, MSGTMP1, MSGTMP2, MSGTMP0, 5 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, MSGTMP1b, MSGTMP2b, MSGTMP0b, 5 + SHA512ROUNDS4 MSG, STATE0, STATE1, MSGTMP2, YTMP0, MSGTMP1, 6 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, MSGTMP2b, YTMP2, MSGTMP1b, 6 + + SHA512ROUNDS4 MSG, STATE0, STATE1, YTMP0, MSGTMP0, MSGTMP2, 7 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, YTMP2, MSGTMP0b, MSGTMP2b, 7 + SHA512ROUNDS4 MSG, STATE0, STATE1, MSGTMP0, MSGTMP1, YTMP0, 8 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, MSGTMP0b, MSGTMP1b, YTMP2, 8 + + SHA512ROUNDS4 MSG, STATE0, STATE1, MSGTMP1, MSGTMP2, MSGTMP0, 9 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, MSGTMP1b, MSGTMP2b, MSGTMP0b, 9 + SHA512ROUNDS4 MSG, STATE0, STATE1, MSGTMP2, YTMP0, MSGTMP1, 10 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, MSGTMP2b, YTMP2, MSGTMP1b, 10 + + SHA512ROUNDS4 MSG, STATE0, STATE1, YTMP0, MSGTMP0, MSGTMP2, 11 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, YTMP2, MSGTMP0b, MSGTMP2b, 11 + SHA512ROUNDS4 MSG, STATE0, STATE1, MSGTMP0, MSGTMP1, YTMP0, 12 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, MSGTMP0b, MSGTMP1b, YTMP2, 12 + + SHA512ROUNDS4 MSG, STATE0, STATE1, MSGTMP1, MSGTMP2, MSGTMP0, 13 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, MSGTMP1b, MSGTMP2b, MSGTMP0b, 13 + SHA512ROUNDS4 MSG, STATE0, STATE1, MSGTMP2, YTMP0, MSGTMP1, 14 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, MSGTMP2b, YTMP2, MSGTMP1b, 14 + + SHA512ROUNDS4 MSG, STATE0, STATE1, YTMP0, MSGTMP0, MSGTMP2, 15 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, YTMP2, MSGTMP0b, MSGTMP2b, 15 + SHA512ROUNDS4 MSG, STATE0, STATE1, MSGTMP0, MSGTMP1, YTMP0, 16 + SHA512ROUNDS4 MSGb, STATE0b, STATE1b, MSGTMP0b, MSGTMP1b, YTMP2, 16 + + SHA512ROUNDS4_FINAL MSG, STATE0, STATE1, MSGTMP1, MSGTMP2, MSGTMP0, 17 + SHA512ROUNDS4_FINAL MSGb, STATE0b, STATE1b, MSGTMP1b, MSGTMP2b, MSGTMP0b, 17 + SHA512ROUNDS4_FINAL MSG, STATE0, STATE1, MSGTMP2, YTMP0, MSGTMP1, 18 + SHA512ROUNDS4_FINAL MSGb, STATE0b, STATE1b, MSGTMP2b, YTMP2, MSGTMP1b, 18 + + ;; R76-79 + vpaddq MSG, YTMP0, [SHA512_CONSTS+32*19] + vpaddq MSGb, YTMP2, [SHA512_CONSTS+32*19] + vsha512rnds2 STATE1, STATE0, XWORD(MSG) + vsha512rnds2 STATE1b, STATE0b, XWORD(MSGb) + vperm2i128 MSG, MSG, MSG, 0x01 + vperm2i128 MSGb, MSGb, MSGb, 0x01 + vsha512rnds2 STATE0, STATE1, XWORD(MSG) + vsha512rnds2 STATE0b, STATE1b, XWORD(MSGb) + + vpaddq STATE0, STATE0, [rsp + frame.ABEF_SAVE] + vpaddq STATE1, STATE1, [rsp + frame.CDGH_SAVE] + vpaddq STATE0b, STATE0b, [rsp + frame.ABEF_SAVEb] + vpaddq STATE1b, STATE1b, [rsp + frame.CDGH_SAVEb] + + lea INP, [INP+128] + lea INPb, [INPb+128] + + dec NUM_BLKS + jne .block_loop + + ;; Update input pointers + mov [args + _data_ptr_sha512 + 0*PTR_SZ], INP + mov [args + _data_ptr_sha512 + 1*PTR_SZ], INPb + + ; Reorder and write back the hash value + vperm2i128 MSGTMP0, STATE0, STATE1, 0x31 + vperm2i128 MSGTMP1, STATE0b, STATE1b, 0x31 + vperm2i128 MSGTMP2, STATE0, STATE1, 0x20 + vperm2i128 YTMP0, STATE0b, STATE1b, 0x20 + vpermq STATE0, MSGTMP0, 0xb1 + vpermq STATE1, MSGTMP2, 0xb1 + vpermq STATE0b, MSGTMP1, 0xb1 + vpermq STATE1b, YTMP0, 0xb1 + + ;; update digests + vmovdqu [args + 0*SHA512NI_DIGEST_ROW_SIZE], STATE0 + vmovdqu [args + 0*SHA512NI_DIGEST_ROW_SIZE + 32], STATE1 + vmovdqu [args + 1*SHA512NI_DIGEST_ROW_SIZE], STATE0b + vmovdqu [args + 1*SHA512NI_DIGEST_ROW_SIZE + 32], STATE1b + + vzeroupper + +.done_hash: + + mov rsp, r11 + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; void call_sha512_ni_x2_avx2_from_c(SHA512_ARGS *args, UINT64 size_in_blocks); +MKGLOBAL(call_sha512_ni_x2_avx2_from_c,function,internal) +call_sha512_ni_x2_avx2_from_c: + FUNC_SAVE + call sha512_ni_x2 + FUNC_RESTORE + ret + +mksection stack-noexec diff --git a/lib/avx2_t4/sha_ni_avx2.c b/lib/avx2_t4/sha_ni_avx2.c index 7cdab60a..2a08e992 100644 --- a/lib/avx2_t4/sha_ni_avx2.c +++ b/lib/avx2_t4/sha_ni_avx2.c @@ -26,6 +26,7 @@ *******************************************************************************/ #include "include/sha_generic.h" +#include "include/sha_mb_mgr.h" #include "include/arch_avx2_type4.h" /* ========================================================================== */ @@ -68,54 +69,60 @@ sha512_ni_avx2(const void *data, const uint64_t length, void *digest) /* ========================================================================== */ /* - * SHA384 API for JOB API + * SHA384 MB API for JOB API */ IMB_JOB * submit_job_sha384_ni_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { - const void *msg = (job->src + job->hash_start_src_offset_in_bytes); - const uint64_t length = job->msg_len_to_hash_in_bytes; - uint64_t tag[8]; - +#ifdef SMX_NI + return submit_flush_job_sha_512(state, job, 2, 1, 384, IMB_SHA_384_BLOCK_SIZE, + SHA384_PAD_SIZE, call_sha512_ni_x2_avx2_from_c, 1); +#else (void) state; - - sha384_ni_avx2(msg, length, tag); - memcpy(job->auth_tag_output, tag, job->auth_tag_output_len_in_bytes); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; + (void) job; + return NULL; +#endif /* ifdef SMX_NI */ } IMB_JOB * flush_job_sha384_ni_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { +#ifdef SMX_NI + return submit_flush_job_sha_512(state, job, 2, 0, 384, IMB_SHA_384_BLOCK_SIZE, + SHA384_PAD_SIZE, call_sha512_ni_x2_avx2_from_c, 1); +#else (void) state; (void) job; return NULL; +#endif /* ifdef SMX_NI */ } /* ========================================================================== */ /* - * SHA512 API for JOB API + * SHA512 MB API for JOB API */ IMB_JOB * submit_job_sha512_ni_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { - const void *msg = (job->src + job->hash_start_src_offset_in_bytes); - const uint64_t length = job->msg_len_to_hash_in_bytes; - uint64_t tag[8]; - +#ifdef SMX_NI + return submit_flush_job_sha_512(state, job, 2, 1, 512, IMB_SHA_512_BLOCK_SIZE, + SHA512_PAD_SIZE, call_sha512_ni_x2_avx2_from_c, 1); +#else (void) state; - - sha512_ni_avx2(msg, length, tag); - memcpy(job->auth_tag_output, tag, job->auth_tag_output_len_in_bytes); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; + (void) job; + return NULL; +#endif /* ifdef SMX_NI */ } IMB_JOB * flush_job_sha512_ni_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { +#ifdef SMX_NI + return submit_flush_job_sha_512(state, job, 2, 0, 512, IMB_SHA_512_BLOCK_SIZE, + SHA512_PAD_SIZE, call_sha512_ni_x2_avx2_from_c, 1); +#else (void) state; (void) job; return NULL; +#endif /* ifdef SMX_NI */ } diff --git a/lib/avx512_t1/mb_mgr_avx512_t1.c b/lib/avx512_t1/mb_mgr_avx512_t1.c index 2f899eda..b1f9e3ed 100644 --- a/lib/avx512_t1/mb_mgr_avx512_t1.c +++ b/lib/avx512_t1/mb_mgr_avx512_t1.c @@ -442,6 +442,9 @@ init_mb_mgr_avx512_t1_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_AVX512; + /* Set architecture type for future checks */ + state->used_arch_type = 1; + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/avx512_t1/sha_mb_avx512.c b/lib/avx512_t1/sha_mb_avx512.c index 4c2cfc07..30bc20fa 100644 --- a/lib/avx512_t1/sha_mb_avx512.c +++ b/lib/avx512_t1/sha_mb_avx512.c @@ -111,7 +111,7 @@ IMB_JOB * submit_job_sha384_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 8, 1, 384, IMB_SHA_512_BLOCK_SIZE, - SHA384_PAD_SIZE, call_sha512_x8_avx512_from_c); + SHA384_PAD_SIZE, call_sha512_x8_avx512_from_c, 0); } IMB_DLL_LOCAL @@ -119,7 +119,7 @@ IMB_JOB * flush_job_sha384_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 8, 0, 384, IMB_SHA_512_BLOCK_SIZE, - SHA384_PAD_SIZE, call_sha512_x8_avx512_from_c); + SHA384_PAD_SIZE, call_sha512_x8_avx512_from_c, 0); } /* ========================================================================== */ @@ -132,7 +132,7 @@ IMB_JOB * submit_job_sha512_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 8, 1, 512, IMB_SHA_512_BLOCK_SIZE, - SHA512_PAD_SIZE, call_sha512_x8_avx512_from_c); + SHA512_PAD_SIZE, call_sha512_x8_avx512_from_c, 0); } IMB_DLL_LOCAL @@ -140,5 +140,5 @@ IMB_JOB * flush_job_sha512_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 8, 0, 512, IMB_SHA_512_BLOCK_SIZE, - SHA512_PAD_SIZE, call_sha512_x8_avx512_from_c); + SHA512_PAD_SIZE, call_sha512_x8_avx512_from_c, 0); } diff --git a/lib/avx512_t2/mb_mgr_avx512_t2.c b/lib/avx512_t2/mb_mgr_avx512_t2.c index 04e5c002..af3a0bd4 100644 --- a/lib/avx512_t2/mb_mgr_avx512_t2.c +++ b/lib/avx512_t2/mb_mgr_avx512_t2.c @@ -449,6 +449,9 @@ init_mb_mgr_avx512_t2_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_AVX512; + /* Set architecture type for future checks */ + state->used_arch_type = 2; + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/avx_t1/mb_mgr_avx_t1.c b/lib/avx_t1/mb_mgr_avx_t1.c index a536badc..29a182c2 100644 --- a/lib/avx_t1/mb_mgr_avx_t1.c +++ b/lib/avx_t1/mb_mgr_avx_t1.c @@ -345,6 +345,9 @@ init_mb_mgr_avx_t1_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_AVX; + /* Set architecture type for future checks */ + state->used_arch_type = 1; + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/avx_t1/sha_mb_avx.c b/lib/avx_t1/sha_mb_avx.c index e452f132..4c7f269d 100644 --- a/lib/avx_t1/sha_mb_avx.c +++ b/lib/avx_t1/sha_mb_avx.c @@ -111,7 +111,7 @@ IMB_JOB * submit_job_sha384_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 2, 1, 384, IMB_SHA_512_BLOCK_SIZE, - SHA384_PAD_SIZE, call_sha512_x2_avx_from_c); + SHA384_PAD_SIZE, call_sha512_x2_avx_from_c, 0); } IMB_DLL_LOCAL @@ -119,7 +119,7 @@ IMB_JOB * flush_job_sha384_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 2, 0, 384, IMB_SHA_512_BLOCK_SIZE, - SHA384_PAD_SIZE, call_sha512_x2_avx_from_c); + SHA384_PAD_SIZE, call_sha512_x2_avx_from_c, 0); } /* ========================================================================== */ @@ -132,7 +132,7 @@ IMB_JOB * submit_job_sha512_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 2, 1, 512, IMB_SHA_512_BLOCK_SIZE, - SHA512_PAD_SIZE, call_sha512_x2_avx_from_c); + SHA512_PAD_SIZE, call_sha512_x2_avx_from_c, 0); } IMB_DLL_LOCAL @@ -140,5 +140,5 @@ IMB_JOB * flush_job_sha512_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 2, 0, 512, IMB_SHA_512_BLOCK_SIZE, - SHA512_PAD_SIZE, call_sha512_x2_avx_from_c); + SHA512_PAD_SIZE, call_sha512_x2_avx_from_c, 0); } diff --git a/lib/avx_t2/mb_mgr_avx_t2.c b/lib/avx_t2/mb_mgr_avx_t2.c index 4b74a36e..b36744fc 100644 --- a/lib/avx_t2/mb_mgr_avx_t2.c +++ b/lib/avx_t2/mb_mgr_avx_t2.c @@ -350,6 +350,9 @@ init_mb_mgr_avx_t2_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_AVX; + /* Set architecture type for future checks */ + state->used_arch_type = 2; + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/include/arch_avx2_type4.h b/lib/include/arch_avx2_type4.h index 2a23fefa..84ab9129 100644 --- a/lib/include/arch_avx2_type4.h +++ b/lib/include/arch_avx2_type4.h @@ -75,11 +75,6 @@ flush_job_sha384_ni_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); IMB_JOB * flush_job_sha512_ni_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); -IMB_JOB * -submit_job_sha384_ni_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); -IMB_JOB * -submit_job_sha512_ni_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); - IMB_JOB * flush_job_hmac_sha_384_ni_avx2(MB_MGR_HMAC_SHA_512_OOO *state); IMB_JOB * @@ -90,4 +85,7 @@ submit_job_hmac_sha_384_ni_avx2(MB_MGR_HMAC_SHA_512_OOO *state, IMB_JOB *job); IMB_JOB * submit_job_hmac_sha_512_ni_avx2(MB_MGR_HMAC_SHA_512_OOO *state, IMB_JOB *job); +void +call_sha512_ni_x2_avx2_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); + #endif /* IMB_ASM_AVX2_T4_H */ diff --git a/lib/include/constants.inc b/lib/include/constants.inc index 152c1ca3..a259ea0a 100644 --- a/lib/include/constants.inc +++ b/lib/include/constants.inc @@ -59,6 +59,7 @@ ;; Sanity checks to fail build if not satisfied %define SHA1NI_DIGEST_ROW_SIZE (NUM_SHA1_DIGEST_WORDS * SHA1_DIGEST_WORD_SIZE) %define SHA256NI_DIGEST_ROW_SIZE (NUM_SHA256_DIGEST_WORDS * SHA256_DIGEST_WORD_SIZE) +%define SHA512NI_DIGEST_ROW_SIZE (NUM_SHA512_DIGEST_WORDS * SHA512_DIGEST_WORD_SIZE) %define MD5_BLK_SZ 128 ; in bytes %define SHA1_BLK_SZ 64 ; in bytes diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 08fe979c..b15d566c 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -333,6 +333,7 @@ typedef struct { uint64_t unused_lanes; HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_SHA1_LANES]; uint32_t num_lanes_inuse; + uint32_t total_num_lanes; uint64_t road_block; } MB_MGR_HMAC_SHA_1_OOO; @@ -342,6 +343,7 @@ typedef struct { uint64_t unused_lanes; HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_SHA1_LANES]; uint32_t num_lanes_inuse; + uint32_t total_num_lanes; uint64_t road_block; } MB_MGR_SHA_1_OOO; @@ -351,6 +353,7 @@ typedef struct { uint64_t unused_lanes; HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_SHA256_LANES]; uint32_t num_lanes_inuse; + uint32_t total_num_lanes; uint64_t road_block; } MB_MGR_HMAC_SHA_256_OOO; @@ -360,6 +363,7 @@ typedef struct { uint64_t unused_lanes; HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_SHA256_LANES]; uint32_t num_lanes_inuse; + uint32_t total_num_lanes; uint64_t road_block; } MB_MGR_SHA_256_OOO; @@ -368,6 +372,7 @@ typedef struct { DECLARE_ALIGNED(uint16_t lens[8], 16); uint64_t unused_lanes; HMAC_SHA512_LANE_DATA ldata[AVX512_NUM_SHA512_LANES]; + uint32_t total_num_lanes; uint64_t road_block; } MB_MGR_HMAC_SHA_512_OOO; @@ -377,6 +382,7 @@ typedef struct { uint64_t unused_lanes; HMAC_SHA512_LANE_DATA ldata[AVX512_NUM_SHA512_LANES]; uint32_t num_lanes_inuse; + uint32_t total_num_lanes; uint64_t road_block; } MB_MGR_SHA_512_OOO; @@ -391,6 +397,7 @@ typedef struct { uint64_t unused_lanes; HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_MD5_LANES]; uint32_t num_lanes_inuse; + uint32_t total_num_lanes; uint64_t road_block; } MB_MGR_HMAC_MD5_OOO; diff --git a/lib/include/mb_mgr_burst.h b/lib/include/mb_mgr_burst.h index fc5bf650..092cc16b 100644 --- a/lib/include/mb_mgr_burst.h +++ b/lib/include/mb_mgr_burst.h @@ -551,6 +551,131 @@ submit_burst_hmac_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, co return completed_jobs; } +__forceinline uint32_t +submit_burst_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const int run_check, + const IMB_HASH_ALG hash_alg) +{ + uint32_t i, completed_jobs = 0; + + if (run_check) { + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + /* validate job */ + if (is_job_invalid(state, job, IMB_CIPHER_NULL, hash_alg, IMB_DIR_ENCRYPT, + job->key_len_in_bytes)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + + if (hash_alg == IMB_AUTH_SHA_1) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_SHA1(state->sha_1_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = FLUSH_JOB_SHA1(state->sha_1_ooo, job)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else if (hash_alg == IMB_AUTH_SHA_224) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_SHA224(state->sha_224_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = FLUSH_JOB_SHA224(state->sha_224_ooo, job)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else if (hash_alg == IMB_AUTH_SHA_256) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_SHA256(state->sha_256_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = FLUSH_JOB_SHA256(state->sha_256_ooo, job)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else if (hash_alg == IMB_AUTH_SHA_384) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_SHA384(state->sha_384_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = FLUSH_JOB_SHA384(state->sha_384_ooo, job)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else if (hash_alg == IMB_AUTH_SHA_512) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_SHA512(state->sha_512_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = FLUSH_JOB_SHA512(state->sha_512_ooo, job)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } + + return completed_jobs; +} + __forceinline uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash, const int run_check) @@ -580,6 +705,16 @@ submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs case IMB_AUTH_HMAC_SHA_512: return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_HMAC_SHA_512); + case IMB_AUTH_SHA_1: + return submit_burst_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_SHA_1); + case IMB_AUTH_SHA_224: + return submit_burst_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_SHA_224); + case IMB_AUTH_SHA_256: + return submit_burst_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_SHA_256); + case IMB_AUTH_SHA_384: + return submit_burst_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_SHA_384); + case IMB_AUTH_SHA_512: + return submit_burst_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_SHA_512); default: break; } diff --git a/lib/include/mb_mgr_datastruct.inc b/lib/include/mb_mgr_datastruct.inc index da06c842..3f4c0085 100644 --- a/lib/include/mb_mgr_datastruct.inc +++ b/lib/include/mb_mgr_datastruct.inc @@ -321,6 +321,7 @@ FIELD _lens, 32, 32 FIELD _unused_lanes, 8, 8 FIELD _ldata, _HMAC_SHA1_LANE_DATA_size*MAX_SHA1_LANES, _HMAC_SHA1_LANE_DATA_align FIELD _num_lanes_inuse_sha1, 4, 4 +FIELD _total_num_lanes_sha1, 4, 4 FIELD _road_block_sha1, 8, 8 END_FIELDS %assign _MB_MGR_HMAC_SHA_1_OOO_size _FIELD_OFFSET @@ -348,6 +349,7 @@ FIELD _lens_sha256, 16*2, 16 FIELD _unused_lanes_sha256, 8, 8 FIELD _ldata_sha256, _HMAC_SHA1_LANE_DATA_size * MAX_SHA256_LANES, _HMAC_SHA1_LANE_DATA_align FIELD _num_lanes_inuse_sha256, 4, 4 +FIELD _total_num_lanes_sha256, 4, 4 FIELD _road_block_sha256, 8, 8 END_FIELDS %assign _MB_MGR_HMAC_SHA_256_OOO_size _FIELD_OFFSET @@ -375,6 +377,7 @@ FIELD _args_sha512, _SHA512_ARGS_size, _SHA512_ARGS_align FIELD _lens_sha512, 16, 16 FIELD _unused_lanes_sha512, 8, 8 FIELD _ldata_sha512, _SHA512_LANE_DATA_size * MAX_SHA512_LANES, _SHA512_LANE_DATA_align +FIELD _total_num_lanes_sha512, 4, 4 FIELD _road_block_sha512, 8, 8 END_FIELDS %assign _MB_MGR_HMAC_SHA_512_OOO_size _FIELD_OFFSET @@ -404,6 +407,7 @@ FIELD _lens_md5, MAX_MD5_LANES*2, 16 FIELD _unused_lanes_md5, 8, 8 FIELD _ldata_md5, _HMAC_SHA1_LANE_DATA_size * MAX_MD5_LANES, _HMAC_SHA1_LANE_DATA_align FIELD _num_lanes_inuse_md5, 4, 8 +FIELD _total_num_lanes_md5, 4, 4 FIELD _road_block_md5, 8, 8 END_FIELDS %assign _MB_MGR_HMAC_MD5_OOO_size _FIELD_OFFSET diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 7d1ebbee..702dc371 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -70,6 +70,18 @@ copy_bswap8_array_mb(void *dst, const void *src, const size_t num, const size_t outp[i] = bswap8(inp[lane + i * offset]); } +__forceinline void +copy_bswap8_array_mb_ni(void *dst, const void *src, const size_t num, const unsigned lane, + const int digest_row_sz) +{ + uint64_t *outp = (uint64_t *) dst; + const uint64_t *inp = (const uint64_t *) src; + size_t i; + + for (i = 0; i < num; i++) + outp[i] = bswap8(inp[digest_row_sz * lane + i]); +} + __forceinline void sha1_mb_init_digest(uint32_t *digest, const unsigned lane) { @@ -155,6 +167,19 @@ sha384_mb_init_digest(uint64_t *digest, const unsigned lane) digest[lane + 7 * 8] = SHA384_H7; } +__forceinline void +sha384_ni_mb_init_digest(uint64_t *digest, const unsigned lane) +{ + digest[8 * lane + 0] = SHA384_H0; + digest[8 * lane + 1] = SHA384_H1; + digest[8 * lane + 2] = SHA384_H2; + digest[8 * lane + 3] = SHA384_H3; + digest[8 * lane + 4] = SHA384_H4; + digest[8 * lane + 5] = SHA384_H5; + digest[8 * lane + 6] = SHA384_H6; + digest[8 * lane + 7] = SHA384_H7; +} + __forceinline void sha512_mb_init_digest(uint64_t *digest, const unsigned lane) { @@ -168,6 +193,19 @@ sha512_mb_init_digest(uint64_t *digest, const unsigned lane) digest[lane + 7 * 8] = SHA512_H7; } +__forceinline void +sha512_ni_mb_init_digest(uint64_t *digest, const unsigned lane) +{ + digest[8 * lane + 0] = SHA512_H0; + digest[8 * lane + 1] = SHA512_H1; + digest[8 * lane + 2] = SHA512_H2; + digest[8 * lane + 3] = SHA512_H3; + digest[8 * lane + 4] = SHA512_H4; + digest[8 * lane + 5] = SHA512_H5; + digest[8 * lane + 6] = SHA512_H6; + digest[8 * lane + 7] = SHA512_H7; +} + __forceinline void sha_mb_generic_init(void *digest, const int sha_type, const unsigned lane) { @@ -192,6 +230,10 @@ sha_ni_mb_generic_init(void *digest, const int sha_type, const unsigned lane) sha224_ni_mb_init_digest(digest, lane); else if (sha_type == 256) sha256_ni_mb_init_digest(digest, lane); + else if (sha_type == 384) + sha384_ni_mb_init_digest(digest, lane); + else if (sha_type == 512) + sha512_ni_mb_init_digest(digest, lane); } __forceinline void @@ -219,6 +261,10 @@ sha_ni_mb_generic_write_digest(void *dst, const void *src, const int sha_type, c copy_bswap4_array_mb_ni(dst, src, NUM_SHA_224_DIGEST_WORDS, lane, 8); else if (sha_type == 256) copy_bswap4_array_mb_ni(dst, src, NUM_SHA_256_DIGEST_WORDS, lane, 8); + else if (sha_type == 384) + copy_bswap8_array_mb_ni(dst, src, NUM_SHA_384_DIGEST_WORDS, lane, 8); + else if (sha_type == 512) + copy_bswap8_array_mb_ni(dst, src, NUM_SHA_512_DIGEST_WORDS, lane, 8); } __forceinline void @@ -545,7 +591,8 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, const unsigned __forceinline IMB_JOB * submit_flush_job_sha_512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job, const unsigned max_jobs, const int is_submit, const int sha_type, const uint64_t blk_size, - const uint64_t pad_size, void (*fn)(SHA512_ARGS *, uint64_t)) + const uint64_t pad_size, void (*fn)(SHA512_ARGS *, uint64_t), + const int shani) { unsigned lane, min_idx; IMB_JOB *ret_job = NULL; @@ -561,7 +608,10 @@ submit_flush_job_sha_512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job, const unsigned state->num_lanes_inuse++; state->args.data_ptr[lane] = job->src + job->hash_start_src_offset_in_bytes; - sha_mb_generic_init(state->args.digest, sha_type, lane); + if (shani) + sha_ni_mb_generic_init(state->args.digest, sha_type, lane); + else + sha_mb_generic_init(state->args.digest, sha_type, lane); /* copy job data in and set up initial blocks */ state->ldata[lane].job_in_lane = job; @@ -656,8 +706,12 @@ submit_flush_job_sha_512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job, const unsigned /* put back processed packet into unused lanes, set job as complete */ state->unused_lanes = (state->unused_lanes << 4) | min_idx; state->num_lanes_inuse--; - sha_mb_generic_write_digest(ret_job->auth_tag_output, state->args.digest, sha_type, 8, - min_idx); + if (shani) + sha_ni_mb_generic_write_digest(ret_job->auth_tag_output, state->args.digest, + sha_type, min_idx); + else + sha_mb_generic_write_digest(ret_job->auth_tag_output, state->args.digest, sha_type, + 8, min_idx); ret_job->status |= IMB_STATUS_COMPLETED_AUTH; state->ldata[min_idx].job_in_lane = NULL; return ret_job; diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 03cee069..004c6cc0 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -961,8 +961,10 @@ typedef struct IMB_MGR { uint64_t flags; /**< passed to alloc_mb_mgr() */ uint64_t features; /**< reflects features of multi-buffer instance */ - uint64_t reserved[5]; /**< reserved for the future */ - uint32_t used_arch; /**< Architecture being used */ + uint64_t reserved[4]; /**< reserved for the future */ + uint8_t reserved2[7]; /**< reserved for the future */ + uint8_t used_arch_type; /**< Architecture type being used */ + uint32_t used_arch; /**< Architecture being used */ int imb_errno; /**< per mb_mgr error status */ @@ -3891,6 +3893,50 @@ imb_self_test_set_cb(IMB_MGR *state, imb_self_test_cb_t cb_fn, void *cb_arg); */ IMB_DLL_EXPORT int imb_self_test_get_cb(IMB_MGR *state, imb_self_test_cb_t *cb_fn, void **cb_arg); + +/** + * @brief API to get a string with the architecture type being used. + * + * init_mb_mgr_XXX() must be called before this function call, + * where XXX is the desired architecture (can be auto). + * + * @param [in] state pointer to IMB_MGR + * @param [out] arch_type string with architecture type + * @param [out] description string with description of the arch type + * + * @return operation status. + * @retval 0 success + * @retval IMB_ERR_NULL_MBMGR invalid \a mb_mgr pointer + */ +IMB_DLL_EXPORT int +imb_get_arch_type_string(const IMB_MGR *state, const char **arch_type, const char **description); + +/** + * @brief Retrieves minimum burst size for good performance on hash algorithms. + * + * Depending on the architecture used, this function returns the minimum + * burst size to be used for good performance on the hash-only burst API. + * Note that this will not return a value for best performance, but the minimum needed + * to start maximizing the CPU core (i.e. enough buffers to utilize efficiently the CPU core + * resources, taking into account that when buffers have different sizes, a higher burst size is + * recommended). + * + * The output burst size may also be 1 (in case of a synchronous single-buffer implementation + * or 0 if the algorithm is not supported by the API). + * + * @param [in] mb_mgr pointer to IMB MGR structure + * @param [in] algo hash algorithm + * @param [out] out_burst_size pointer to store min burst size + * + * @return operation status. + * @retval 0 success + * @retval IMB_ERR_HASH_ALGO not supported \a algo + * @retval IMB_ERR_NULL_MBMGR invalid \a mb_mgr pointer + * @retval IMB_ERR_NULL_BURST invalid \a out_burst_size pointer + */ +IMB_DLL_EXPORT int +imb_hash_burst_get_size(const IMB_MGR *mb_mgr, const IMB_HASH_ALG algo, unsigned *out_burst_size); + #ifdef __cplusplus } #endif diff --git a/lib/libIPSec_MB.def b/lib/libIPSec_MB.def index 4f5c283c..5ad31e45 100644 --- a/lib/libIPSec_MB.def +++ b/lib/libIPSec_MB.def @@ -728,3 +728,5 @@ EXPORTS submit_job_nocheck_avx2_t4 @702 get_next_job_avx2_t4 @703 get_completed_job_avx2_t4 @704 + imb_hash_burst_get_size @705 + imb_get_arch_type_string @706 diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 6badc270..844d3839 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -329,6 +329,9 @@ init_mb_mgr_sse_no_aesni_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_NOAESNI; + /* Set architecture type for future checks */ + state->used_arch_type = 1; + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/sse_t1/mb_mgr_sse_t1.c b/lib/sse_t1/mb_mgr_sse_t1.c index e09f7c5f..1f8eaee2 100644 --- a/lib/sse_t1/mb_mgr_sse_t1.c +++ b/lib/sse_t1/mb_mgr_sse_t1.c @@ -351,6 +351,9 @@ init_mb_mgr_sse_t1_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_SSE; + /* Set architecture type for future checks */ + state->used_arch_type = 1; + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/sse_t1/sha_mb_sse.c b/lib/sse_t1/sha_mb_sse.c index c8dcb904..841717d5 100644 --- a/lib/sse_t1/sha_mb_sse.c +++ b/lib/sse_t1/sha_mb_sse.c @@ -101,7 +101,7 @@ IMB_JOB * submit_job_sha384_sse(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 2, 1, 384, IMB_SHA_512_BLOCK_SIZE, - SHA384_PAD_SIZE, call_sha512_x2_sse_from_c); + SHA384_PAD_SIZE, call_sha512_x2_sse_from_c, 0); } IMB_DLL_LOCAL @@ -109,7 +109,7 @@ IMB_JOB * flush_job_sha384_sse(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 2, 0, 384, IMB_SHA_512_BLOCK_SIZE, - SHA384_PAD_SIZE, call_sha512_x2_sse_from_c); + SHA384_PAD_SIZE, call_sha512_x2_sse_from_c, 0); } /* ========================================================================== */ @@ -122,7 +122,7 @@ IMB_JOB * submit_job_sha512_sse(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 2, 1, 512, IMB_SHA_512_BLOCK_SIZE, - SHA512_PAD_SIZE, call_sha512_x2_sse_from_c); + SHA512_PAD_SIZE, call_sha512_x2_sse_from_c, 0); } IMB_DLL_LOCAL @@ -130,5 +130,5 @@ IMB_JOB * flush_job_sha512_sse(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_512(state, job, 2, 0, 512, IMB_SHA_512_BLOCK_SIZE, - SHA512_PAD_SIZE, call_sha512_x2_sse_from_c); + SHA512_PAD_SIZE, call_sha512_x2_sse_from_c, 0); } diff --git a/lib/sse_t2/mb_mgr_sse_t2.c b/lib/sse_t2/mb_mgr_sse_t2.c index c912f81e..b0d8fb25 100644 --- a/lib/sse_t2/mb_mgr_sse_t2.c +++ b/lib/sse_t2/mb_mgr_sse_t2.c @@ -353,6 +353,9 @@ init_mb_mgr_sse_t2_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_SSE; + /* Set architecture type for future checks */ + state->used_arch_type = 2; + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/sse_t3/mb_mgr_sse_t3.c b/lib/sse_t3/mb_mgr_sse_t3.c index 40e5a1dd..e4a7e85a 100644 --- a/lib/sse_t3/mb_mgr_sse_t3.c +++ b/lib/sse_t3/mb_mgr_sse_t3.c @@ -354,6 +354,9 @@ init_mb_mgr_sse_t3_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_SSE; + /* Set architecture type for future checks */ + state->used_arch_type = 3; + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/win_x64.mak b/lib/win_x64.mak index d582dcb1..6bb188e9 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -377,7 +377,8 @@ lib_objs1 = \ $(OBJ_DIR)\quic_chacha20_poly1305.obj \ $(OBJ_DIR)\hmac_ipad_opad.obj \ $(OBJ_DIR)\cipher_suite_id.obj \ - $(OBJ_DIR)\sm4_sse.obj + $(OBJ_DIR)\sm4_sse.obj \ + $(OBJ_DIR)\capabilities.obj lib_objs2 = \ $(OBJ_DIR)\mb_mgr_aes192_cbc_enc_flush_avx.obj \ @@ -596,6 +597,7 @@ avx2_t4_objs = \ $(OBJ_DIR)\sm3_msg_avx2.obj \ $(OBJ_DIR)\sm3_hmac_avx2.obj \ $(OBJ_DIR)\sha512_x1_ni_avx2.obj \ + $(OBJ_DIR)\sha512_x2_ni_avx2.obj \ $(OBJ_DIR)\sha_ni_avx2.obj \ $(OBJ_DIR)\sha512_hmac_ni_avx2.obj diff --git a/lib/x86_64/alloc.c b/lib/x86_64/alloc.c index 2840788a..9102d7fe 100644 --- a/lib/x86_64/alloc.c +++ b/lib/x86_64/alloc.c @@ -45,11 +45,8 @@ #define ALIGN(x, y) ((x + (y - 1)) & (~(y - 1))) #define OOO_INFO(imb_mgr_ooo_ptr_name__, ooo_mgr_type__) \ - { \ - offsetof(IMB_MGR, imb_mgr_ooo_ptr_name__), \ - ALIGN(sizeof(ooo_mgr_type__), ALIGNMENT), \ - offsetof(ooo_mgr_type__, road_block) \ - } + { offsetof(IMB_MGR, imb_mgr_ooo_ptr_name__), ALIGN(sizeof(ooo_mgr_type__), ALIGNMENT), \ + offsetof(ooo_mgr_type__, road_block) } const struct { size_t ooo_ptr_offset; diff --git a/lib/x86_64/capabilities.c b/lib/x86_64/capabilities.c new file mode 100644 index 00000000..143845ed --- /dev/null +++ b/lib/x86_64/capabilities.c @@ -0,0 +1,135 @@ +/******************************************************************************* + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "intel-ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +int +imb_hash_burst_get_size(const IMB_MGR *mb_mgr, const IMB_HASH_ALG algo, unsigned *out_burst_size) +{ +#ifdef SAFE_PARAM + if (mb_mgr == NULL) + return IMB_ERR_NULL_MBMGR; + + if (out_burst_size == NULL) + return IMB_ERR_NULL_BURST; +#endif + + switch (algo) { + case IMB_AUTH_HMAC_SHA_1: + *out_burst_size = + ((MB_MGR_HMAC_SHA_1_OOO *) (mb_mgr->hmac_sha_1_ooo))->total_num_lanes; + break; + case IMB_AUTH_SHA_1: + *out_burst_size = ((MB_MGR_SHA_1_OOO *) (mb_mgr->sha_1_ooo))->total_num_lanes; + break; + case IMB_AUTH_HMAC_SHA_224: + *out_burst_size = + ((MB_MGR_HMAC_SHA_256_OOO *) (mb_mgr->hmac_sha_224_ooo))->total_num_lanes; + break; + case IMB_AUTH_SHA_224: + *out_burst_size = ((MB_MGR_SHA_256_OOO *) (mb_mgr->sha_224_ooo))->total_num_lanes; + break; + case IMB_AUTH_HMAC_SHA_256: + *out_burst_size = + ((MB_MGR_HMAC_SHA_256_OOO *) (mb_mgr->hmac_sha_256_ooo))->total_num_lanes; + break; + case IMB_AUTH_SHA_256: + *out_burst_size = ((MB_MGR_SHA_256_OOO *) (mb_mgr->sha_256_ooo))->total_num_lanes; + break; + case IMB_AUTH_HMAC_SHA_384: + *out_burst_size = + ((MB_MGR_HMAC_SHA_512_OOO *) (mb_mgr->hmac_sha_384_ooo))->total_num_lanes; + break; + case IMB_AUTH_SHA_384: + *out_burst_size = ((MB_MGR_SHA_512_OOO *) (mb_mgr->sha_384_ooo))->total_num_lanes; + break; + case IMB_AUTH_HMAC_SHA_512: + *out_burst_size = + ((MB_MGR_HMAC_SHA_512_OOO *) (mb_mgr->hmac_sha_512_ooo))->total_num_lanes; + break; + case IMB_AUTH_SHA_512: + *out_burst_size = ((MB_MGR_SHA_512_OOO *) (mb_mgr->sha_512_ooo))->total_num_lanes; + break; + default: + *out_burst_size = 0; + return IMB_ERR_HASH_ALGO; + } + + return 0; +} + +int +imb_get_arch_type_string(const IMB_MGR *state, const char **arch_type, const char **description) +{ +#ifdef SAFE_PARAM + if (state == NULL) + return IMB_ERR_NULL_MBMGR; + if (arch_type == NULL) + return EINVAL; +#endif + struct arch_type_map { + IMB_ARCH arch; + uint8_t type; + const char *arch_type; + const char *description; + }; + + const struct arch_type_map arch_type_mappings[] = { + { IMB_ARCH_NOAESNI, 0, "AESNI Emulation", "CPU ISA: SSE" }, + { IMB_ARCH_SSE, 1, "SSE Type 1", "CPU ISA: AES, PCLMUL, SSE" }, + { IMB_ARCH_SSE, 2, "SSE Type 2", "CPU ISA: AES, PCLMUL, SSE, SHA-NI" }, + { IMB_ARCH_SSE, 3, "SSE Type 3", "CPU ISA: AES, PCLMUL, SSE, SHA-NI, GFNI" }, + { IMB_ARCH_AVX, 1, "AVX Type 1", "CPU ISA: AES, PCLMUL, SSE, AVX" }, + { IMB_ARCH_AVX, 2, "AVX Type 2", "CPU ISA: AES, PCLMUL, SSE, AVX, SHA-NI" }, + { IMB_ARCH_AVX2, 1, "AVX2 Type 1", "CPU ISA: AES, PCLMUL, SSE, AVX, AVX2" }, + { IMB_ARCH_AVX2, 2, "AVX2 Type 2", + "CPU ISA: VAES, VPCLMUL, SSE, AVX, AVX2, SHA-NI, GFNI" }, + { IMB_ARCH_AVX2, 3, "AVX2 Type 3", + "CPU ISA: VAES, VPCLMUL, SSE, AVX, AVX2, SHA-NI, GFNI, IFMA" }, + { IMB_ARCH_AVX2, 4, "AVX2 Type 4", + "CPU ISA: VAES, VPCLMUL, SSE, AVX, AVX2, SHA-NI, GFNI, IFMA, SHA512-NI, SM3-NI, " + "SM4-NI" }, + { IMB_ARCH_AVX512, 1, "AVX512 Type 1", + "CPU ISA: AES, PCLMUL, SSE, AVX, AVX2, AVX512" }, + { IMB_ARCH_AVX512, 2, "AVX512 Type 2", + "CPU ISA: VAES, VPCLMUL, SSE, AVX, AVX2, AVX512, GFNI, SHA-NI" }, + }; + + for (unsigned int i = 0; i < IMB_DIM(arch_type_mappings); i++) { + if (arch_type_mappings[i].arch == state->used_arch && + arch_type_mappings[i].type == state->used_arch_type) { + *arch_type = arch_type_mappings[i].arch_type; + if (description != NULL) + *description = arch_type_mappings[i].description; + + break; + } + *arch_type = "Invalid arch type"; + } + return 0; +} diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index bdba93e8..bb015009 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -133,6 +133,7 @@ ooo_mgr_hmac_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_1_OOO, road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + p_mgr->total_num_lanes = num_lanes; for (i = 0; i < num_lanes; i++) { p_mgr->ldata[i].extra_block[IMB_SHA1_BLOCK_SIZE] = 0x80; @@ -163,6 +164,7 @@ ooo_mgr_hmac_sha224_reset(void *p_ooo_mgr, const unsigned num_lanes) memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_256_OOO, road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + p_mgr->total_num_lanes = num_lanes; for (i = 0; i < num_lanes; i++) { p_mgr->ldata[i].extra_block[IMB_SHA_256_BLOCK_SIZE] = 0x80; @@ -193,6 +195,7 @@ ooo_mgr_hmac_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes) memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_256_OOO, road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + p_mgr->total_num_lanes = num_lanes; for (i = 0; i < num_lanes; i++) { p_mgr->ldata[i].extra_block[IMB_SHA_256_BLOCK_SIZE] = 0x80; @@ -223,6 +226,7 @@ ooo_mgr_hmac_sha384_reset(void *p_ooo_mgr, const unsigned num_lanes) memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_512_OOO, road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + p_mgr->total_num_lanes = num_lanes; for (i = 0; i < num_lanes; i++) { p_mgr->ldata[i].extra_block[IMB_SHA_384_BLOCK_SIZE] = 0x80; @@ -259,6 +263,7 @@ ooo_mgr_hmac_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes) memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_512_OOO, road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + p_mgr->total_num_lanes = num_lanes; for (i = 0; i < num_lanes; i++) { p_mgr->ldata[i].extra_block[IMB_SHA_512_BLOCK_SIZE] = 0x80; @@ -295,6 +300,7 @@ ooo_mgr_hmac_md5_reset(void *p_ooo_mgr, const unsigned num_lanes) memset(p_mgr, 0, offsetof(MB_MGR_HMAC_MD5_OOO, road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + p_mgr->total_num_lanes = num_lanes; for (i = 0; i < num_lanes; i++) { p_mgr->ldata[i].extra_block[64] = 0x80; @@ -340,6 +346,7 @@ ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) MB_MGR_SHA_1_OOO *p_mgr = (MB_MGR_SHA_1_OOO *) p_ooo_mgr; memset(p_mgr, 0, offsetof(MB_MGR_SHA_1_OOO, road_block)); + p_mgr->total_num_lanes = num_lanes; if (num_lanes == 2) p_mgr->unused_lanes = 0xF10; /* SHANI */ @@ -358,6 +365,7 @@ ooo_mgr_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes) MB_MGR_SHA_256_OOO *p_mgr = (MB_MGR_SHA_256_OOO *) p_ooo_mgr; memset(p_mgr, 0, offsetof(MB_MGR_SHA_256_OOO, road_block)); + p_mgr->total_num_lanes = num_lanes; if (num_lanes == 2) p_mgr->unused_lanes = 0xF10; /* SHANI */ @@ -376,6 +384,7 @@ ooo_mgr_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes) MB_MGR_SHA_512_OOO *p_mgr = (MB_MGR_SHA_512_OOO *) p_ooo_mgr; memset(p_mgr, 0, offsetof(MB_MGR_SHA_512_OOO, road_block)); + p_mgr->total_num_lanes = num_lanes; if (num_lanes == AVX_NUM_SHA512_LANES) p_mgr->unused_lanes = 0xF10; diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index c03f107c..935dbf28 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -325,10 +325,7 @@ static const uint8_t tdes_ede_cbc_cipher_text[] = { }; #define ADD_CIPHER_VECTOR(_cmode, _key, _iv, _plain, _cipher, _descr) \ - { \ - _cmode, _key, sizeof(_key), _iv, sizeof(_iv), _plain, sizeof(_plain), _cipher, \ - _descr \ - } + { _cmode, _key, sizeof(_key), _iv, sizeof(_iv), _plain, sizeof(_plain), _cipher, _descr } static const struct self_test_cipher_vector cipher_vectors[] = { ADD_CIPHER_VECTOR(IMB_CIPHER_CBC, aes_cbc_128_key, aes_cbc_128_iv, aes_cbc_128_plain_text, @@ -575,9 +572,7 @@ const uint8_t sha512_digest[] = { 0x20, 0x4a, 0x8f, 0xc6, 0xdd, 0xa8, 0x2f, 0x0a 0x03, 0x54, 0xec, 0x63, 0x12, 0x38, 0xca, 0x34, 0x45 }; #define ADD_SHA_VECTOR(_hmode, _msg, _digest, _descr) \ - { \ - _hmode, NULL, 0, _msg, sizeof(_msg), _digest, sizeof(_digest), NULL, 0, _descr \ - } + { _hmode, NULL, 0, _msg, sizeof(_msg), _digest, sizeof(_digest), NULL, 0, _descr } /* * Test vector from https://csrc.nist.gov/csrc/media/publications/fips/198/ @@ -685,8 +680,8 @@ static const uint8_t hmac_sha512_digest[] = { 0xfc, 0x25, 0xe2, 0x40, 0x65, 0x8c #define ADD_HMAC_SHA_VECTOR(_hmode, _key, _msg, _digest, _descr) \ { \ - _hmode, _key, sizeof(_key), _msg, sizeof(_msg), _digest, sizeof(_digest), NULL, 0, \ - _descr \ + _hmode, _key, sizeof(_key), _msg, sizeof(_msg), _digest, sizeof(_digest), \ + NULL, 0, _descr \ } /* @@ -712,8 +707,8 @@ static const uint8_t aes_cmac_256_tag[] = { 0x15, 0x67, 0x27, 0xDC, 0x08, 0x78, #define ADD_CMAC_VECTOR(_hmode, _key, _msg, _digest, _descr) \ { \ - _hmode, _key, sizeof(_key), _msg, sizeof(_msg), _digest, sizeof(_digest), NULL, 0, \ - _descr \ + _hmode, _key, sizeof(_key), _msg, sizeof(_msg), _digest, sizeof(_digest), \ + NULL, 0, _descr \ } /* @@ -768,10 +763,8 @@ static const uint8_t aes_gmac_256_tag[] = { 0x77, 0x46, 0x0D, 0x6F, 0xB1, 0x87, 0x46, 0xAD, 0xCD, 0xFB, 0xB7, 0xF9, 0x13, 0xA1 }; #define ADD_GMAC_VECTOR(_hmode, _key, _iv, _msg, _tag, _descr) \ - { \ - _hmode, _key, sizeof(_key), _msg, sizeof(_msg), _tag, sizeof(_tag), _iv, \ - sizeof(_iv), _descr \ - } + { _hmode, _key, sizeof(_key), _msg, sizeof(_msg), \ + _tag, sizeof(_tag), _iv, sizeof(_iv), _descr } static const struct self_test_hash_vector hash_vectors[] = { ADD_SHA_VECTOR(IMB_AUTH_SHA_1, sha_message, sha1_digest, "SHA1"), @@ -1095,10 +1088,9 @@ static const uint8_t aes_gcm_256_tag[] = { 0x76, 0xfc, 0x6e, 0xce, 0x0f, 0x4e, 0 0xcd, 0xdf, 0x88, 0x53, 0xbb, 0x2d, 0x55, 0x1b }; #define ADD_GCM_VECTOR(_key, _iv, _aad, _plain, _cipher, _tag, _descr) \ - { \ - IMB_AUTH_AES_GMAC, IMB_CIPHER_GCM, _key, sizeof(_key), _iv, sizeof(_iv), _aad, \ - sizeof(_aad), _plain, sizeof(_plain), _cipher, _tag, sizeof(_tag), _descr \ - } + { IMB_AUTH_AES_GMAC, IMB_CIPHER_GCM, _key, sizeof(_key), _iv, \ + sizeof(_iv), _aad, sizeof(_aad), _plain, sizeof(_plain), \ + _cipher, _tag, sizeof(_tag), _descr } static const struct self_test_aead_gcm_vector aead_gcm_vectors[] = { ADD_GCM_VECTOR(aes_gcm_128_key, aes_gcm_128_iv, aes_gcm_128_aad, aes_gcm_128_plain_text, @@ -1357,11 +1349,9 @@ static const uint8_t aes_ccm_256_cipher_text[] = { 0x21, 0x61, 0x63, 0xDE, 0xCF, static const uint8_t aes_ccm_256_tag[] = { 0xCA, 0x8A, 0xFA, 0xA2, 0x3F, 0x22, 0x3E, 0x64 }; #define ADD_CCM_VECTOR(_key, _nonce, _aad, _plain, _cipher, _tag, _descr) \ - { \ - IMB_AUTH_AES_CCM, IMB_CIPHER_CCM, _key, sizeof(_key), _nonce, sizeof(_nonce), \ - _aad, sizeof(_aad), _plain, sizeof(_plain), _cipher, _tag, sizeof(_tag), \ - _descr \ - } + { IMB_AUTH_AES_CCM, IMB_CIPHER_CCM, _key, sizeof(_key), _nonce, \ + sizeof(_nonce), _aad, sizeof(_aad), _plain, sizeof(_plain), \ + _cipher, _tag, sizeof(_tag), _descr } static const struct self_test_aead_ccm_vector aead_ccm_vectors[] = { ADD_CCM_VECTOR(aes_ccm_128_key, aes_ccm_128_nonce, aes_ccm_128_aad, aes_ccm_128_plain_text, diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 242a4f66..d2fd1e22 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -1318,6 +1318,161 @@ translate_cipher_mode(const enum test_cipher_mode_e test_mode) return c_mode; } +/* + * This function translates enum test_hash_alg_e to be used by ipsec_mb + * library + */ +static IMB_HASH_ALG +translate_hash_alg(const enum test_hash_alg_e test_mode) +{ + IMB_HASH_ALG hash_alg = IMB_AUTH_NULL; + + switch (test_mode) { + case TEST_SHA1: + hash_alg = IMB_AUTH_SHA_1; + break; + case TEST_SHA_224: + hash_alg = IMB_AUTH_SHA_224; + break; + case TEST_SHA_256: + hash_alg = IMB_AUTH_SHA_256; + break; + case TEST_SHA_384: + hash_alg = IMB_AUTH_SHA_384; + break; + case TEST_SHA_512: + hash_alg = IMB_AUTH_SHA_512; + break; + case TEST_SHA1_HMAC: + hash_alg = IMB_AUTH_HMAC_SHA_1; + break; + case TEST_SHA_224_HMAC: + hash_alg = IMB_AUTH_HMAC_SHA_224; + break; + case TEST_SHA_256_HMAC: + hash_alg = IMB_AUTH_HMAC_SHA_256; + break; + case TEST_SHA_384_HMAC: + hash_alg = IMB_AUTH_HMAC_SHA_384; + break; + case TEST_SHA_512_HMAC: + hash_alg = IMB_AUTH_HMAC_SHA_512; + break; + case TEST_XCBC: + hash_alg = IMB_AUTH_AES_XCBC; + break; + case TEST_HASH_CCM: + hash_alg = IMB_AUTH_AES_CCM; + break; + case TEST_HASH_GCM: + if (segment_size != 0) + hash_alg = IMB_AUTH_GCM_SGL; + else + hash_alg = IMB_AUTH_AES_GMAC; + break; + case TEST_DOCSIS_CRC32: + hash_alg = IMB_AUTH_DOCSIS_CRC32; + break; + case TEST_NULL_HASH: + hash_alg = IMB_AUTH_NULL; + break; + case TEST_HASH_CMAC: + hash_alg = IMB_AUTH_AES_CMAC; + break; + case TEST_HASH_CMAC_BITLEN: + hash_alg = IMB_AUTH_AES_CMAC_BITLEN; + break; + case TEST_HASH_CMAC_256: + hash_alg = IMB_AUTH_AES_CMAC_256; + break; + case TEST_HASH_POLY1305: + hash_alg = IMB_AUTH_POLY1305; + break; + case TEST_AEAD_POLY1305: + if (segment_size != 0) + hash_alg = IMB_AUTH_CHACHA20_POLY1305_SGL; + else + hash_alg = IMB_AUTH_CHACHA20_POLY1305; + break; + case TEST_PON_CRC_BIP: + hash_alg = IMB_AUTH_PON_CRC_BIP; + break; + case TEST_ZUC_EIA3: + hash_alg = IMB_AUTH_ZUC_EIA3_BITLEN; + break; + case TEST_ZUC256_EIA3: + hash_alg = IMB_AUTH_ZUC256_EIA3_BITLEN; + break; + case TEST_SNOW3G_UIA2: + hash_alg = IMB_AUTH_SNOW3G_UIA2_BITLEN; + break; + case TEST_KASUMI_UIA1: + hash_alg = IMB_AUTH_KASUMI_UIA1; + break; + case TEST_AES_GMAC_128: + hash_alg = IMB_AUTH_AES_GMAC_128; + break; + case TEST_AES_GMAC_192: + hash_alg = IMB_AUTH_AES_GMAC_192; + break; + case TEST_AES_GMAC_256: + hash_alg = IMB_AUTH_AES_GMAC_256; + break; + case TEST_AUTH_GHASH: + hash_alg = IMB_AUTH_GHASH; + break; + case TEST_AUTH_SNOW_V_AEAD: + hash_alg = IMB_AUTH_SNOW_V_AEAD; + break; + case TEST_CRC32_ETHERNET_FCS: + hash_alg = IMB_AUTH_CRC32_ETHERNET_FCS; + break; + case TEST_CRC32_SCTP: + hash_alg = IMB_AUTH_CRC32_SCTP; + break; + case TEST_CRC32_WIMAX_OFDMA_DATA: + hash_alg = IMB_AUTH_CRC32_WIMAX_OFDMA_DATA; + break; + case TEST_CRC24_LTE_A: + hash_alg = IMB_AUTH_CRC24_LTE_A; + break; + case TEST_CRC24_LTE_B: + hash_alg = IMB_AUTH_CRC24_LTE_B; + break; + case TEST_CRC16_X25: + hash_alg = IMB_AUTH_CRC16_X25; + break; + case TEST_CRC16_FP_DATA: + hash_alg = IMB_AUTH_CRC16_FP_DATA; + break; + case TEST_CRC11_FP_HEADER: + hash_alg = IMB_AUTH_CRC11_FP_HEADER; + break; + case TEST_CRC10_IUUP_DATA: + hash_alg = IMB_AUTH_CRC10_IUUP_DATA; + break; + case TEST_CRC8_WIMAX_OFDMA_HCS: + hash_alg = IMB_AUTH_CRC8_WIMAX_OFDMA_HCS; + break; + case TEST_CRC7_FP_HEADER: + hash_alg = IMB_AUTH_CRC7_FP_HEADER; + break; + case TEST_CRC6_IUUP_HEADER: + hash_alg = IMB_AUTH_CRC6_IUUP_HEADER; + break; + case TEST_AUTH_SM3: + hash_alg = IMB_AUTH_SM3; + break; + case TEST_SM3_HMAC: + hash_alg = IMB_AUTH_HMAC_SM3; + break; + default: + break; + } + + return hash_alg; +} + static uint32_t get_next_size(const uint32_t index) { @@ -1847,173 +2002,81 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, const uint32_t num_iter, uint8 job_template.auth_tag_output = (uint8_t *) digest; + /* Translating enum to the API's one */ + job_template.hash_alg = translate_hash_alg(params->hash_alg); switch (params->hash_alg) { - case TEST_SHA1: - job_template.hash_alg = IMB_AUTH_SHA_1; - break; - case TEST_SHA_224: - job_template.hash_alg = IMB_AUTH_SHA_224; - break; - case TEST_SHA_256: - job_template.hash_alg = IMB_AUTH_SHA_256; - break; - case TEST_SHA_384: - job_template.hash_alg = IMB_AUTH_SHA_384; - break; - case TEST_SHA_512: - job_template.hash_alg = IMB_AUTH_SHA_512; - break; case TEST_XCBC: job_template.u.XCBC._k1_expanded = k1_expanded; job_template.u.XCBC._k2 = k2; job_template.u.XCBC._k3 = k3; - job_template.hash_alg = IMB_AUTH_AES_XCBC; - break; - case TEST_HASH_CCM: - job_template.hash_alg = IMB_AUTH_AES_CCM; - break; - case TEST_HASH_GCM: - if (segment_size != 0) - job_template.hash_alg = IMB_AUTH_GCM_SGL; - else - job_template.hash_alg = IMB_AUTH_AES_GMAC; - break; - case TEST_DOCSIS_CRC32: - job_template.hash_alg = IMB_AUTH_DOCSIS_CRC32; - break; - case TEST_NULL_HASH: - job_template.hash_alg = IMB_AUTH_NULL; break; case TEST_HASH_CMAC: job_template.u.CMAC._key_expanded = k1_expanded; job_template.u.CMAC._skey1 = k2; job_template.u.CMAC._skey2 = k3; - job_template.hash_alg = IMB_AUTH_AES_CMAC; break; case TEST_HASH_CMAC_BITLEN: job_template.u.CMAC._key_expanded = k1_expanded; job_template.u.CMAC._skey1 = k2; job_template.u.CMAC._skey2 = k3; - job_template.hash_alg = IMB_AUTH_AES_CMAC_BITLEN; break; case TEST_HASH_CMAC_256: job_template.u.CMAC._key_expanded = k1_expanded; job_template.u.CMAC._skey1 = k2; job_template.u.CMAC._skey2 = k3; - job_template.hash_alg = IMB_AUTH_AES_CMAC_256; break; case TEST_HASH_POLY1305: job_template.u.POLY1305._key = k1_expanded; - job_template.hash_alg = IMB_AUTH_POLY1305; - break; - case TEST_AEAD_POLY1305: - if (segment_size != 0) - job_template.hash_alg = IMB_AUTH_CHACHA20_POLY1305_SGL; - else - job_template.hash_alg = IMB_AUTH_CHACHA20_POLY1305; break; case TEST_PON_CRC_BIP: - job_template.hash_alg = IMB_AUTH_PON_CRC_BIP; job_template.cipher_start_src_offset_in_bytes = 8; break; case TEST_ZUC_EIA3: - job_template.hash_alg = IMB_AUTH_ZUC_EIA3_BITLEN; job_template.u.ZUC_EIA3._key = k3; job_template.u.ZUC_EIA3._iv = (uint8_t *) &auth_iv; break; case TEST_ZUC256_EIA3: - job_template.hash_alg = IMB_AUTH_ZUC256_EIA3_BITLEN; job_template.u.ZUC_EIA3._key = k3; job_template.u.ZUC_EIA3._iv = (uint8_t *) &auth_iv; break; case TEST_SNOW3G_UIA2: - job_template.hash_alg = IMB_AUTH_SNOW3G_UIA2_BITLEN; job_template.u.SNOW3G_UIA2._key = k3; job_template.u.SNOW3G_UIA2._iv = (uint8_t *) &auth_iv; break; case TEST_KASUMI_UIA1: - job_template.hash_alg = IMB_AUTH_KASUMI_UIA1; job_template.u.KASUMI_UIA1._key = k3; break; case TEST_AES_GMAC_128: - job_template.hash_alg = IMB_AUTH_AES_GMAC_128; IMB_AES128_GCM_PRE(mb_mgr, gcm_key, &gdata_key); job_template.u.GMAC._key = &gdata_key; job_template.u.GMAC._iv = (uint8_t *) &auth_iv; job_template.u.GMAC.iv_len_in_bytes = 12; break; case TEST_AES_GMAC_192: - job_template.hash_alg = IMB_AUTH_AES_GMAC_192; IMB_AES192_GCM_PRE(mb_mgr, gcm_key, &gdata_key); job_template.u.GMAC._key = &gdata_key; job_template.u.GMAC._iv = (uint8_t *) &auth_iv; job_template.u.GMAC.iv_len_in_bytes = 12; break; case TEST_AES_GMAC_256: - job_template.hash_alg = IMB_AUTH_AES_GMAC_256; IMB_AES256_GCM_PRE(mb_mgr, gcm_key, &gdata_key); job_template.u.GMAC._key = &gdata_key; job_template.u.GMAC._iv = (uint8_t *) &auth_iv; job_template.u.GMAC.iv_len_in_bytes = 12; break; case TEST_AUTH_GHASH: - job_template.hash_alg = IMB_AUTH_GHASH; IMB_GHASH_PRE(mb_mgr, gcm_key, &gdata_key); job_template.u.GHASH._key = &gdata_key; job_template.u.GHASH._init_tag = (uint8_t *) &auth_iv; break; - case TEST_AUTH_SNOW_V_AEAD: - job_template.hash_alg = IMB_AUTH_SNOW_V_AEAD; - break; - case TEST_CRC32_ETHERNET_FCS: - job_template.hash_alg = IMB_AUTH_CRC32_ETHERNET_FCS; - break; - case TEST_CRC32_SCTP: - job_template.hash_alg = IMB_AUTH_CRC32_SCTP; - break; - case TEST_CRC32_WIMAX_OFDMA_DATA: - job_template.hash_alg = IMB_AUTH_CRC32_WIMAX_OFDMA_DATA; - break; - case TEST_CRC24_LTE_A: - job_template.hash_alg = IMB_AUTH_CRC24_LTE_A; - break; - case TEST_CRC24_LTE_B: - job_template.hash_alg = IMB_AUTH_CRC24_LTE_B; - break; - case TEST_CRC16_X25: - job_template.hash_alg = IMB_AUTH_CRC16_X25; - break; - case TEST_CRC16_FP_DATA: - job_template.hash_alg = IMB_AUTH_CRC16_FP_DATA; - break; - case TEST_CRC11_FP_HEADER: - job_template.hash_alg = IMB_AUTH_CRC11_FP_HEADER; - break; - case TEST_CRC10_IUUP_DATA: - job_template.hash_alg = IMB_AUTH_CRC10_IUUP_DATA; - break; - case TEST_CRC8_WIMAX_OFDMA_HCS: - job_template.hash_alg = IMB_AUTH_CRC8_WIMAX_OFDMA_HCS; - break; - case TEST_CRC7_FP_HEADER: - job_template.hash_alg = IMB_AUTH_CRC7_FP_HEADER; - break; - case TEST_CRC6_IUUP_HEADER: - job_template.hash_alg = IMB_AUTH_CRC6_IUUP_HEADER; - break; - case TEST_AUTH_SM3: - job_template.hash_alg = IMB_AUTH_SM3; - break; case TEST_SM3_HMAC: job_template.u.HMAC._hashed_auth_key_xor_ipad = (uint8_t *) ipad; job_template.u.HMAC._hashed_auth_key_xor_opad = (uint8_t *) opad; - job_template.hash_alg = IMB_AUTH_HMAC_SM3; break; default: /* HMAC hash algorithm */ job_template.u.HMAC._hashed_auth_key_xor_ipad = (uint8_t *) ipad; job_template.u.HMAC._hashed_auth_key_xor_opad = (uint8_t *) opad; - job_template.hash_alg = (IMB_HASH_ALG) params->hash_alg; break; } if (tag_size == 0) @@ -3961,15 +4024,28 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - /* currently only HMAC-SHAx algs supported by hash-only burst API */ - if (test_api == TEST_API_HASH_BURST && - ((custom_job_params.hash_alg != TEST_SHA1_HMAC) && - (custom_job_params.hash_alg != TEST_SHA_224_HMAC) && - (custom_job_params.hash_alg != TEST_SHA_256_HMAC) && - (custom_job_params.hash_alg != TEST_SHA_384_HMAC) && - (custom_job_params.hash_alg != TEST_SHA_512_HMAC))) { - fprintf(stderr, "Unsupported hash-only burst API algorithm selected\n"); - return EXIT_FAILURE; + /* only a few algorithms support the hash-only burst API */ + if (test_api == TEST_API_HASH_BURST) { + uint32_t optim_burst_size; + IMB_MGR *aux_mgr = alloc_mb_mgr(0); + + if (aux_mgr == NULL) { + fprintf(stderr, "Error allocating MB_MGR structure!\n"); + return EXIT_FAILURE; + } + init_mb_mgr_auto(aux_mgr, NULL); + + if (imb_hash_burst_get_size(aux_mgr, translate_hash_alg(custom_job_params.hash_alg), + &optim_burst_size) == IMB_ERR_HASH_ALGO) { + fprintf(stderr, "Unsupported hash-only burst API algorithm selected\n"); + free_mb_mgr(aux_mgr); + return EXIT_FAILURE; + } + + if (optim_burst_size > burst_size) + fprintf(stderr, "NOTE: Burst size is lower than the minimum size for an " + "optimal performance\n"); + free_mb_mgr(aux_mgr); } if (test_api == TEST_API_DIRECT && ((custom_job_params.cipher_mode != TEST_GCM) && @@ -4161,18 +4237,39 @@ main(int argc, char *argv[]) } } - if (archs[ARCH_SSE]) { - IMB_MGR *p_mgr = alloc_mb_mgr(flags); + IMB_MGR *p_mgr = alloc_mb_mgr(flags); - if (p_mgr == NULL) { - fprintf(stderr, "Error allocating MB_MGR structure!\n"); - return EXIT_FAILURE; + if (p_mgr == NULL) { + fprintf(stderr, "Error allocating MB_MGR structure!\n"); + return EXIT_FAILURE; + } + + fprintf(stderr, "Testing "); + for (enum arch_type_e arch = ARCH_SSE; arch <= ARCH_AVX512; arch++) { + if (archs[arch] == 0) + continue; + + switch (arch) { + case ARCH_SSE: + init_mb_mgr_sse(p_mgr); + break; + case ARCH_AVX: + init_mb_mgr_avx(p_mgr); + break; + case ARCH_AVX2: + init_mb_mgr_avx2(p_mgr); + break; + default: /* ARCH_AV512 */ + init_mb_mgr_avx512(p_mgr); + break; } - init_mb_mgr_sse(p_mgr); - fprintf(stderr, "%s SHA extensions (shani) for SSE arch\n", - (p_mgr->features & IMB_FEATURE_SHANI) ? "Using" : "Not using"); - free_mb_mgr(p_mgr); + const char *arch_type; + + imb_get_arch_type_string(p_mgr, &arch_type, NULL); + fprintf(stderr, "\"%s\" ", arch_type); } + fprintf(stderr, "implementation/s\n"); + free_mb_mgr(p_mgr); memset(t_info, 0, sizeof(t_info)); init_offsets(cache_type); diff --git a/test/fuzz-app/job_api_fuzz_test.c b/test/fuzz-app/job_api_fuzz_test.c index dee0a3fc..5d1644b3 100644 --- a/test/fuzz-app/job_api_fuzz_test.c +++ b/test/fuzz-app/job_api_fuzz_test.c @@ -34,9 +34,8 @@ #include #include -#define BUFF_SIZE (32 * 1024 * 1024) -#define MAX_BURST_JOBS 32 -#define MAX_SGL_SEGS 32 +#define BUFF_SIZE (32 * 1024 * 1024) +#define MAX_SGL_SEGS 32 int LLVMFuzzerTestOneInput(const uint8_t *, size_t); @@ -492,7 +491,7 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) if (dataSize < sizeof(IMB_JOB)) return 0; - if (num_jobs > MAX_BURST_JOBS || num_jobs == 0 || key_len == 0) + if (num_jobs > IMB_MAX_BURST_SIZE || num_jobs == 0 || key_len == 0) return 0; if (cipher_dir != NULL) { @@ -574,7 +573,7 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) IMB_SUBMIT_JOB(p_mgr); } } else if (burst) { - IMB_JOB *jobs[MAX_BURST_JOBS] = { NULL }; + IMB_JOB *jobs[IMB_MAX_BURST_SIZE] = { NULL }; while (IMB_GET_NEXT_BURST(p_mgr, num_jobs, jobs) < (uint32_t) num_jobs) IMB_FLUSH_BURST(p_mgr, num_jobs, jobs); @@ -609,7 +608,7 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) IMB_SUBMIT_BURST(p_mgr, num_jobs, jobs); } else if (cipher_burst) { - IMB_JOB jobs[MAX_BURST_JOBS] = { 0 }; + IMB_JOB jobs[IMB_MAX_BURST_SIZE] = { 0 }; for (i = 0; i < num_jobs; i++) { job = &jobs[i]; @@ -636,7 +635,7 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) IMB_SUBMIT_CIPHER_BURST(p_mgr, jobs, num_jobs, cipher, dir, key_len); } else if (hash_burst) { - IMB_JOB jobs[MAX_BURST_JOBS] = { 0 }; + IMB_JOB jobs[IMB_MAX_BURST_SIZE] = { 0 }; for (i = 0; i < num_jobs; i++) { job = &jobs[i]; diff --git a/test/kat-app/aes_cbc_test.c b/test/kat-app/aes_cbc_test.c index 3bc8f062..1c6ea1bf 100644 --- a/test/kat-app/aes_cbc_test.c +++ b/test/kat-app/aes_cbc_test.c @@ -36,8 +36,6 @@ #include "utils.h" #include "cipher_test.h" -#define MAX_BURST_JOBS 64 - int cbc_test(struct IMB_MGR *mb_mgr); @@ -185,7 +183,7 @@ test_aes_many_burst(struct IMB_MGR *mb_mgr, void *enc_keys, void *dec_keys, cons const int dir, const int order, const IMB_CIPHER_MODE cipher, const int in_place, const int key_len, const int num_jobs) { - struct IMB_JOB *job, *jobs[MAX_BURST_JOBS] = { NULL }; + struct IMB_JOB *job, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; uint8_t padding[16]; uint8_t **targets = malloc(num_jobs * sizeof(void *)); int i, completed_jobs, jobs_rx = 0, ret = -1; @@ -291,7 +289,7 @@ test_aes_many_cipher_burst(struct IMB_MGR *mb_mgr, void *enc_keys, void *dec_key const int dir, const IMB_CIPHER_MODE cipher, const int in_place, const int key_len, const int num_jobs) { - struct IMB_JOB *job, jobs[MAX_BURST_JOBS]; + struct IMB_JOB *job, jobs[IMB_MAX_BURST_SIZE]; uint8_t padding[16]; uint8_t **targets = malloc(num_jobs * sizeof(void *)); int i, completed_jobs, jobs_rx = 0, ret = -1; @@ -545,7 +543,7 @@ test_cbc_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx128, int cbc_test(struct IMB_MGR *mb_mgr) { - const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, MAX_BURST_JOBS }; + const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, IMB_MAX_BURST_SIZE }; unsigned i; int errors = 0; struct test_suite_context ctx128; diff --git a/test/kat-app/aes_cfb_test.c b/test/kat-app/aes_cfb_test.c index 775c8196..c1199480 100644 --- a/test/kat-app/aes_cfb_test.c +++ b/test/kat-app/aes_cfb_test.c @@ -36,8 +36,6 @@ #include "utils.h" #include "cipher_test.h" -#define MAX_BURST_JOBS 64 - int cfb_test(struct IMB_MGR *mb_mgr); diff --git a/test/kat-app/aes_test.c b/test/kat-app/aes_test.c index e40970c8..f8dfd870 100644 --- a/test/kat-app/aes_test.c +++ b/test/kat-app/aes_test.c @@ -35,8 +35,6 @@ #include "utils.h" -#define MAX_BURST_JOBS 64 - int aes_test(struct IMB_MGR *mb_mgr); @@ -838,11 +836,17 @@ static const uint8_t DOCRC17_CT[] = { #define DOCRC17_FRAME_LEN DIM(DOCRC17_PT) #define MK_DOCRC_VEC(_n) \ - { \ - _n##_FRAME_LEN, _n##_KEY, _n##_KEY_LEN, _n##_IV, _n##_PT, _n##_CT, \ - _n##_HASH_OFFSET, _n##_HASH_LENGTH, _n##_CIPHER_OFFSET, \ - _n##_CIPHER_LENGTH, _n##_CRC \ - } + { _n##_FRAME_LEN, \ + _n##_KEY, \ + _n##_KEY_LEN, \ + _n##_IV, \ + _n##_PT, \ + _n##_CT, \ + _n##_HASH_OFFSET, \ + _n##_HASH_LENGTH, \ + _n##_CIPHER_OFFSET, \ + _n##_CIPHER_LENGTH, \ + _n##_CRC } struct docsis_crc_vector { uint64_t frame_len; @@ -1023,7 +1027,7 @@ test_aes_many_burst(struct IMB_MGR *mb_mgr, void *enc_keys, void *dec_keys, cons const int dir, const int order, const IMB_CIPHER_MODE cipher, const int in_place, const int key_len, const int num_jobs) { - struct IMB_JOB *job, *jobs[MAX_BURST_JOBS] = { NULL }; + struct IMB_JOB *job, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; uint8_t padding[16]; uint8_t **targets = malloc(num_jobs * sizeof(void *)); int i, completed_jobs, jobs_rx = 0, ret = -1; @@ -1454,7 +1458,7 @@ test_docrc_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx128, int aes_test(struct IMB_MGR *mb_mgr) { - const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, MAX_BURST_JOBS }; + const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, IMB_MAX_BURST_SIZE }; unsigned i; int errors = 0; struct test_suite_context ctx128; diff --git a/test/kat-app/api_test.c b/test/kat-app/api_test.c index 39d1615f..ca0bbd23 100644 --- a/test/kat-app/api_test.c +++ b/test/kat-app/api_test.c @@ -38,8 +38,6 @@ #define __func__ __FUNCTION__ #endif -#define MAX_BURST_JOBS 32 - int api_test(struct IMB_MGR *mb_mgr); @@ -644,8 +642,8 @@ static int is_submit_burst_invalid(struct IMB_MGR *mb_mgr, const struct IMB_JOB *job, const int test_num, int expected_errnum) { - IMB_JOB *jobs[MAX_BURST_JOBS] = { NULL }; - uint32_t i, completed_jobs, n_jobs = MAX_BURST_JOBS; + IMB_JOB *jobs[IMB_MAX_BURST_SIZE] = { NULL }; + uint32_t i, completed_jobs, n_jobs = IMB_MAX_BURST_SIZE; int err; while (IMB_GET_NEXT_BURST(mb_mgr, n_jobs, jobs) < n_jobs) @@ -693,8 +691,8 @@ is_submit_burst_invalid(struct IMB_MGR *mb_mgr, const struct IMB_JOB *job, const static int test_burst_api(struct IMB_MGR *mb_mgr) { - struct IMB_JOB *job = NULL, *jobs[MAX_BURST_JOBS] = { NULL }; - uint32_t i, completed_jobs, n_jobs = MAX_BURST_JOBS; + struct IMB_JOB *job = NULL, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; + uint32_t i, completed_jobs, n_jobs = IMB_MAX_BURST_SIZE; struct IMB_JOB **null_jobs = NULL; int err; diff --git a/test/kat-app/gcm_ctr_vectors_test.h b/test/kat-app/gcm_ctr_vectors_test.h index 5c455878..2c6735df 100644 --- a/test/kat-app/gcm_ctr_vectors_test.h +++ b/test/kat-app/gcm_ctr_vectors_test.h @@ -51,21 +51,14 @@ struct gcm_ctr_vector { }; #define vector(N) \ - { \ - K##N, (KBITS(K##N)), IV##N, sizeof(IV##N), A##N, A##N##_len, P##N, sizeof(P##N), \ - C##N, T##N, sizeof(T##N) \ - } + { K##N, (KBITS(K##N)), IV##N, sizeof(IV##N), A##N, A##N##_len, \ + P##N, sizeof(P##N), C##N, T##N, sizeof(T##N) } #define extra_vector(N) \ - { \ - K##N, (KBITS(K##N)), IV##N, sizeof(IV##N), A##N, A##N##_len, P##N, P##N##_len, \ - C##N, T##N, sizeof(T##N) \ - } + { K##N, (KBITS(K##N)), IV##N, sizeof(IV##N), A##N, A##N##_len, \ + P##N, P##N##_len, C##N, T##N, sizeof(T##N) } #define ghash_vector(N) \ - { \ - K##N, (KBITS(K##N)), NULL, 0, NULL, 0, P##N, sizeof(P##N), NULL, T##N, \ - sizeof(T##N) \ - } + { K##N, (KBITS(K##N)), NULL, 0, NULL, 0, P##N, sizeof(P##N), NULL, T##N, sizeof(T##N) } struct MB_MGR; extern int diff --git a/test/kat-app/hmac_sha1_test.c b/test/kat-app/hmac_sha1_test.c index 301497a5..167b90fd 100644 --- a/test/kat-app/hmac_sha1_test.c +++ b/test/kat-app/hmac_sha1_test.c @@ -38,8 +38,6 @@ int hmac_sha1_test(struct IMB_MGR *mb_mgr); -#define MAX_BURST_JOBS 32 - extern const struct mac_test hmac_sha1_test_kat_json[]; static int hmac_sha1_job_ok(const struct mac_test *vec, const struct IMB_JOB *job, const uint8_t *auth, @@ -178,7 +176,7 @@ test_hmac_sha1(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const uint32_ static int test_hmac_sha1_burst(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const uint32_t num_jobs) { - struct IMB_JOB *job, *jobs[MAX_BURST_JOBS] = { NULL }; + struct IMB_JOB *job, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; uint8_t padding[16]; uint8_t **auths = malloc(num_jobs * sizeof(void *)); uint32_t i = 0, jobs_rx = 0; @@ -288,7 +286,7 @@ static int test_hmac_sha1_hash_burst(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const uint32_t num_jobs) { - struct IMB_JOB *job, jobs[MAX_BURST_JOBS] = { 0 }; + struct IMB_JOB *job, jobs[IMB_MAX_BURST_SIZE] = { 0 }; uint8_t padding[16]; uint8_t **auths = malloc(num_jobs * sizeof(void *)); uint32_t i = 0, jobs_rx = 0; @@ -442,7 +440,7 @@ hmac_sha1_test(struct IMB_MGR *mb_mgr) uint32_t num_jobs; test_suite_start(&ts, "HMAC-SHA1"); - for (num_jobs = 1; num_jobs <= MAX_BURST_JOBS; num_jobs++) + for (num_jobs = 1; num_jobs <= IMB_MAX_BURST_SIZE; num_jobs++) test_hmac_sha1_std_vectors(mb_mgr, num_jobs, &ts); errors = test_suite_end(&ts); diff --git a/test/kat-app/hmac_sm3_test.c b/test/kat-app/hmac_sm3_test.c index 474b94f4..2a9ffa19 100644 --- a/test/kat-app/hmac_sm3_test.c +++ b/test/kat-app/hmac_sm3_test.c @@ -38,8 +38,6 @@ int hmac_sm3_test(struct IMB_MGR *mb_mgr); -#define MAX_BURST_JOBS 32 - extern const struct mac_test hmac_sm3_test_kat_json[]; static int hmac_sm3_job_ok(const struct mac_test *vec, const struct IMB_JOB *job, const uint8_t *auth, @@ -169,7 +167,7 @@ test_hmac_sm3(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const uint32_t static int test_hmac_sm3_burst(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const uint32_t num_jobs) { - struct IMB_JOB *job, *jobs[MAX_BURST_JOBS] = { NULL }; + struct IMB_JOB *job, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; uint8_t padding[16]; uint8_t **auths = malloc(num_jobs * sizeof(void *)); uint32_t i = 0, jobs_rx = 0; @@ -320,7 +318,7 @@ hmac_sm3_test(struct IMB_MGR *mb_mgr) uint32_t num_jobs; test_suite_start(&ts, "SM3"); - for (num_jobs = 1; num_jobs <= MAX_BURST_JOBS; num_jobs++) + for (num_jobs = 1; num_jobs <= IMB_MAX_BURST_SIZE; num_jobs++) test_hmac_sm3_std_vectors(mb_mgr, num_jobs, &ts); errors = test_suite_end(&ts); diff --git a/test/kat-app/pon_test.c b/test/kat-app/pon_test.c index 77e9ddd8..abfccfc4 100644 --- a/test/kat-app/pon_test.c +++ b/test/kat-app/pon_test.c @@ -337,16 +337,12 @@ static const uint8_t OUT13_PON[] = { #define LENCIPH13_PON (LENBIP13_PON - OFFSET13_PON) #define ponvector(tname) \ - { \ - KEY##tname, IV##tname, IN##tname, OUT##tname, BIPOUT##tname, LENBIP##tname, \ - LENCIPH##tname, OFFSET##tname \ - } + { KEY##tname, IV##tname, IN##tname, OUT##tname, \ + BIPOUT##tname, LENBIP##tname, LENCIPH##tname, OFFSET##tname } #define pon_no_ctr_vector(tname) \ - { \ - NULL, NULL, IN##tname, OUT##tname, BIPOUT##tname, LENBIP##tname, LENCIPH##tname, \ - OFFSET##tname \ - } + { NULL, NULL, IN##tname, OUT##tname, \ + BIPOUT##tname, LENBIP##tname, LENCIPH##tname, OFFSET##tname } static const struct pon_test_vector { const uint8_t *key; diff --git a/test/kat-app/sha_test.c b/test/kat-app/sha_test.c index 8d6e41c7..4da90535 100644 --- a/test/kat-app/sha_test.c +++ b/test/kat-app/sha_test.c @@ -172,6 +172,118 @@ test_sha(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const int num_jobs, return ret; } +static int +test_sha_hash_burst(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const int num_jobs, + const int sha_type) +{ + struct IMB_JOB *job, jobs[IMB_MAX_BURST_SIZE] = { 0 }; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + int i = 0, jobs_rx = 0, ret = -1; + int completed_jobs = 0; + IMB_HASH_ALG hash_alg; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + switch (sha_type) { + case 1: + hash_alg = IMB_AUTH_SHA_1; + break; + case 224: + hash_alg = IMB_AUTH_SHA_224; + break; + case 256: + hash_alg = IMB_AUTH_SHA_256; + break; + case 384: + hash_alg = IMB_AUTH_SHA_384; + break; + case 512: + default: + hash_alg = IMB_AUTH_SHA_512; + break; + } + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = vec->tagSize / 8 + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize / 8; + job->src = (const void *) vec->msg; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = hash_alg; + + job->user_data = auths[i]; + } + + completed_jobs = IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, num_jobs, hash_alg); + if (completed_jobs != num_jobs) { + int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, imb_get_strerror(err)); + goto end; + } else { + printf("submit_burst error: not enough " + "jobs returned!\n"); + goto end; + } + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %u status not complete!\n", i + 1); + goto end; + } + + if (!sha_job_ok(vec, job, job->user_data, padding, sizeof(padding))) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + +end: + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + +end2: + if (auths != NULL) + free(auths); + + return ret; +} + static void test_sha_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *sha1_ctx, struct test_suite_context *sha224_ctx, struct test_suite_context *sha256_ctx, @@ -226,6 +338,12 @@ test_sha_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *sha1_ctx, } else { test_suite_update(ctx, 1, 0); } + if (test_sha_hash_burst(mb_mgr, v, num_jobs, sha_type)) { + printf("error #%zu\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } } } diff --git a/test/kat-app/sm4_cbc_test.c b/test/kat-app/sm4_cbc_test.c index 730f8693..550285ca 100644 --- a/test/kat-app/sm4_cbc_test.c +++ b/test/kat-app/sm4_cbc_test.c @@ -36,8 +36,6 @@ #include "utils.h" #include "cipher_test.h" -#define MAX_BURST_JOBS 64 - int sm4_cbc_test(struct IMB_MGR *mb_mgr); @@ -252,7 +250,7 @@ test_sm4_cbc_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx, int sm4_cbc_test(struct IMB_MGR *mb_mgr) { - const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, MAX_BURST_JOBS }; + const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, IMB_MAX_BURST_SIZE }; unsigned i; int errors = 0; struct test_suite_context ctx; diff --git a/test/kat-app/sm4_ecb_test.c b/test/kat-app/sm4_ecb_test.c index d60635aa..8ebafdb1 100644 --- a/test/kat-app/sm4_ecb_test.c +++ b/test/kat-app/sm4_ecb_test.c @@ -36,8 +36,6 @@ #include "utils.h" #include "cipher_test.h" -#define MAX_BURST_JOBS 64 - int sm4_ecb_test(struct IMB_MGR *mb_mgr); @@ -250,7 +248,7 @@ test_sm4_ecb_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx, int sm4_ecb_test(struct IMB_MGR *mb_mgr) { - const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, MAX_BURST_JOBS }; + const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, IMB_MAX_BURST_SIZE }; unsigned i; int errors = 0; struct test_suite_context ctx; diff --git a/test/kat-app/zuc_eea3_test.c b/test/kat-app/zuc_eea3_test.c index 760ffa2c..5c05bba4 100644 --- a/test/kat-app/zuc_eea3_test.c +++ b/test/kat-app/zuc_eea3_test.c @@ -52,8 +52,6 @@ #define MAX_BUFFER_LENGTH_IN_BITS 5670 /* biggest test is EIA test 5 */ #define MAX_BUFFER_LENGTH_IN_BYTES ((MAX_BUFFER_LENGTH_IN_BITS) + 7) / 8 -#define MAX_BURST_JOBS 32 - enum api_type { TEST_DIRECT_API, TEST_SINGLE_JOB_API, TEST_BURST_JOB_API }; int @@ -312,7 +310,7 @@ submit_burst_eea3_jobs(struct IMB_MGR *mb_mgr, uint8_t **const keys, uint8_t **c const unsigned int num_jobs, const unsigned int key_len, const unsigned int *iv_lens) { - IMB_JOB *job, *jobs[MAX_BURST_JOBS] = { NULL }; + IMB_JOB *job, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; unsigned int i; unsigned int jobs_rx = 0; uint32_t completed_jobs = 0; diff --git a/test/kat-app/zuc_eia3_test.c b/test/kat-app/zuc_eia3_test.c index 7edde3bb..4fcd10d1 100644 --- a/test/kat-app/zuc_eia3_test.c +++ b/test/kat-app/zuc_eia3_test.c @@ -52,8 +52,6 @@ #define MAX_BUFFER_LENGTH_IN_BITS 5670 /* biggest test is EIA test 5 */ #define MAX_BUFFER_LENGTH_IN_BYTES ((MAX_BUFFER_LENGTH_IN_BITS) + 7) / 8 -#define MAX_BURST_JOBS 32 - enum api_type { TEST_DIRECT_API, TEST_SINGLE_JOB_API, TEST_BURST_JOB_API }; int @@ -291,7 +289,7 @@ submit_burst_eia3_jobs(struct IMB_MGR *mb_mgr, uint8_t **const keys, uint8_t **c const unsigned int num_jobs, const unsigned int key_sz, const size_t *tag_lens, const size_t *iv_lens) { - IMB_JOB *job, *jobs[MAX_BURST_JOBS] = { NULL }; + IMB_JOB *job, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; unsigned int i; unsigned int jobs_rx = 0; uint32_t completed_jobs = 0; diff --git a/test/mp-app/CMakeLists.txt b/test/mp-app/CMakeLists.txt index 23c51b94..839abc71 100644 --- a/test/mp-app/CMakeLists.txt +++ b/test/mp-app/CMakeLists.txt @@ -31,15 +31,29 @@ endif() ######################################## # set app and library names ######################################## -set(MP_APP imb-mp) +set(MP_APP_PRI imb-mp-primary) +set(MP_APP_SEC imb-mp-secondary) set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../lib) set(TEST_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../include) ######################################## -# set imb-mp source files +# set imb-mp-primary source files ######################################## -set(MP_APP_SRC_FILES_C - ${CMAKE_CURRENT_SOURCE_DIR}/imb-mp.c +set(MP_APP_PRI_SRC_FILES_C + ${CMAKE_CURRENT_SOURCE_DIR}/imb-mp-primary.c + ${CMAKE_CURRENT_SOURCE_DIR}/mp_shared_mem.c + ${CMAKE_CURRENT_SOURCE_DIR}/mp_imb.c + ${CMAKE_CURRENT_SOURCE_DIR}/mp_alloc.c +) + +######################################## +# set imb-mp-secondary source files +######################################## +set(MP_APP_SEC_SRC_FILES_C + ${CMAKE_CURRENT_SOURCE_DIR}/imb-mp-secondary.c + ${CMAKE_CURRENT_SOURCE_DIR}/mp_shared_mem.c + ${CMAKE_CURRENT_SOURCE_DIR}/mp_imb.c + ${CMAKE_CURRENT_SOURCE_DIR}/mp_alloc.c ) ######################################## @@ -56,19 +70,32 @@ else() include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/unix.cmake) endif() - ######################################## # add targets ######################################## include_directories(${LIB_DIR} ${TEST_INCLUDE_DIR}) link_directories(${LIB_DIR}) -add_executable(${MP_APP} ${MP_APP_SRC_FILES_C}) -target_link_libraries(${MP_APP} PRIVATE ${IPSEC_MB_LIB}) -target_compile_definitions(${MP_APP} PRIVATE ${APP_DEFINES}) +add_executable(${MP_APP_PRI} ${MP_APP_PRI_SRC_FILES_C}) +target_link_libraries(${MP_APP_PRI} PRIVATE ${IPSEC_MB_LIB}) +target_compile_definitions(${MP_APP_PRI} PRIVATE ${APP_DEFINES}) + +add_executable(${MP_APP_SEC} ${MP_APP_SEC_SRC_FILES_C}) +target_link_libraries(${MP_APP_SEC} PRIVATE ${IPSEC_MB_LIB}) +target_compile_definitions(${MP_APP_SEC} PRIVATE ${APP_DEFINES}) + ######################################## # add tests ######################################## -add_test(NAME MP COMMAND ${MP_APP}) +# set working directory for tests +if(IMB_BIN_DIR) + set(TEST_APP_BIN_DIR "${IMB_BIN_DIR}") +else() + set(TEST_APP_BIN_DIR "${CMAKE_CURRENT_BINARY_DIR}") +endif() + +add_test(NAME MULTIPROCESS + COMMAND ${MP_APP_PRI} ${TEST_APP_BIN_DIR}/${MP_APP_SEC} + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) diff --git a/test/mp-app/README.md b/test/mp-app/README.md new file mode 100644 index 00000000..5cc8f3b8 --- /dev/null +++ b/test/mp-app/README.md @@ -0,0 +1,61 @@ +# Multi-Process Test + +This test is designed to exercise fail-over scenario. In this scenario one process, called primary, is actively processing crypto requests. +Another process, called secondary, is passive but it can take over crypto operations at any point of time if the primary process fails. + +## Contents +1. Overview +2. Running +3. Multi-Buffer Manager (MB MGR) Notes +4. OS Implementation Notes + +## 1. Overview + +The test uses two applications, one is used to represent the primary process and another one the secondary one. +Operation flow of two processes looks as described in the table below. + +| PRIMARY PROCESS | SECONDARY PROCESS | +| :-------- | -----------: | +| [START] | | +| INITIALIZE INFO SHARED MEMORY | | +| INITIALIZE DATA SHARED MEMORY | | +| INITIALIZE MB MGR | | +| INITIALIZE MEMORY FOR 15 CRYPTO OPERATIONS | | +| PRODUCE EXPECTED CRYPTO RESULTS (out-of-place) | | +| SEND 15 CRYPTO REQUESTS (in-place) AND COLLECT RESPONSES IF ANY | | +| UPDATE INFO SHARED MEMORY WITH CRYPTO REQUEST/RESPONSE DETAILS | | +| START SECONDARY PROCESS --> | --> [START] | +| | OPEN INFO SHARED MEMORY | +| | OPEN DATA SHARED MEMORY | +| | INITIALIZE MB MGR IN SHARED MEMORY | +| | COMPLETE OUTSTANDING CRYPTO OPERATIONS | +| | CHECK NUMBER OF PROCESSED BUFFERS IS EQUAL NUMBER OF REQUESTS | +| CHECK CRYPTO RESPONSE BUFFERS MATCH EXPECTED OUTPUT <-- | <-- [END] | +| FREE ALLOCATED RESOURCES | | +| [END] | | + +## 2. Running + +Primary process requires path name of the secondary process application as an argument. +Example: `> ./imb-mp-primary ${PWD}/imb-mp-secondary` + +## 3. Multi-Buffer Manager (MB MGR) Notes + +See `mp_imb.c` file and `init_imb()` function as reference for setting up multi-buffer manager in the primary process and fail-over initialization in the secondary process. +Note that for fail-over scenario, all crypto operations need to be done on data structures allocated in shared memory. Virtual addresses of the buffers and associated data structures need to be identical in the primary and secondary processes. + +## 4. OS Implementation notes + +### Linux + +No issues encountered. Virtual address given automatically to the primary process shared memory at `mmap()` is good to map in the secondary process. + +### FreeBSD + +Using arbitrary virtual address to allocate common virtual addresses for the primary and secondary processes. Address growth from the bottom to top. +`MAP_PREFAULT_READ` was found to be required on FreeBSD system when calling `mmap()`. Otherwise the secondary process was crashing when accessing open shared memory. + +### Windows + +Shared memory handle cannot be closed by the primary process before executing `system()` otherwise secondary process cannot map named shared memory. +Using maximum application address from `GetSystemInfo()` to allocate common virtual addresses for the primary and secondary processes. Stack based approach applied here and moving from the top address down. diff --git a/test/mp-app/imb-mp-primary.c b/test/mp-app/imb-mp-primary.c new file mode 100644 index 00000000..97851519 --- /dev/null +++ b/test/mp-app/imb-mp-primary.c @@ -0,0 +1,390 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include + +#include + +#include "mp_alloc.h" +#include "mp_shared_mem.h" +#include "mp_imb.h" +#include "mp_info_context.h" + +#if defined(__MINGW32__) + +static int +mp_primary(const char *name2) +{ + (void) name2; + printf("Multi-Process test not executed.\n"); + return 0; +} + +#else + +#if defined(__linux__) || defined(__FreeBSD__) +#include +#include /* close() and unlink() */ +#endif + +#ifdef _WIN32 +#include /* _mktemp() */ +#endif + +/* + * ============================================================================= + * ============================================================================= + * Primary processes + */ + +/* + * Process data + * - primary process allocates and initializes them + * - secondary process only picks them up + * All shared memory allocations will have the same virtual address + * in primary and secondary processes. + */ + +static int +alloc_crypto_op_data(struct info_context *ctx, struct allocator *app_alloc, const int is_pri) +{ + if (!is_pri) + return 0; + + /* + * Primary process does all memory allocations in shared memory and + * stores pointers in data section that secondary process will inherit + */ + size_t i; + + ctx->exp_enc_key = mp_alloc(app_alloc, 11 * 16, 16); + ctx->exp_dec_key = mp_alloc(app_alloc, 11 * 16, 16); + ctx->aes_key = mp_alloc(app_alloc, 16, 0); + ctx->iv = mp_alloc(app_alloc, 16, 0); + + for (i = 0; i < IMB_DIM(ctx->buffer_table_in_out); i++) { + ctx->buffer_table_in_out[i] = mp_alloc(app_alloc, buffer_size, 4); + if (ctx->buffer_table_in_out[i] == NULL) + break; + memset(ctx->buffer_table_in_out[i], (int) ~i, buffer_size); + + ctx->buffer_table_ref[i] = mp_alloc(app_alloc, buffer_size, 4); + if (ctx->buffer_table_ref[i] == NULL) + break; + memset(ctx->buffer_table_ref[i], 0, buffer_size); + } + + if (ctx->exp_enc_key == NULL || ctx->exp_dec_key == NULL || ctx->aes_key == NULL || + ctx->iv == NULL || i < IMB_DIM(ctx->buffer_table_in_out)) + return -1; + + return 0; +} + +static int +prepare_reference_output(struct info_context *ctx, const int is_pri) +{ + if (!is_pri) + return 0; + + /* Create key schedule and set IV */ + memset(ctx->aes_key, 0xaa, 16); + IMB_AES_KEYEXP_128(ctx->mb_mgr, ctx->aes_key, ctx->exp_enc_key, ctx->exp_dec_key); + + memset(ctx->iv, 0x55, 16); + + /* + * Use allocated manager to get reference answers + */ + ctx->jobs_sent = 0; + ctx->jobs_received = 0; + + if (submit_aes_cbc_enc_jobs(ctx->mb_mgr, ctx->buffer_table_in_out, ctx->buffer_table_ref, + IMB_DIM(ctx->buffer_table_in_out), &ctx->jobs_received, + &ctx->jobs_sent, ctx->exp_enc_key, ctx->iv, buffer_size) != 0) + return -1; + + if (flush_aes_cbc_enc_jobs(ctx->mb_mgr, &ctx->jobs_received) != 0) + return -1; + + if (ctx->jobs_sent != IMB_DIM(ctx->buffer_table_in_out)) + return -1; + + ctx->jobs_sent = 0; + ctx->jobs_received = 0; + return 0; +} + +static char * +randomize_shm_name(const char *name) +{ + if (name == NULL) + return NULL; + + char temp[8]; + + memset(temp, 0, sizeof(temp)); + strncpy(temp, "XXXXXX", sizeof(temp) - 1); + +#if defined(__linux__) || defined(__FreeBSD__) + int fd = mkstemp(temp); + + if (fd == -1) + return NULL; + + close(fd); + unlink(temp); +#endif + +#ifdef _WIN32 + (void) _mktemp(temp); +#endif + + const size_t name_len = strlen(name); + const size_t temp_len = strlen(temp); + const size_t new_len = name_len + temp_len + 1; + char *new_name = malloc(new_len); + + if (new_name == NULL) + return NULL; + + const int ret_len = snprintf(new_name, new_len, "%s%s", name, temp); + + if (ret_len >= (int) new_len || ret_len < 0) { + free(new_name); + return NULL; + } + + return new_name; +} + +static int +mp_primary(const char *name2) +{ + const int is_pri = 1; + + char *shm_info_uname = randomize_shm_name(SHM_INFO_NAME); + + if (shm_info_uname == NULL) + return -1; + + char *shm_data_uname = randomize_shm_name(SHM_DATA_NAME); + + if (shm_data_uname == NULL) { + free(shm_info_uname); + return -1; + } + + fprintf(stdout, "PRIMARY: init start %p, %s, %s\n", (void *) imb_get_errno, shm_info_uname, + shm_data_uname); + + struct shared_memory app_shm, info_shm; + struct info_context *ctx = NULL; + struct allocator app_alloc; + + if (shm_create(&info_shm, is_pri, shm_info_uname, SHM_INFO_SIZE, NULL) != 0) { + free(shm_info_uname); + free(shm_data_uname); + return -1; + } + + /* cast info shared memory onto info context structure */ + ctx = (struct info_context *) info_shm.ptr; + memset(ctx, 0, sizeof(*ctx)); + + if (shm_create(&app_shm, is_pri, shm_data_uname, SHM_DATA_SIZE, NULL) != 0) { + (void) shm_destroy(&info_shm, is_pri); + free(shm_info_uname); + free(shm_data_uname); + return -1; + } + + /* secondary process needs to mmap app/data shared memory at this address */ + ctx->app_mmap = app_shm.ptr; + + /* init allocator on app/data shared memory */ + mp_init(&app_alloc, app_shm.ptr, app_shm.size); + + /* init IMB */ + ctx->mb_mgr = init_imb(NULL, &app_alloc, is_pri); + if (ctx->mb_mgr == NULL) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + free(shm_info_uname); + free(shm_data_uname); + return -1; + } + + fprintf(stdout, "PRIMARY: init complete\n"); + + /* allocate data for crypto operations */ + if (alloc_crypto_op_data(ctx, &app_alloc, is_pri) != 0) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + free(shm_info_uname); + free(shm_data_uname); + return -1; + } + + /* generate reference output data */ + if (prepare_reference_output(ctx, is_pri) != 0) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + free(shm_info_uname); + free(shm_data_uname); + return -1; + } + + /* send jobs in primary process */ + ctx->jobs_sent = 0; + ctx->jobs_received = 0; + if (submit_aes_cbc_enc_jobs(ctx->mb_mgr, ctx->buffer_table_in_out, ctx->buffer_table_in_out, + IMB_DIM(ctx->buffer_table_in_out), &ctx->jobs_received, + &ctx->jobs_sent, ctx->exp_enc_key, ctx->iv, buffer_size) != 0) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + free(shm_info_uname); + free(shm_data_uname); + return -1; + } + + fprintf(stdout, "PRIMARY: sent %u AES-128-CBC encrypt jobs\n", ctx->jobs_sent); + fprintf(stdout, "PRIMARY: received %u AES-128-CBC encrypt jobs\n", ctx->jobs_received); + + if (ctx->jobs_sent != IMB_DIM(ctx->buffer_table_in_out)) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + free(shm_info_uname); + free(shm_data_uname); + return -1; + } + + /* + * - spawn secondary process now + * - let the secondary perform the flush operation + * - wait for the secondary process to complete and check the results + */ + fprintf(stdout, "PRIMARY: starting SECONDARY process now\n"); + + const size_t cmd_length = + strlen(name2) + 1 + strlen(shm_info_uname) + 1 + strlen(shm_data_uname) + 1; + char *cmd = malloc(cmd_length); + + if (cmd == NULL) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + free(shm_info_uname); + free(shm_data_uname); + return -1; + } + + memset(cmd, 0, cmd_length); + + const int cmd_length_ret = + snprintf(cmd, cmd_length, "%s %s %s", name2, shm_info_uname, shm_data_uname); + + if (cmd_length_ret >= (int) cmd_length || cmd_length_ret < 0) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + free(shm_info_uname); + free(shm_data_uname); + free(cmd); + return -1; + } + + const int status = system(cmd); + + free(cmd); + +#ifdef _WIN32 + const int err = (status != EXIT_SUCCESS); +#endif + +#if defined(__linux__) || defined(__FreeBSD__) + const int err = (!WIFEXITED(status)) || (WEXITSTATUS(status) != EXIT_SUCCESS); +#endif + + if (err != 0) { + fprintf(stderr, "PRIMARY: SECONDARY process failed\n"); + fprintf(stdout, "MULTI-PROCESS TEST: FAILED\n"); + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + free(shm_info_uname); + free(shm_data_uname); + return -1; + } + + fprintf(stdout, "PRIMARY: SECONDARY has finished\n"); + + /* + * Child process exited normally - let's check the answers + */ + unsigned mismatch = 0; + + for (size_t i = 0; i < IMB_DIM(ctx->buffer_table_in_out); i++) + if (memcmp(ctx->buffer_table_in_out[i], ctx->buffer_table_ref[i], buffer_size) != 0) + mismatch++; + + fprintf(stdout, "MULTI-PROCESS TEST: %s\n", mismatch ? "FAILED " : "PASSED"); + + fprintf(stdout, "PRIMARY: finished\n"); + + /* clean up and exit */ + if (shm_destroy(&info_shm, is_pri) != 0) { + (void) shm_destroy(&app_shm, is_pri); + free(shm_info_uname); + free(shm_data_uname); + return -1; + } + if (shm_destroy(&app_shm, is_pri) != 0) { + free(shm_info_uname); + free(shm_data_uname); + return -1; + } + + free(shm_info_uname); + free(shm_data_uname); + return 0; +} +#endif /* _WIN32 || __linux__ || __FreeBSD__ */ + +int +main(int argc, char **argv) +{ + int ret = -1; + + if (argc > 1) + ret = mp_primary(argv[1]); + else + fprintf(stderr, + "ERROR: argument required! Command syntax: %s \n", + argv[0]); + + return (ret == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/test/mp-app/imb-mp-secondary.c b/test/mp-app/imb-mp-secondary.c new file mode 100644 index 00000000..90d86ffa --- /dev/null +++ b/test/mp-app/imb-mp-secondary.c @@ -0,0 +1,145 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include + +#include + +#include "mp_shared_mem.h" +#include "mp_imb.h" +#include "mp_info_context.h" + +#if defined(__MINGW32__) + +static int +mp_secondary(const char *shm_info_uname, const char *shm_data_uname) +{ + (void) shm_info_uname; + (void) shm_data_uname; + printf("Multi-Process test not executed.\n"); + return 0; +} + +#else + +/* + * ============================================================================= + * ============================================================================= + * Secondary processes + */ + +static int +mp_secondary(const char *shm_info_uname, const char *shm_data_uname) +{ + const int is_pri = 0; + struct shared_memory app_shm, info_shm; + struct info_context *ctx = NULL; + + fprintf(stdout, "SECONDARY: init start %p, %s, %s\n", (void *) imb_get_errno, + shm_info_uname, shm_data_uname); + + if (shm_create(&info_shm, is_pri, shm_info_uname, SHM_INFO_SIZE, NULL) != 0) + return -1; + + /* cast info shared memory onto info context structure */ + ctx = (struct info_context *) info_shm.ptr; + + /* check if any jobs were sent */ + if (ctx->jobs_sent == 0) { + (void) shm_destroy(&info_shm, is_pri); + return -1; + } + + if (shm_create(&app_shm, is_pri, shm_data_uname, SHM_DATA_SIZE, ctx->app_mmap) != 0) { + (void) shm_destroy(&info_shm, is_pri); + return -1; + } + + /* init IMB */ + if (init_imb(ctx->mb_mgr, NULL, is_pri) == NULL) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + + fprintf(stdout, "SECONDARY: init complete\n"); + + /* flush jobs sent by primary process */ + unsigned jobs_received_now = 0; + + if (flush_aes_cbc_enc_jobs(ctx->mb_mgr, &jobs_received_now) != 0) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + + ctx->jobs_received += jobs_received_now; + + fprintf(stdout, "SECONDARY: received %u (total %u) AES-128-CBC encrypt jobs\n", + jobs_received_now, ctx->jobs_received); + + if (ctx->jobs_sent != ctx->jobs_received) { + fprintf(stderr, "SECONDARY: expected %u jobs, received %u\n", ctx->jobs_sent, + ctx->jobs_received); + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + + fprintf(stdout, "SECONDARY: finished\n"); + +#ifdef _WIN32 + _flushall(); +#endif + + /* clean up and exit */ + if (shm_destroy(&info_shm, is_pri) != 0) { + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + if (shm_destroy(&app_shm, is_pri) != 0) + return -1; + + return 0; +} +#endif /* _WIN32 || __linux__ || __FreeBSD__ */ + +int +main(int argc, char **argv) +{ + int ret = -1; + + (void) argc; + (void) argv; + + if (argc == 3) + ret = mp_secondary(argv[1], argv[2]); + + return (ret == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/test/mp-app/imb-mp.c b/test/mp-app/imb-mp.c deleted file mode 100644 index b831c653..00000000 --- a/test/mp-app/imb-mp.c +++ /dev/null @@ -1,490 +0,0 @@ -/***************************************************************************** - Copyright (c) 2024, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -#include -#include - -#include - -#ifdef LINUX -#include -#include -#include -#include -#include - -int is_secondary = 0; - -/* - * ============================================================================= - * ============================================================================= - * Shared memory create and destroy - */ - -#define SHM_SIZE (2ULL * 1024ULL * 1024ULL) -#define SHM_NAME "mp-app-shared-memory" - -void *mmap_ptr = NULL; -void *alloc_ptr = NULL; -size_t alloc_offset = 0; - -int shm_fd; - -static int -shm_destroy(void) -{ - int ret = 0; - - if (alloc_ptr != NULL) - if (munmap(alloc_ptr, SHM_SIZE) != 0) - ret = -1; - alloc_ptr = NULL; - - if (shm_fd == -1) - if (close(shm_fd) != 0) - ret = -1; - shm_fd = -1; - - if (!is_secondary) { - if (shm_unlink(SHM_NAME) != 0) - ret = -1; - } - - return ret; -} - -static int -shm_create(void) -{ - /* create the shared memory object */ - if (is_secondary) - shm_fd = shm_open(SHM_NAME, O_RDWR, 0666); - else - shm_fd = shm_open(SHM_NAME, O_CREAT | O_RDWR, 0666); - if (shm_fd == -1) - return -1; - - /* configure the size of the shared memory object */ - if (!is_secondary) { - if (ftruncate(shm_fd, SHM_SIZE) != 0) { - (void) shm_destroy(); - return -1; - } - } - - /* - * memory map the shared memory object - * - secondary process maps shared memory into the same region as the primary process - */ - if (is_secondary) - alloc_ptr = mmap(mmap_ptr, SHM_SIZE, PROT_WRITE, MAP_SHARED, shm_fd, 0); - else - alloc_ptr = mmap(0, SHM_SIZE, PROT_WRITE, MAP_SHARED, shm_fd, 0); - - if (alloc_ptr == MAP_FAILED) { - (void) shm_destroy(); - return -1; - } - - mmap_ptr = alloc_ptr; - return 0; -} - -/* - * ============================================================================= - * ============================================================================= - * Shared memory allocator - */ - -/** - * @brief Simple memory allocator from the shared memory pool - * - * @param length data size to allocate in bytes - * @param alignment 0 or any power of 2 to align memory allocation to - * - * @return Pointer to allocated memory - * @retval NULL allocation error - */ -static void * -mp_alloc(const size_t length, const size_t alignment) -{ - if (alloc_ptr == NULL) - return NULL; - - if ((alloc_offset + length) > SHM_SIZE) - return NULL; - - if (alignment > 1) { - const size_t align_mask = alignment - 1; - - alloc_offset = (alloc_offset + align_mask) & (~align_mask); - } - - void *ptr = ((char *) alloc_ptr + alloc_offset); - - alloc_offset += length; - - return ptr; -} - -/* - * ============================================================================= - * ============================================================================= - * Primary and secondary processes - */ - -/* - * Process data - * - primary process allocates and initializes them - * - secondary process only picks them up - * All shared memory allocations will have the same virtual address - * in primary and secondary processes. - */ -const size_t buffer_size = 17 * 16; - -static unsigned jobs_sent = 0; -static unsigned jobs_received = 0; - -static void *p_mgr; -static void *exp_enc_key; -static void *exp_dec_key; -static void *aes_key; -static void *iv; -static void *buffer_table_in[15]; -static void *buffer_table_out[15]; -static IMB_MGR *mb_mgr; - -static int -submit_aes_cbc_enc_jobs(IMB_MGR *p_mgr, void **in, void **out, const size_t n, const int do_flush) -{ - for (size_t i = 0; i < n; i++) { - IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); - - memset(job, 0, sizeof(*job)); - - job->cipher_direction = IMB_DIR_ENCRYPT; - job->chain_order = IMB_ORDER_CIPHER_HASH; - job->src = in[i]; - job->dst = out[i]; - job->cipher_mode = IMB_CIPHER_CBC; - job->enc_keys = exp_enc_key; - job->dec_keys = NULL; - job->key_len_in_bytes = IMB_KEY_128_BYTES; - - job->iv = iv; - job->iv_len_in_bytes = 16; - job->cipher_start_src_offset_in_bytes = 0; - job->msg_len_to_cipher_in_bytes = buffer_size; - job->hash_alg = IMB_AUTH_NULL; - - IMB_JOB *job_ret = IMB_SUBMIT_JOB(p_mgr); - - if (imb_get_errno(p_mgr) == 0) - jobs_sent++; - else - return -1; - - if (job_ret != NULL) - jobs_received++; - } - - if (!do_flush) - return 0; - - while (IMB_FLUSH_JOB(p_mgr) != NULL) { - const int err = imb_get_errno(p_mgr); - - if (err != 0) { - fprintf(stderr, "!Flush error: %s!\n", imb_get_strerror(err)); - return -1; - } else - jobs_received++; - } - - return 0; -} - -static int -init_imb_and_buffers(void) -{ - if (!is_secondary) { - /* - * Primary process does all memory allocations in shared memory and - * stores pointers in data section that secondary process will inherit - */ - size_t i; - - p_mgr = mp_alloc(imb_get_mb_mgr_size(), 64); - exp_enc_key = mp_alloc(11 * 16, 16); - exp_dec_key = mp_alloc(11 * 16, 16); - aes_key = mp_alloc(16, 0); - iv = mp_alloc(16, 0); - - for (i = 0; i < IMB_DIM(buffer_table_in); i++) { - buffer_table_in[i] = mp_alloc(buffer_size, 4); - if (buffer_table_in[i] == NULL) - break; - memset(buffer_table_in[i], ~i, buffer_size); - - buffer_table_out[i] = mp_alloc(buffer_size, 4); - if (buffer_table_out[i] == NULL) - break; - memset(buffer_table_out[i], 0, buffer_size); - } - - if (p_mgr == NULL || exp_enc_key == NULL || exp_dec_key == NULL || - aes_key == NULL || iv == NULL || i < IMB_DIM(buffer_table_in)) - return -1; - - /* - * Set up multi-buffer manager in the shared memory - * - imb_set_pointers_mb_mgr() call with reset parameter is required - * Normally, alloc_mb_mgr() clears memory and sets selected feature flags. - * - it is followed with init_mb_mgr_auto() call - */ - mb_mgr = imb_set_pointers_mb_mgr(p_mgr, 0, 1); - if (mb_mgr == NULL) - return -1; - - IMB_ARCH arch; - - init_mb_mgr_auto(mb_mgr, &arch); - - if (imb_get_errno(mb_mgr) != 0) - return -1; - - } else { - /* - * Secondary process picks allocations done by primary process and - * resets functions pointers in the manager - */ - mb_mgr = imb_set_pointers_mb_mgr(p_mgr, 0, 0); - } - - if (mb_mgr == NULL) - return -1; - - if (!is_secondary) { - /* Create key schedule and set IV */ - memset(aes_key, 0xaa, 16); - IMB_AES_KEYEXP_128(mb_mgr, aes_key, exp_enc_key, exp_dec_key); - - memset(iv, 0x55, 16); - - /* - * Use temporary manager to get reference answers - */ - IMB_MGR *tmp_mgr = alloc_mb_mgr(0); - - if (tmp_mgr == NULL) - return -1; - - IMB_ARCH arch; - - init_mb_mgr_auto(tmp_mgr, &arch); - - if (imb_get_errno(tmp_mgr) != 0) { - free_mb_mgr(tmp_mgr); - return -1; - } - - jobs_sent = 0; - jobs_received = 0; - - if (submit_aes_cbc_enc_jobs(tmp_mgr, buffer_table_in, buffer_table_out, - IMB_DIM(buffer_table_in), 1) != 0) { - free_mb_mgr(tmp_mgr); - return -1; - } - - if (jobs_sent != IMB_DIM(buffer_table_in)) { - free_mb_mgr(tmp_mgr); - return -1; - } - - jobs_sent = 0; - jobs_received = 0; - - free_mb_mgr(tmp_mgr); - } - - return 0; -} - -static int -mp_secondary(void) -{ - is_secondary = 1; - - if (jobs_sent == 0) - return -1; - - if (shm_create() != 0) - return -1; - - if (init_imb_and_buffers() != 0) { - (void) shm_destroy(); - return -1; - } - - unsigned jobs_received_now = 0; - - while (IMB_FLUSH_JOB(mb_mgr) != NULL) { - const int err = imb_get_errno(mb_mgr); - - if (err != 0) { - fprintf(stderr, "SECONDARY: flush error: %s!\n", imb_get_strerror(err)); - (void) shm_destroy(); - return -1; - } else { - jobs_received_now++; - jobs_received++; - } - } - - fprintf(stdout, "SECONDARY: received %u (total %u) AES-128-CBC encrypt jobs\n", - jobs_received_now, (unsigned) jobs_received); - - if (jobs_sent != jobs_received) { - fprintf(stderr, "SECONDARY: expected %u jobs, received %u\n", (unsigned) jobs_sent, - (unsigned) jobs_received); - (void) shm_destroy(); - return -1; - } - - if (shm_destroy() != 0) - return -1; - return 0; -} - -static int -mp_primary(void) -{ - is_secondary = 0; - - if (shm_create() != 0) - return -1; - - if (init_imb_and_buffers() != 0) { - (void) shm_destroy(); - return -1; - } - - jobs_sent = 0; - jobs_received = 0; - if (submit_aes_cbc_enc_jobs(mb_mgr, buffer_table_in, buffer_table_in, - IMB_DIM(buffer_table_in), 0) != 0) { - (void) shm_destroy(); - return -1; - } - - fprintf(stdout, "PRIMARY: sent %u AES-128-CBC encrypt jobs\n", (unsigned) jobs_sent); - fprintf(stdout, "PRIMARY: received %u AES-128-CBC encrypt jobs\n", - (unsigned) jobs_received); - - if (jobs_sent != IMB_DIM(buffer_table_in)) { - (void) shm_destroy(); - return -1; - } - - /* - * - fork now - * - let the child perform the flush operation - * - wait for child to complete - */ - pid_t pid = fork(); - - if (pid < 0) { - fprintf(stderr, "PRIMARY: Fork failed\n"); - return -1; - } - - if (pid == 0) { - /* child process - secondary process */ - const int status = mp_secondary(); - - (void) shm_destroy(); - fflush(stderr); - fflush(stdout); - exit((status != 0) ? EXIT_FAILURE : EXIT_SUCCESS); - } else { - /* parent waits for the child to finish */ - int wstatus = 0; - - wait(&wstatus); - - const int err = (!WIFEXITED(wstatus)) || (WEXITSTATUS(wstatus) != EXIT_SUCCESS); - - if (err != 0) { - fprintf(stderr, "PRIMARY: secondary process failed\n"); - fprintf(stdout, "MULTI-PROCESS TEST: FAILED\n"); - (void) shm_destroy(); - return -1; - } else { - /* - * Child process exited normally - let's check the answers - */ - unsigned mismatch = 0; - - for (size_t i = 0; i < IMB_DIM(buffer_table_in); i++) - if (memcmp(buffer_table_in[i], buffer_table_out[i], buffer_size) != - 0) - mismatch++; - - fprintf(stdout, "MULTI-PROCESS TEST: %s\n", - mismatch ? "FAILED " : "PASSED"); - } - } - - if (shm_destroy() != 0) - return -1; - - return 0; -} - -#else /* LINUX */ - -static int -mp_primary(void) -{ - fprintf(stdout, "MULTI-PROCESS TEST: NOT RUN\n"); - return 0; -} - -#endif /* !LINUX */ - -int -main(int argc, char **argv) -{ - (void) argc; - (void) argv; - - if (mp_primary() != 0) - return EXIT_FAILURE; - - return EXIT_SUCCESS; -} diff --git a/test/mp-app/mp_alloc.c b/test/mp-app/mp_alloc.c new file mode 100644 index 00000000..30394d94 --- /dev/null +++ b/test/mp-app/mp_alloc.c @@ -0,0 +1,58 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "mp_alloc.h" + +void +mp_init(struct allocator *a, void *ptr, const size_t size) +{ + a->ptr = ptr; + a->size = size; + a->offset = 0; +} + +void * +mp_alloc(struct allocator *a, const size_t length, const size_t alignment) +{ + if (a->ptr == NULL) + return NULL; + + if ((a->offset + length) > a->size) + return NULL; + + if (alignment > 1) { + const size_t align_mask = alignment - 1; + + a->offset = (a->offset + align_mask) & (~align_mask); + } + + void *ptr = ((char *) a->ptr + a->offset); + + a->offset += length; + + return ptr; +} diff --git a/test/mp-app/mp_alloc.h b/test/mp-app/mp_alloc.h new file mode 100644 index 00000000..a1fc7c06 --- /dev/null +++ b/test/mp-app/mp_alloc.h @@ -0,0 +1,68 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#ifndef MP_ALLOC_H +#define MP_ALLOC_H + +#include + +/* + * ============================================================================= + * ============================================================================= + * Basic shared memory allocator + */ + +struct allocator { + void *ptr; + size_t offset; + size_t size; +}; + +/** + * @brief Simple memory allocator initialization + * + * @param a pointer to allocator instance structure + * @param ptr pointer to memory chunk base pointer + * @param size memory chunk size in bytes + */ +void +mp_init(struct allocator *a, void *ptr, const size_t size); + +/** + * @brief Simple memory allocator from the shared memory pool + * + * @param a pointer to allocator instance structure + * @param length data size to allocate in bytes + * @param alignment 0 or any power of 2 to align memory allocation to + * + * @return Pointer to allocated memory + * @retval NULL allocation error + */ +void * +mp_alloc(struct allocator *a, const size_t length, const size_t alignment); + +#endif /* MP_ALLOC_H */ diff --git a/test/mp-app/mp_imb.c b/test/mp-app/mp_imb.c new file mode 100644 index 00000000..d05913c7 --- /dev/null +++ b/test/mp-app/mp_imb.c @@ -0,0 +1,145 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include +#include "mp_alloc.h" +#include "mp_imb.h" + +/* + * ============================================================================= + * ============================================================================= + * IMB submit, flush and init functions + */ + +int +flush_aes_cbc_enc_jobs(IMB_MGR *p_mgr, unsigned *jobs_received) +{ + if (p_mgr == NULL || jobs_received == NULL) + return -2; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) { + const int err = imb_get_errno(p_mgr); + + if (err != 0) { + fprintf(stderr, "!Flush error: %s!\n", imb_get_strerror(err)); + return -1; + } else { + *jobs_received = *jobs_received + 1; + } + } + + return 0; +} + +int +submit_aes_cbc_enc_jobs(IMB_MGR *p_mgr, void **in, void **out, const size_t n, + unsigned *jobs_received, unsigned *jobs_sent, void *exp_enc_key, void *iv, + const size_t msg_size) +{ + if (p_mgr == NULL || in == NULL || out == NULL || n == 0 || jobs_received == NULL || + jobs_sent == NULL || exp_enc_key == NULL || iv == NULL) + return -2; + + for (size_t i = 0; i < n; i++) { + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + memset(job, 0, sizeof(*job)); + + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->src = in[i]; + job->dst = out[i]; + job->cipher_mode = IMB_CIPHER_CBC; + job->enc_keys = exp_enc_key; + job->dec_keys = NULL; + job->key_len_in_bytes = IMB_KEY_128_BYTES; + + job->iv = iv; + job->iv_len_in_bytes = 16; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = msg_size; + job->hash_alg = IMB_AUTH_NULL; + + IMB_JOB *job_ret = IMB_SUBMIT_JOB(p_mgr); + + if (imb_get_errno(p_mgr) == 0) + *jobs_sent = *jobs_sent + 1; + else + return -1; + + if (job_ret != NULL) + *jobs_received = *jobs_received + 1; + } + + return 0; +} + +IMB_MGR * +init_imb(IMB_MGR *in_mb_mgr, struct allocator *app_alloc, const int is_pri) +{ + if (is_pri) { + /* + * Primary process does all memory allocations in shared memory and + * stores pointers in data section that secondary process will inherit + */ + void *p_mgr = mp_alloc(app_alloc, imb_get_mb_mgr_size(), 64); + + if (p_mgr == NULL) + return NULL; + + /* + * Set up multi-buffer manager in the shared memory + * - imb_set_pointers_mb_mgr() call with reset parameter is required + * Normally, alloc_mb_mgr() clears memory and sets selected feature flags. + * - it is followed with init_mb_mgr_auto() call + */ + + IMB_MGR *mb_mgr = imb_set_pointers_mb_mgr(p_mgr, 0, 1); + + if (mb_mgr == NULL) + return NULL; + + init_mb_mgr_auto(mb_mgr, NULL); + + if (imb_get_errno(mb_mgr) != 0) + return NULL; + + return mb_mgr; + + } else { + /* + * Secondary process picks allocations done by primary process and + * resets functions pointers in the manager + */ + void *p_mgr = (void *) in_mb_mgr; + IMB_MGR *mb_mgr = imb_set_pointers_mb_mgr(p_mgr, 0, 0); + + return mb_mgr; + } +} diff --git a/test/mp-app/mp_imb.h b/test/mp-app/mp_imb.h new file mode 100644 index 00000000..602adbab --- /dev/null +++ b/test/mp-app/mp_imb.h @@ -0,0 +1,51 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#ifndef MP_IMB_H +#define MP_IMB_H + +#include +#include "mp_alloc.h" + +/* + * ============================================================================= + * ============================================================================= + * IMB submit, flush and init API + */ + +int +flush_aes_cbc_enc_jobs(IMB_MGR *p_mgr, unsigned *jobs_received); + +int +submit_aes_cbc_enc_jobs(IMB_MGR *p_mgr, void **in, void **out, const size_t n, + unsigned *jobs_received, unsigned *jobs_sent, void *exp_enc_key, void *iv, + const size_t msg_size); + +IMB_MGR * +init_imb(IMB_MGR *in_mb_mgr, struct allocator *app_alloc, const int is_pri); + +#endif /* MP_IMB_H */ diff --git a/test/mp-app/mp_info_context.h b/test/mp-app/mp_info_context.h new file mode 100644 index 00000000..dd87a813 --- /dev/null +++ b/test/mp-app/mp_info_context.h @@ -0,0 +1,50 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#ifndef INFO_CONTEXT_H +#define INFO_CONTEXT_H + +#include + +const size_t buffer_size = 17 * 16; + +struct info_context { + unsigned jobs_sent; + unsigned jobs_received; + + void *app_mmap; + + IMB_MGR *mb_mgr; + void *exp_enc_key; + void *exp_dec_key; + void *aes_key; + void *iv; + void *buffer_table_in_out[15]; + void *buffer_table_ref[15]; +}; + +#endif /* INFO_CONTEXT_H */ diff --git a/test/mp-app/mp_shared_mem.c b/test/mp-app/mp_shared_mem.c new file mode 100644 index 00000000..816cafdf --- /dev/null +++ b/test/mp-app/mp_shared_mem.c @@ -0,0 +1,321 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "mp_shared_mem.h" + +#if defined(__MINGW32__) + +int +shm_destroy(struct shared_memory *sm, const int is_pri) +{ + (void) is_pri; + sm->name = NULL; + sm->size = 0; + sm->ptr = NULL; + return 0; +} + +int +shm_create(struct shared_memory *sm, const int is_pri, const char *name, const size_t size, + void *mmap_ptr) +{ + (void) is_pri; + (void) mmap_ptr; + sm->name = name; + sm->size = size; + sm->ptr = NULL; + return 0; +} + +#else + +#if defined(__linux__) || defined(__FreeBSD__) +#include +#include +#include +#include +#endif + +#ifdef _WIN32 +/* + * Disable C5105 to workaround warning coming from winbase.h file + * "Windows Kits\10\include\10.0.19041.0\um\winbase.h(9531): warning + * C5105: macro expansion producing 'defined' has undefined behavior" + */ +#pragma warning(disable : 5105) + +#include +#include +#include +#endif + +/* + * ============================================================================= + * ============================================================================= + * Linux & FreeBSD: Shared memory create and destroy + */ + +#if defined(__FreeBSD__) || defined(__linux__) +int +shm_destroy(struct shared_memory *sm, const int is_pri) +{ + int ret = 0; + + if (!is_pri) + if (munmap(sm->ptr, sm->size) != 0) { + perror("shm_destroy()"); + ret = -1; + } + sm->ptr = NULL; + + if (is_pri) + if (shm_unlink(sm->name) != 0) { + perror("shm_destroy()"); + ret = -1; + } + + sm->name = NULL; + sm->size = 0; + return ret; +} + +int +shm_create(struct shared_memory *sm, const int is_pri, const char *name, const size_t size, + void *mmap_ptr) +{ + int fd = -1; + + sm->name = name; + sm->size = size; + sm->ptr = MAP_FAILED; + + /* create the shared memory object */ + if (is_pri) { + fd = shm_open(sm->name, O_RDWR, 0666); + if (fd != -1) { + printf("shm_open(): %s already exists!\n", sm->name); + close(fd); + return -1; + } + fd = shm_open(sm->name, O_CREAT | O_RDWR, 0666); + } else { + fd = shm_open(sm->name, O_RDWR, 0666); + } + + if (fd == -1) { + perror("shm_create()"); + return -1; + } + + /* configure the size of the shared memory object */ + if (is_pri) { + if (ftruncate(fd, sm->size) != 0) { + perror("shm_create()"); + (void) shm_destroy(sm, is_pri); + close(fd); + return -1; + } + } + + /* + * memory map the shared memory object + * - secondary process maps shared memory into the same region as the primary process + */ + if (is_pri) { +#ifdef __FreeBSD__ + static char *base = (char *) 0x900000000; /* arbitrary VA to start mapping from */ + const size_t page_sz = (size_t) (getpagesize() - 1); + + sm->ptr = mmap((void *) base, sm->size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PREFAULT_READ | MAP_SHARED, fd, 0); + base += ((sm->size + page_sz) & (~page_sz)); +#else + sm->ptr = mmap(0, sm->size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); +#endif + } else { + if (mmap_ptr == NULL) { +#ifdef __FreeBSD__ + const int flags = MAP_PREFAULT_READ | MAP_SHARED; +#else + const int flags = MAP_SHARED; +#endif + + sm->ptr = mmap(NULL, sm->size, PROT_READ | PROT_WRITE, flags, fd, 0); + } else { +#ifdef __FreeBSD__ + const int flags = MAP_PREFAULT_READ | MAP_SHARED | MAP_FIXED; +#else + const int flags = MAP_SHARED | MAP_FIXED; +#endif + + sm->ptr = mmap(mmap_ptr, sm->size, PROT_READ | PROT_WRITE, flags, fd, 0); + if (mmap_ptr != sm->ptr) { + printf("mmap() %p != mmap_ptr %p\n", sm->ptr, mmap_ptr); + (void) shm_destroy(sm, is_pri); + return -1; + } + } + } + + close(fd); + + if (sm->ptr == MAP_FAILED) { + perror("shm_create()"); + fprintf(stderr, "!mmap() of %s shared memory error\n", sm->name); + (void) shm_destroy(sm, is_pri); + return -1; + } + + return 0; +} +#endif /* __linux__ || __FreeBSD__ */ + +/* + * ============================================================================= + * ============================================================================= + * Windows: Shared memory create and destroy + */ + +#ifdef _WIN32 +static void +printLastError(const DWORD error) +{ + LPSTR message = NULL; + + FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR) &message, 0, + NULL); + + if (message) { + fprintf(stderr, "ERROR: %s\n", message); + LocalFree(message); + } +} + +int +shm_destroy(struct shared_memory *sm, const int is_pri) +{ + int ret = 0; + + if (sm->ptr != NULL) + UnmapViewOfFile(sm->ptr); + + if (sm->fd != INVALID_HANDLE_VALUE) + CloseHandle(sm->fd); + + sm->ptr = NULL; + sm->name = NULL; + sm->size = 0; + sm->fd = INVALID_HANDLE_VALUE; + return ret; +} + +int +shm_create(struct shared_memory *sm, const int is_pri, const TCHAR *name, const size_t size, + void *mmap_ptr) +{ + static char *base = NULL; + static SYSTEM_INFO si; + + if (base == NULL) { + GetSystemInfo(&si); + + const size_t allocMask = si.dwAllocationGranularity - 1; + const uintptr_t new_base = (uintptr_t) si.lpMaximumApplicationAddress; + + /* align base mapping address to allocation granularity */ + base = (char *) (new_base & (~allocMask)); + } + + HANDLE fd = INVALID_HANDLE_VALUE; + + sm->name = name; + sm->size = size; + sm->ptr = NULL; + sm->fd = INVALID_HANDLE_VALUE; + + if (is_pri) { + fd = CreateFileMappingA((HANDLE) INVALID_HANDLE_VALUE, /* Use the page file */ + NULL, PAGE_READWRITE, 0, (DWORD) sm->size, sm->name); + } else { + fd = OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, sm->name); + } + + if (fd == INVALID_HANDLE_VALUE) { + fprintf(stderr, "ERROR: %s failed to create shared memory object (%d)\n", sm->name, + GetLastError()); + printLastError(GetLastError()); + return -1; + } + + /* Map the shared memory object into the address space of the current process */ + if (is_pri) { + const size_t allocMask = si.dwAllocationGranularity - 1; + const size_t allocAlignedSize = (size + allocMask) & (~allocMask); + + /* adjust mapping address to allocation size (aligned to allocation granularity) */ + base = base - allocAlignedSize; + + sm->ptr = (void *) MapViewOfFileEx(fd, FILE_MAP_ALL_ACCESS, /* Read/write access */ + 0, 0, /* offset = 0 */ + 0 /* map all object */, (LPVOID) base); + + if (sm->ptr != (void *) base) { + fprintf(stderr, "!mmap(%p) = %p %s shared memory error\n", base, sm->ptr, + sm->name); + printLastError(GetLastError()); + (void) shm_destroy(sm, is_pri); + return -1; + } + } else { + sm->ptr = (void *) MapViewOfFileEx(fd, FILE_MAP_ALL_ACCESS, /* Read/write access */ + 0, 0, /* Offset = 0 */ + 0 /* map all object */, (LPVOID) mmap_ptr); + if ((mmap_ptr != NULL) && (sm->ptr != mmap_ptr)) { + fprintf(stderr, "!mmap(%p) = %p %s shared memory error\n", mmap_ptr, + sm->ptr, sm->name); + printLastError(GetLastError()); + (void) shm_destroy(sm, is_pri); + return -1; + } + } + + if (sm->ptr == NULL) { + fprintf(stderr, "ERROR: %s failed to map view of shared memory (%d)\n", sm->name, + GetLastError()); + printLastError(GetLastError()); + (void) shm_destroy(sm, is_pri); + return -1; + } + + sm->fd = fd; + return 0; +} +#endif + +#endif /* _WIN32 || __linux__ || __FreeBSD__ */ diff --git a/test/mp-app/mp_shared_mem.h b/test/mp-app/mp_shared_mem.h new file mode 100644 index 00000000..0201b0e2 --- /dev/null +++ b/test/mp-app/mp_shared_mem.h @@ -0,0 +1,105 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#ifndef MP_SHARED_MEM_H +#define MP_SHARED_MEM_H + +#include + +/* + * ============================================================================= + * ============================================================================= + * Shared memory definitions + */ + +#if defined(__MINGW32__) + +struct shared_memory { + size_t size; + const char *name; + void *ptr; +}; + +#else + +#ifdef _WIN32 +/* + * Disable C5105 to workaround warning coming from winbase.h file + * "Windows Kits\10\include\10.0.19041.0\um\winbase.h(9531): warning + * C5105: macro expansion producing 'defined' has undefined behavior" + */ +#pragma warning(disable : 5105) + +#include +#include +#endif + +struct shared_memory { + size_t size; +#ifdef _WIN32 + const TCHAR *name; + HANDLE fd; +#else + const char *name; +#endif + void *ptr; +}; + +#define SHM_DATA_SIZE (2ULL * 1024ULL * 1024ULL) +#define SHM_INFO_SIZE (4ULL * 1024ULL) + +#ifdef _WIN32 +#define SHM_DATA_NAME TEXT("Local\\MpAppShmData") +#define SHM_INFO_NAME TEXT("Local\\MpAppShmInfo") +#endif + +#ifdef __linux__ +#define SHM_DATA_NAME "mp-app-shm-data" +#define SHM_INFO_NAME "mp-app-shm-info" +#endif + +#ifdef __FreeBSD__ +#define SHM_DATA_NAME "/tmp/mp-app-shm-data" +#define SHM_INFO_NAME "/tmp/mp-app-shm-info" +#endif + +#endif /* _WIN32 || __linux__ || __FreeBSD__ */ + +/* + * ============================================================================= + * ============================================================================= + * Shared memory API + */ + +int +shm_destroy(struct shared_memory *sm, const int is_pri); + +int +shm_create(struct shared_memory *sm, const int is_pri, const char *name, const size_t size, + void *mmap_ptr); + +#endif /* MP_SHARED_MEM_H */