From af27b8bed4c7bff04679309c651dfa7dffa578db Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Mon, 5 Dec 2022 12:12:54 +0200 Subject: test: sched_perf: print SHM in verbose mode Print SHM memory reservations which may be useful information when debugging. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/performance/odp_sched_perf.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index cd1afdf60..a97686c12 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -1375,6 +1375,9 @@ int main(int argc, char **argv) if (create_queues(global)) return -1; + if (global->test_options.verbose) + odp_shm_print_all(); + /* Start workers */ start_workers(global, instance); -- cgit v1.2.3 From 7c69ea604ecaaa2e953ef9bad6cc6365b6c5ce64 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 5 Dec 2022 16:06:23 +0200 Subject: validation: shm: fix shmem_test_info() block size Use test default block size if odp_shm_capability_t.max_size is zero. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- test/validation/api/shmem/shmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/validation/api/shmem/shmem.c b/test/validation/api/shmem/shmem.c index 2083101bd..86d070a4a 100644 --- a/test/validation/api/shmem/shmem.c +++ b/test/validation/api/shmem/shmem.c @@ -277,7 +277,7 @@ static void shmem_test_info(void) int support_pa = 0; int support_iova = 0; - if (_global_shm_capa.max_size < size) + if (_global_shm_capa.max_size && _global_shm_capa.max_size < size) size = _global_shm_capa.max_size; if (_global_shm_capa.max_align < align) -- cgit v1.2.3 From 2dde0be2817a0833c447f46083b4fd278863bcdb Mon Sep 17 00:00:00 2001 From: Malvika Gupta Date: Wed, 12 Oct 2022 00:15:37 +0000 Subject: linux-gen: crypto: add support for Arm-optimzed IPSec multi-buffer library - Enable detection of the Arm-optimized IPSec Multi-buffer library. - Integrate ZUC128-EEA3 and ZUC128-EIA3 algorithms. - Integrate ZUC256-EEA3 and ZUC256-EIA3 algorithms. Signed-off-by: Malvika Gupta Signed-off-by: Tianyu Li Reviewed-by: Ruifeng Wang Reviewed-by: Matias Elo Reviewed-by: Janne Peltonen --- DEPENDENCIES | 60 +- include/odp/autoheader_internal.h.in | 3 + platform/linux-generic/Makefile.am | 6 + platform/linux-generic/libodp-linux.pc.in | 2 +- platform/linux-generic/m4/configure.m4 | 3 +- platform/linux-generic/m4/odp_crypto.m4 | 12 +- platform/linux-generic/m4/odp_ipsec_mb.m4 | 19 + platform/linux-generic/odp_crypto_ipsecmb.c | 964 ++++++++++++++++++++++++++++ 8 files changed, 1049 insertions(+), 20 deletions(-) create mode 100644 platform/linux-generic/m4/odp_ipsec_mb.m4 create mode 100644 platform/linux-generic/odp_crypto_ipsecmb.c diff --git a/DEPENDENCIES b/DEPENDENCIES index 0a1bea32b..34a6a0aa6 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -124,11 +124,39 @@ Prerequisites for building the OpenDataPlane (ODP) API $ ./configure --with-crypto=armv8crypto $ make -3.4 Netmap packet I/O support (optional) +3.4 Multi-buffer Crypto for IPsec Library (optional) + + Multi-Buffer Crypto for IPsec Library is a set of functions that + implement authentication and encryption processing for IPsec, these functions + take advantage of SIMD instructions to improve performance. + + Note ODP assumes that IPSec MB library is compiled with SAFE_PARAM enabled + (enabled by default), otherwise crypto operations with too long cipher/auth + ranges will have undefined behaviour. + +3.4.1 Building Multi-buffer Crypto for IPSec Library for Arm + + # Checkout and build Arm code + $ git clone https://git.gitlab.arm.com/arm-reference-solutions/ipsec-mb.git + $ cd ipsec-mb/ + $ git checkout SECLIB-IPSEC-2022.05.25 + $ make + $ sudo make install + + For additional instructions, refer to README.md in crypto library repository. + +3.4.2 Building ODP with Multi-buffer IPSec Library + $ ./bootstrap + + # Compile and build ODP with Multi-buffer IPSec library + $ ./configure --with-crypto=ipsecmb + $ make + +3.5 Netmap packet I/O support (optional) Netmap accelerated ODP packet I/O. -3.4.1 Building netmap kernel modules +3.5.1 Building netmap kernel modules ODP works at least with the latest release version of netmap, which is currently v13.0. However, if possible one should try to use the latest netmap @@ -172,14 +200,14 @@ Prerequisites for building the OpenDataPlane (ODP) API $ ./configure --kernel-sources= $ make -3.4.2 Building ODP +3.5.2 Building ODP $ cd $ ./bootstrap $ ./configure --with-netmap-path= $ make -3.4.3 Inserting netmap kernel modules +3.5.3 Inserting netmap kernel modules In order to use netmap I/O you need to insert at least the core netmap kernel module. @@ -197,13 +225,13 @@ Prerequisites for building the OpenDataPlane (ODP) API To restore the original drivers you should be able to use modprobe. -3.4.4 Running ODP with netmap I/O +3.5.4 Running ODP with netmap I/O ODP applications will use netmap for packet I/O by default as long as the netmap kernel module is loaded. If socket I/O is desired instead, it can be activated by setting the environment variable ODP_PKTIO_DISABLE_NETMAP. -3.5 DPDK packet I/O support (optional) +3.6 DPDK packet I/O support (optional) Use DPDK for ODP packet I/O. Currently supported DPDK versions are v19.11, v20.11 (recommended), v21.11. @@ -212,7 +240,7 @@ Prerequisites for building the OpenDataPlane (ODP) API https://github.com/OpenDataPlane/odp-dpdk.git for a full DPDK based ODP implementation. -3.5.1 DPDK pktio requirements +3.6.1 DPDK pktio requirements DPDK pktio adds a dependency to NUMA library. # Debian/Ubuntu @@ -221,11 +249,11 @@ Prerequisites for building the OpenDataPlane (ODP) API # CentOS/RedHat/Fedora $ sudo yum install numactl-devel -3.5.2 Native DPDK install +3.6.2 Native DPDK install # Debian/Ubuntu starting from 20.04 $ sudo apt-get install dpdk-dev -3.5.3 Build DPDK v19.11 from source +3.6.3 Build DPDK v19.11 from source $ git clone https://dpdk.org/git/dpdk-stable --branch 19.11 --depth 1 ./ # Make and edit DPDK configuration @@ -243,7 +271,7 @@ Prerequisites for building the OpenDataPlane (ODP) API # Configure ODP $ ./configure --with-dpdk-path= -3.5.4 Build DPDK v20.11 and onwards from source +3.6.4 Build DPDK v20.11 and onwards from source $ git clone https://dpdk.org/git/dpdk-stable --branch --depth 1 ./ # Prepare the build directory @@ -263,7 +291,7 @@ Prerequisites for building the OpenDataPlane (ODP) API # Or, if DPDK was not installed to the default location, set PKG_CONFIG_PATH: $ PKG_CONFIG_PATH=/install/lib/x86_64-linux-gnu/pkgconfig ./configure --enable-dpdk -3.5.5 Setup system +3.6.5 Setup system # Load DPDK modules $ sudo modprobe uio @@ -275,7 +303,7 @@ Prerequisites for building the OpenDataPlane (ODP) API 512 x 2MB huge pages. All this can be done with the DPDK setup script (/usertools/dpdk-setup.sh). -3.5.6 Running ODP with DPDK pktio +3.6.6 Running ODP with DPDK pktio ODP applications will try use DPDK for packet I/O by default. If some other I/O type is desired instead, DPDK I/O can be disabled by setting the @@ -291,7 +319,7 @@ Prerequisites for building the OpenDataPlane (ODP) API 1024MB of memory: $ sudo ODP_PKTIO_DPDK_PARAMS="-m 1024" ./test/performance/odp_l2fwd -i 0 -c 1 -3.6 AF_XDP socket based packet I/O support (optional) +3.7 AF_XDP socket based packet I/O support (optional) Use AF_XDP socket for packet I/O. At the moment, only zero-copy variant is supported, requiring a kernel version 5.4 or higher. Additionally, if packet @@ -308,7 +336,7 @@ Prerequisites for building the OpenDataPlane (ODP) API Note that, currently, AF_XDP socket packet I/O cannot be instantiated if DPDK zero-copy is enabled. -3.6.1 AF_XDP socket packet I/O requirements +3.7.1 AF_XDP socket packet I/O requirements AF_XDP socket packet I/O implementation requires libxdp and libbpf libraries. They can be fetched from XDP-project in GitHub: @@ -330,14 +358,14 @@ Prerequisites for building the OpenDataPlane (ODP) API $ cd $ make install -3.6.2 Build ODP with AF_XDP socket packet I/O support +3.7.2 Build ODP with AF_XDP socket packet I/O support After building and installing libxdp and libbpf, ODP can be configured to be built with AF_XDP support (pass PKG_CONFIG_PATH if needed). $ ./configure --enable-xdp -3.6.3 Running ODP with AF_XDP socket packet I/O +3.7.3 Running ODP with AF_XDP socket packet I/O AF_XDP socket packet I/Os bind to TRX-combined queues. Based on the packet prosessing needs, NIC(s) of the environment should be configured diff --git a/include/odp/autoheader_internal.h.in b/include/odp/autoheader_internal.h.in index 33d9f280f..a42b34ad0 100644 --- a/include/odp/autoheader_internal.h.in +++ b/include/odp/autoheader_internal.h.in @@ -35,4 +35,7 @@ /* Define to 1 to enable XDP support */ #undef _ODP_PKTIO_XDP +/* Define to 1 to enable IPSec MB crypto support */ +#undef _ODP_IPSECMB + #endif diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index e762148aa..3310e9603 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -278,10 +278,15 @@ if WITH_ARMV8_CRYPTO __LIB__libodp_linux_la_SOURCES += \ arch/aarch64/odp_crypto_armv8.c else +if WITH_IPSECMB_CRYPTO +__LIB__libodp_linux_la_SOURCES += \ + odp_crypto_ipsecmb.c +else __LIB__libodp_linux_la_SOURCES += \ odp_crypto_null.c endif endif +endif if ODP_ABI_COMPAT __LIB__libodp_linux_la_SOURCES += \ odp_atomic_api.c \ @@ -438,6 +443,7 @@ __LIB__libodp_linux_la_LIBADD += $(DPDK_LIBS_LIBODP) __LIB__libodp_linux_la_LIBADD += $(PTHREAD_LIBS) __LIB__libodp_linux_la_LIBADD += $(TIMER_LIBS) __LIB__libodp_linux_la_LIBADD += $(LIBXDP_LIBS) +__LIB__libodp_linux_la_LIBADD += $(IPSEC_MB_LIBS) if ODP_PKTIO_PCAP __LIB__libodp_linux_la_LIBADD += $(PCAP_LIBS) diff --git a/platform/linux-generic/libodp-linux.pc.in b/platform/linux-generic/libodp-linux.pc.in index f9a339fb8..05ba5b9d6 100644 --- a/platform/linux-generic/libodp-linux.pc.in +++ b/platform/linux-generic/libodp-linux.pc.in @@ -8,5 +8,5 @@ Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Requires.private: libconfig@AARCH64CRYPTO_PKG@ Libs: -L${libdir} -l@ODP_LIB_NAME@ @ATOMIC_LIBS_NON_ABI_COMPAT@ -Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ @LIBXDP_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ +Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ @LIBXDP_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ @IPSEC_MB_LIBS@ Cflags: -I${includedir} diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 4f3365ea6..61d57634f 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -25,13 +25,14 @@ AM_CONDITIONAL([ODP_PKTIO_PCAP], [test x$have_pcap = xyes]) m4_include([platform/linux-generic/m4/odp_libconfig.m4]) m4_include([platform/linux-generic/m4/odp_openssl.m4]) m4_include([platform/linux-generic/m4/odp_crypto.m4]) +m4_include([platform/linux-generic/m4/odp_ipsec_mb.m4]) m4_include([platform/linux-generic/m4/odp_pcapng.m4]) m4_include([platform/linux-generic/m4/odp_netmap.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) m4_include([platform/linux-generic/m4/odp_xdp.m4]) ODP_SCHEDULER -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${IPSEC_MB_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. diff --git a/platform/linux-generic/m4/odp_crypto.m4 b/platform/linux-generic/m4/odp_crypto.m4 index 9bb99f7dd..1cec6edb4 100644 --- a/platform/linux-generic/m4/odp_crypto.m4 +++ b/platform/linux-generic/m4/odp_crypto.m4 @@ -3,7 +3,7 @@ # Select default crypto implementation AC_ARG_WITH([crypto], [AS_HELP_STRING([--with-crypto], - [Choose crypto implementation (openssl/armv8crypto/null)] + [Choose crypto implementation (openssl/armv8crypto/ipsecmb/null)] [[default=openssl] (linux-generic)])], [], [with_crypto=openssl]) @@ -14,7 +14,7 @@ AS_IF([test "x$with_crypto" = "xyes"], [with_crypto=openssl]) AS_IF([test "x$with_crypto" = "xno"], [with_crypto=null]) AS_IF([test "x$with_crypto" = "xopenssl" -a "x$with_openssl" = "xno"], [with_crypto=null]) -AS_IF([test "x$with_crypto" != "xopenssl" -a "x$with_crypto" != "xarmv8crypto" -a "x$with_crypto" != "xnull"], +AS_IF([test "x$with_crypto" != "xopenssl" -a "x$with_crypto" != "xarmv8crypto" -a "x$with_crypto" != "xipsecmb" -a "x$with_crypto" != "xnull"], [AC_MSG_ERROR([Invalid crypto implementation name])]) ########################################################################## @@ -31,10 +31,18 @@ AS_IF([test "x$with_crypto" == "xarmv8crypto"], [PKG_CHECK_MODULES([AARCH64CRYPTO], [libAArch64crypto]) AARCH64CRYPTO_PKG=", libAArch64crypto" AC_SUBST([AARCH64CRYPTO_PKG])]) + AC_CONFIG_COMMANDS_PRE([dnl AM_CONDITIONAL([WITH_ARMV8_CRYPTO], [test "x$with_crypto" == "xarmv8crypto"]) ]) +########################################################################## +# Multi-buffer IPSec library implementation +########################################################################## +AC_CONFIG_COMMANDS_PRE([dnl +AM_CONDITIONAL([WITH_IPSECMB_CRYPTO], [test "x$with_crypto" == "xipsecmb"]) +]) + ########################################################################## # Null implementation ########################################################################## diff --git a/platform/linux-generic/m4/odp_ipsec_mb.m4 b/platform/linux-generic/m4/odp_ipsec_mb.m4 new file mode 100644 index 000000000..3268d94c0 --- /dev/null +++ b/platform/linux-generic/m4/odp_ipsec_mb.m4 @@ -0,0 +1,19 @@ +######################################################################### +# Check for libIPSec_MB availability +######################################################################### +ipsecmb_support=no +AC_CHECK_HEADERS([ipsec-mb.h], + [AC_CHECK_LIB([IPSec_MB], [init_mb_mgr_auto], [ipsecmb_support=yes], + [ipsecmb_support=no])], + [ipsecmb_support=no]) + +AS_IF([test "x$with_crypto" = "xipsecmb" -a "x$ipsecmb_support" = "xno"], + [AC_MSG_ERROR([IPSec MB library not found on this platform])]) + +if test "x$with_crypto" = "xipsecmb"; then + IPSEC_MB_LIBS="-lIPSec_MB" +else + IPSEC_MB_LIBS="" +fi + +AC_SUBST([IPSEC_MB_LIBS]) diff --git a/platform/linux-generic/odp_crypto_ipsecmb.c b/platform/linux-generic/odp_crypto_ipsecmb.c new file mode 100644 index 000000000..d291a7067 --- /dev/null +++ b/platform/linux-generic/odp_crypto_ipsecmb.c @@ -0,0 +1,964 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#define MAX_SESSIONS 4000 +/* Length in bytes */ +#define IPSEC_MB_CRYPTO_MAX_CIPHER_KEY_LENGTH 32 +#define IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH 32 +#define IPSEC_MB_CRYPTO_MAX_IV_LENGTH 32 +#define IPSEC_MB_CRYPTO_MAX_DATA_LENGTH 65536 +#define ZUC_DIGEST_LENGTH 4 + +#define ODP_CRYPTO_IPSEC_MB_SHM_NAME "_odp_crypto_ipsecmb" +/* + * Cipher algorithm capabilities + * + * Keep sorted: first by key length, then by IV length + */ +static const odp_crypto_cipher_capability_t cipher_capa_null[] = { +{.key_len = 0, .iv_len = 0} }; + +static const odp_crypto_cipher_capability_t cipher_capa_zuc_eea3[] = { +{.key_len = 16, .iv_len = 16}, +{.key_len = 32, .iv_len = 25} }; + +/* + * Authentication algorithm capabilities + * + * Keep sorted: first by digest length, then by key length + */ +static const odp_crypto_auth_capability_t auth_capa_null[] = { +{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; + +static const odp_crypto_auth_capability_t auth_capa_zuc_eia3[] = { +{.digest_len = 4, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0}, + .iv_len = 16}, +{.digest_len = 4, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0}, + .iv_len = 25} }; + +/** Forward declaration of session structure */ +typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t; + +/** + * Algorithm handler function prototype + */ +typedef odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session); + +/** + * Per crypto session data structure + */ +struct odp_crypto_generic_session_t { + odp_crypto_generic_session_t *next; + + /* Session creation parameters */ + odp_crypto_session_param_t p; + + odp_bool_t do_cipher_first; + + struct { +#if ODP_DEPRECATED_API + /* Copy of session IV data */ + uint8_t iv_data[IPSEC_MB_CRYPTO_MAX_IV_LENGTH]; +#endif + uint8_t key_data[IPSEC_MB_CRYPTO_MAX_CIPHER_KEY_LENGTH]; + crypto_func_t func; + } cipher; + + struct { + uint8_t key[IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH]; +#if ODP_DEPRECATED_API + uint8_t iv_data[IPSEC_MB_CRYPTO_MAX_IV_LENGTH]; +#endif + crypto_func_t func; + } auth; + + unsigned int idx; +}; + +typedef struct odp_crypto_global_s odp_crypto_global_t; + +struct odp_crypto_global_s { + odp_spinlock_t lock; + odp_crypto_generic_session_t *free; + odp_crypto_generic_session_t sessions[MAX_SESSIONS]; +}; + +static odp_crypto_global_t *global; + +typedef struct crypto_local_t { + uint8_t buffer[IPSEC_MB_CRYPTO_MAX_DATA_LENGTH]; + IMB_MGR *mb_mgr; +} crypto_local_t; + +static __thread crypto_local_t local; + +static +odp_crypto_generic_session_t *alloc_session(void) +{ + odp_crypto_generic_session_t *session = NULL; + + odp_spinlock_lock(&global->lock); + session = global->free; + if (session) { + global->free = session->next; + session->next = NULL; + } + odp_spinlock_unlock(&global->lock); + + if (!session) + return NULL; + + session->idx = session - global->sessions; + + return session; +} + +static +void free_session(odp_crypto_generic_session_t *session) +{ + odp_spinlock_lock(&global->lock); + session->next = global->free; + global->free = session; + odp_spinlock_unlock(&global->lock); +} + +static +odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) +{ + odp_packet_hdr_t *hdr = packet_hdr(pkt); + + return &hdr->crypto_op_result; +} + +static odp_crypto_alg_err_t +null_crypto_routine(odp_packet_t pkt ODP_UNUSED, + const odp_crypto_packet_op_param_t *param ODP_UNUSED, + odp_crypto_generic_session_t *session ODP_UNUSED) +{ + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static +odp_crypto_alg_err_t zuc_eea3_cipher_op(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + IMB_MGR *mb_mgr = local.mb_mgr; + uint8_t *iv_ptr; + uint32_t in_pos = param->cipher_range.offset; + uint32_t in_len = param->cipher_range.length; + +#if ODP_DEPRECATED_API + if (param->cipher_iv_ptr) + iv_ptr = param->cipher_iv_ptr; + else if (session->p.cipher_iv.data) + iv_ptr = session->cipher.iv_data; + else + return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->cipher_iv_ptr; + _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); +#endif + + uint32_t seg_len = 0; + uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); + + if (odp_unlikely(seg_len < in_len)) { + if (odp_unlikely(in_len > IPSEC_MB_CRYPTO_MAX_DATA_LENGTH)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Packet is segmented within the cipher range. Copy the cipher + * range to a contiguous buffer. */ + odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); + + data = local.buffer; + } + + if (session->p.cipher_key.length == 16) { + /* ZUC128 EEA3 */ + IMB_ZUC_EEA3_1_BUFFER(mb_mgr, session->cipher.key_data, + iv_ptr, + data, + data, + in_len); + } else { + /* Only 16 and 32 byte keys are supported + * ZUC256 EEA3 */ + IMB_ZUC256_EEA3_1_BUFFER(mb_mgr, session->cipher.key_data, + iv_ptr, + data, + data, + in_len); + } + if (odp_unlikely(imb_get_errno(mb_mgr) != 0)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + if (odp_unlikely(seg_len < in_len)) + odp_packet_copy_from_mem(pkt, in_pos, in_len, data); + + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static int process_zuc_eea3_param(odp_crypto_generic_session_t *session) +{ + if (!((16 == session->p.cipher_key.length && + 16 == session->p.cipher_iv_len) || + (32 == session->p.cipher_key.length && + 25 == session->p.cipher_iv_len))) + return -1; + + memcpy(session->cipher.key_data, session->p.cipher_key.data, + session->p.cipher_key.length); + + session->cipher.func = zuc_eea3_cipher_op; + + return 0; +} + +static +odp_crypto_alg_err_t auth_zuc_eia3_gen(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + IMB_MGR *mb_mgr = local.mb_mgr; + uint8_t *iv_ptr; + uint32_t in_pos = param->auth_range.offset; + uint32_t in_len = param->auth_range.length; + uint32_t auth_tag; + +#if ODP_DEPRECATED_API + if (param->auth_iv_ptr) + iv_ptr = param->auth_iv_ptr; + else if (session->p.auth_iv.data) + iv_ptr = session->auth.iv_data; + else + return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->auth_iv_ptr; + _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); +#endif + + uint32_t seg_len = 0; + uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); + + if (odp_unlikely(seg_len < in_len)) { + if (odp_unlikely(in_len > IPSEC_MB_CRYPTO_MAX_DATA_LENGTH)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Packet is segmented within the auth range. Copy the auth + * range to a contiguous buffer. */ + odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); + + data = local.buffer; + } + + if (session->p.auth_key.length == 16) { + /* ZUC128 EIA3 */ + IMB_ZUC_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &auth_tag); + } else { + /* Only 16 and 32 byte keys are supported + * ZUC256 EIA3 */ + IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &auth_tag); + } + if (odp_unlikely(imb_get_errno(mb_mgr) != 0)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Copy to the output location */ + odp_packet_copy_from_mem(pkt, param->hash_result_offset, + session->p.auth_digest_len, + &auth_tag); + + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static +odp_crypto_alg_err_t auth_zuc_eia3_check(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + IMB_MGR *mb_mgr = local.mb_mgr; + uint8_t *iv_ptr; + uint32_t in_pos = param->auth_range.offset; + uint32_t in_len = param->auth_range.length; + uint32_t bytes = ZUC_DIGEST_LENGTH; + uint32_t hash_in; + uint32_t hash_out; + + /* Copy current value out and clear it before authentication */ + odp_packet_copy_to_mem(pkt, param->hash_result_offset, + bytes, &hash_in); + + if (odp_unlikely(session->p.hash_result_in_auth_range)) + _odp_packet_set_data(pkt, param->hash_result_offset, 0, bytes); + +#if ODP_DEPRECATED_API + if (param->auth_iv_ptr) + iv_ptr = param->auth_iv_ptr; + else if (session->p.auth_iv.data) + iv_ptr = session->auth.iv_data; + else + return ODP_CRYPTO_ALG_ERR_IV_INVALID; +#else + iv_ptr = param->auth_iv_ptr; + _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); +#endif + + uint32_t seg_len = 0; + uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); + + if (odp_unlikely(seg_len < in_len)) { + if (odp_unlikely(in_len > IPSEC_MB_CRYPTO_MAX_DATA_LENGTH)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Packet is segmented within the auth range. Copy the auth + * range to a contiguous buffer. */ + odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); + + data = local.buffer; + } + + if (session->p.auth_key.length == 16) { + /* ZUC128 EIA3 */ + IMB_ZUC_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &hash_out); + } else { + /* Only 16 and 32 byte keys are supported + * ZUC256 EIA3 */ + IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, session->auth.key, + iv_ptr, + data, + param->auth_range.length * 8, + &hash_out); + } + if (odp_unlikely(imb_get_errno(mb_mgr) != 0)) + return ODP_CRYPTO_ALG_ERR_DATA_SIZE; + + /* Verify match */ + if (hash_in != hash_out) + return ODP_CRYPTO_ALG_ERR_ICV_CHECK; + + return ODP_CRYPTO_ALG_ERR_NONE; +} + +static int process_auth_zuc_eia3_param(odp_crypto_generic_session_t *session) +{ + if (!((16 == session->p.auth_key.length && + 16 == session->p.auth_iv_len) || + (32 == session->p.auth_key.length && + 25 == session->p.auth_iv_len))) + return -1; + + if (ODP_CRYPTO_OP_ENCODE == session->p.op) + session->auth.func = auth_zuc_eia3_gen; + else + session->auth.func = auth_zuc_eia3_check; + + if (session->p.auth_digest_len != ZUC_DIGEST_LENGTH) + return -1; + + memcpy(session->auth.key, session->p.auth_key.data, + session->p.auth_key.length); + + return 0; +} + +int odp_crypto_capability(odp_crypto_capability_t *capa) +{ + if (NULL == capa) + return -1; + + memset(capa, 0, sizeof(odp_crypto_capability_t)); + + capa->sync_mode = ODP_SUPPORT_PREFERRED; + capa->async_mode = ODP_SUPPORT_YES; + capa->queue_type_plain = 1; + capa->queue_type_sched = 1; + + capa->ciphers.bit.null = 1; + capa->auths.bit.null = 1; + + capa->ciphers.bit.zuc_eea3 = 1; + capa->auths.bit.zuc_eia3 = 1; + + capa->max_sessions = MAX_SESSIONS; + + return 0; +} + +int odp_crypto_cipher_capability(odp_cipher_alg_t cipher, + odp_crypto_cipher_capability_t dst[], + int num_copy) +{ + const odp_crypto_cipher_capability_t *src; + int num; + int size = sizeof(odp_crypto_cipher_capability_t); + + switch (cipher) { + case ODP_CIPHER_ALG_NULL: + src = cipher_capa_null; + num = sizeof(cipher_capa_null) / size; + break; + case ODP_CIPHER_ALG_ZUC_EEA3: + src = cipher_capa_zuc_eea3; + num = sizeof(cipher_capa_zuc_eea3) / size; + break; + default: + return -1; + } + + if (num < num_copy) + num_copy = num; + + memcpy(dst, src, num_copy * size); + + return num; +} + +int odp_crypto_auth_capability(odp_auth_alg_t auth, + odp_crypto_auth_capability_t dst[], int num_copy) +{ + const odp_crypto_auth_capability_t *src; + int num; + int size = sizeof(odp_crypto_auth_capability_t); + + switch (auth) { + case ODP_AUTH_ALG_NULL: + src = auth_capa_null; + num = sizeof(auth_capa_null) / size; + break; + case ODP_AUTH_ALG_ZUC_EIA3: + src = auth_capa_zuc_eia3; + num = sizeof(auth_capa_zuc_eia3) / size; + break; + default: + return -1; + } + + if (num < num_copy) + num_copy = num; + + memcpy(dst, src, num_copy * size); + + return num; +} + +int +odp_crypto_session_create(const odp_crypto_session_param_t *param, + odp_crypto_session_t *session_out, + odp_crypto_ses_create_err_t *status) +{ + int rc = 0; + odp_crypto_generic_session_t *session; + + if (odp_global_ro.disable.crypto) { + _ODP_ERR("Crypto is disabled\n"); + /* Dummy output to avoid compiler warning about uninitialized + * variables */ + *status = ODP_CRYPTO_SES_ERR_ENOMEM; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + + session = alloc_session(); + if (NULL == session) { + *status = ODP_CRYPTO_SES_ERR_ENOMEM; + goto err; + } + + session->p = *param; + + if (session->p.cipher_iv_len > IPSEC_MB_CRYPTO_MAX_IV_LENGTH) { + _ODP_DBG("Maximum IV length exceeded\n"); + *status = ODP_CRYPTO_SES_ERR_CIPHER; + goto err; + } + + if (session->p.auth_iv_len > IPSEC_MB_CRYPTO_MAX_IV_LENGTH) { + _ODP_DBG("Maximum auth IV length exceeded\n"); + *status = ODP_CRYPTO_SES_ERR_CIPHER; + goto err; + } + +#if ODP_DEPRECATED_API + /* Copy IV data */ + if (session->p.cipher_iv.data) + memcpy(session->cipher.iv_data, session->p.cipher_iv.data, + session->p.cipher_iv.length); + + if (session->p.auth_iv.data) + memcpy(session->auth.iv_data, session->p.auth_iv.data, + session->p.auth_iv.length); +#endif + + /* Derive order */ + if (ODP_CRYPTO_OP_ENCODE == param->op) + session->do_cipher_first = param->auth_cipher_text; + else + session->do_cipher_first = !param->auth_cipher_text; + + /* Process based on cipher */ + switch (param->cipher_alg) { + case ODP_CIPHER_ALG_NULL: + session->cipher.func = null_crypto_routine; + rc = 0; + break; + case ODP_CIPHER_ALG_ZUC_EEA3: + rc = process_zuc_eea3_param(session); + break; + default: + rc = -1; + } + + if (rc) { + *status = ODP_CRYPTO_SES_ERR_CIPHER; + goto err; + } + + /* Process based on auth */ + switch (param->auth_alg) { + case ODP_AUTH_ALG_NULL: + session->auth.func = null_crypto_routine; + rc = 0; + break; + case ODP_AUTH_ALG_ZUC_EIA3: + rc = process_auth_zuc_eia3_param(session); + break; + default: + rc = -1; + } + + if (rc) { + *status = ODP_CRYPTO_SES_ERR_AUTH; + goto err; + } + + *session_out = (intptr_t)session; + *status = ODP_CRYPTO_SES_ERR_NONE; + return 0; + +err: + /* error status should be set at this moment */ + if (session != NULL) + free_session(session); + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; +} + +int odp_crypto_session_destroy(odp_crypto_session_t session) +{ + odp_crypto_generic_session_t *generic; + + generic = (odp_crypto_generic_session_t *)(intptr_t)session; + memset(generic, 0, sizeof(*generic)); + free_session(generic); + return 0; +} + +#if ODP_DEPRECATED_API +int +odp_crypto_operation(odp_crypto_op_param_t *param, + odp_bool_t *posted, + odp_crypto_op_result_t *result) +{ + odp_crypto_packet_op_param_t packet_param; + odp_packet_t out_pkt = param->out_pkt; + odp_crypto_packet_result_t packet_result; + odp_crypto_op_result_t local_result; + int rc; + + packet_param.session = param->session; + packet_param.cipher_iv_ptr = param->cipher_iv_ptr; + packet_param.auth_iv_ptr = param->auth_iv_ptr; + packet_param.hash_result_offset = param->hash_result_offset; + packet_param.aad_ptr = param->aad_ptr; + packet_param.cipher_range = param->cipher_range; + packet_param.auth_range = param->auth_range; + + rc = odp_crypto_op(¶m->pkt, &out_pkt, &packet_param, 1); + if (rc <= 0) + return -1; + + rc = odp_crypto_result(&packet_result, out_pkt); + if (rc < 0) { + /* + * We cannot fail since odp_crypto_op() has already processed + * the packet. Let's indicate error in the result instead. + */ + packet_result.ok = false; + } + + /* Indicate to caller operation was sync */ + *posted = 0; + + packet_subtype_set(out_pkt, ODP_EVENT_PACKET_BASIC); + + /* Fill in result */ + local_result.ctx = param->ctx; + local_result.pkt = out_pkt; + local_result.cipher_status = packet_result.cipher_status; + local_result.auth_status = packet_result.auth_status; + local_result.ok = packet_result.ok; + + /* + * Be bug-to-bug compatible. Return output packet also through params. + */ + param->out_pkt = out_pkt; + + *result = local_result; + + return 0; +} +#endif + +int _odp_crypto_init_global(void) +{ + size_t mem_size; + odp_shm_t shm; + int idx; + + if (odp_global_ro.disable.crypto) { + _ODP_PRINT("\nODP crypto is DISABLED\n"); + return 0; + } + + /* Calculate the memory size we need */ + mem_size = sizeof(odp_crypto_global_t); + + /* Allocate our globally shared memory */ + shm = odp_shm_reserve(ODP_CRYPTO_IPSEC_MB_SHM_NAME, mem_size, + ODP_CACHE_LINE_SIZE, + 0); + if (ODP_SHM_INVALID == shm) { + _ODP_ERR("unable to allocate crypto pool\n"); + return -1; + } + + global = odp_shm_addr(shm); + + /* Clear it out */ + memset(global, 0, mem_size); + + /* Initialize free list and lock */ + for (idx = 0; idx < MAX_SESSIONS; idx++) { + global->sessions[idx].next = global->free; + global->free = &global->sessions[idx]; + } + odp_spinlock_init(&global->lock); + + return 0; +} + +int _odp_crypto_term_global(void) +{ + int rc = 0; + int ret; + int count = 0; + odp_crypto_generic_session_t *session; + + if (odp_global_ro.disable.crypto) + return 0; + + for (session = global->free; session != NULL; session = session->next) + count++; + if (count != MAX_SESSIONS) { + _ODP_ERR("crypto sessions still active\n"); + rc = -1; + } + + ret = odp_shm_free(odp_shm_lookup(ODP_CRYPTO_IPSEC_MB_SHM_NAME)); + if (ret < 0) { + _ODP_ERR("shm free failed for %s\n", ODP_CRYPTO_IPSEC_MB_SHM_NAME); + rc = -1; + } + + return rc; +} + +int _odp_crypto_init_local(void) +{ + uint64_t flags = 0; + + if (odp_global_ro.disable.crypto) + return 0; + + memset(&local, 0, sizeof(local)); + + local.mb_mgr = alloc_mb_mgr(flags); + if (local.mb_mgr == NULL) + return -1; + + init_mb_mgr_auto(local.mb_mgr, NULL); + + return 0; +} + +int _odp_crypto_term_local(void) +{ + if (odp_global_ro.disable.crypto) + return 0; + + free_mb_mgr(local.mb_mgr); + return 0; +} + +#if ODP_DEPRECATED_API +odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev) +{ + /* This check not mandated by the API specification */ + if (odp_event_type(ev) != ODP_EVENT_CRYPTO_COMPL) + _ODP_ABORT("Event not a crypto completion"); + return (odp_crypto_compl_t)ev; +} + +odp_event_t odp_crypto_compl_to_event(odp_crypto_compl_t completion_event) +{ + return (odp_event_t)completion_event; +} + +void +odp_crypto_compl_result(odp_crypto_compl_t completion_event, + odp_crypto_op_result_t *result) +{ + (void)completion_event; + (void)result; + + /* We won't get such events anyway, so there can be no result */ + _ODP_ASSERT(0); +} + +void +odp_crypto_compl_free(odp_crypto_compl_t completion_event) +{ + odp_event_t ev = odp_crypto_compl_to_event(completion_event); + + odp_buffer_free(odp_buffer_from_event(ev)); +} + +uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl) +{ + return _odp_pri(hdl); +} +#endif /* ODP_DEPRECATED_API */ + +void odp_crypto_session_param_init(odp_crypto_session_param_t *param) +{ + memset(param, 0, sizeof(odp_crypto_session_param_t)); +} + +uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) +{ + return (uint64_t)hdl; +} + +odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) +{ + /* This check not mandated by the API specification */ + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); + + return odp_packet_from_event(ev); +} + +odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) +{ + return odp_packet_to_event(pkt); +} + +int odp_crypto_result(odp_crypto_packet_result_t *result, + odp_packet_t packet) +{ + odp_crypto_packet_result_t *op_result; + + _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == + ODP_EVENT_PACKET_CRYPTO); + + op_result = get_op_result_from_packet(packet); + + memcpy(result, op_result, sizeof(*result)); + + return 0; +} + +static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) +{ + int md_copy; + int rc; + + md_copy = _odp_packet_copy_md_possible(odp_packet_pool(dst), + odp_packet_pool(src)); + if (odp_unlikely(md_copy < 0)) { + _ODP_ERR("Unable to copy packet metadata\n"); + return -1; + } + + rc = odp_packet_copy_from_pkt(dst, 0, src, 0, odp_packet_len(src)); + if (odp_unlikely(rc < 0)) { + _ODP_ERR("Unable to copy packet data\n"); + return -1; + } + + _odp_packet_copy_md(packet_hdr(dst), packet_hdr(src), md_copy); + return 0; +} + +static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *session, + odp_packet_t pkt_in, + odp_packet_t pkt_out) +{ + int rc; + + if (odp_likely(pkt_in == pkt_out)) + return pkt_out; + + if (pkt_out == ODP_PACKET_INVALID) { + odp_pool_t pool = session->p.output_pool; + + _ODP_ASSERT(pool != ODP_POOL_INVALID); + if (pool == odp_packet_pool(pkt_in)) { + pkt_out = pkt_in; + } else { + pkt_out = odp_packet_copy(pkt_in, pool); + if (odp_likely(pkt_out != ODP_PACKET_INVALID)) + odp_packet_free(pkt_in); + } + return pkt_out; + } + rc = copy_data_and_metadata(pkt_out, pkt_in); + if (odp_unlikely(rc < 0)) + return ODP_PACKET_INVALID; + + odp_packet_free(pkt_in); + return pkt_out; +} + +static +int crypto_int(odp_packet_t pkt_in, + odp_packet_t *pkt_out, + const odp_crypto_packet_op_param_t *param) +{ + odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE; + odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE; + odp_crypto_generic_session_t *session; + odp_packet_t out_pkt; + odp_crypto_packet_result_t *op_result; + + session = (odp_crypto_generic_session_t *)(intptr_t)param->session; + + out_pkt = get_output_packet(session, pkt_in, *pkt_out); + if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) + return -1; + + /* Invoke the crypto function */ + if (session->do_cipher_first) { + rc_cipher = session->cipher.func(out_pkt, param, session); + rc_auth = session->auth.func(out_pkt, param, session); + } else { + rc_auth = session->auth.func(out_pkt, param, session); + rc_cipher = session->cipher.func(out_pkt, param, session); + } + + packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); + op_result = get_op_result_from_packet(out_pkt); + op_result->cipher_status.alg_err = rc_cipher; + op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; + op_result->auth_status.alg_err = rc_auth; + op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; + op_result->ok = + (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) && + (rc_auth == ODP_CRYPTO_ALG_ERR_NONE); + + /* Synchronous, simply return results */ + *pkt_out = out_pkt; + + return 0; +} + +int odp_crypto_op(const odp_packet_t pkt_in[], + odp_packet_t pkt_out[], + const odp_crypto_packet_op_param_t param[], + int num_pkt) +{ + int i, rc; + odp_crypto_generic_session_t *session; + + for (i = 0; i < num_pkt; i++) { + session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; + _ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); + + rc = crypto_int(pkt_in[i], &pkt_out[i], ¶m[i]); + if (rc < 0) + break; + } + + return i; +} + +int odp_crypto_op_enq(const odp_packet_t pkt_in[], + const odp_packet_t pkt_out[], + const odp_crypto_packet_op_param_t param[], + int num_pkt) +{ + odp_packet_t pkt; + odp_event_t event; + odp_crypto_generic_session_t *session; + int i, rc; + + for (i = 0; i < num_pkt; i++) { + session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; + _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); + _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); + + pkt = pkt_out[i]; + rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); + if (rc < 0) + break; + + event = odp_packet_to_event(pkt); + if (odp_queue_enq(session->p.compl_queue, event)) { + odp_event_free(event); + break; + } + } + + return i; +} -- cgit v1.2.3 From 108e11f18e574cd160c9bdc457d59cee4acde072 Mon Sep 17 00:00:00 2001 From: Tianyu Li Date: Tue, 1 Nov 2022 07:54:19 +0000 Subject: github_ci: add test for arm64 crypto ipsec_mb Add native build test for ipsec_mb crypto support on arm64. Signed-off-by: Tianyu Li Reviewed-by: Matias Elo --- .github/workflows/ci-pipeline-arm64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index 5d937a0b3..33ac5c65c 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -148,7 +148,7 @@ jobs: conf: ['', '--enable-abi-compat', '--enable-deprecated --enable-helper-deprecated --enable-debug=full', '--enable-dpdk-zero-copy --disable-static-applications', '--disable-host-optimization', '--disable-host-optimization --enable-abi-compat', - '--without-openssl --without-pcap', '--with-crypto=armv8crypto'] + '--without-openssl --without-pcap', '--with-crypto=armv8crypto', '--with-crypto=ipsecmb'] steps: - uses: AutoModality/action-clean@v1.1.0 - uses: actions/checkout@v3 -- cgit v1.2.3 From a8a87fb61273f9a3255f372c834c7fefd831bcd1 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 9 Dec 2022 14:20:49 +0200 Subject: test: performance: crypto: check if crypto operations succeed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add checks for the success of crypto operations. Print an error and abort the test if a crypto operation fails. This prevents failed crypto operations from going unnoticed and producing bogus performance results. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen Reviewed-by: Tuomas Taipale --- test/performance/odp_crypto.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index b381fde31..2f79ae644 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -810,6 +810,8 @@ run_measure_one(crypto_args_t *cargs, } packets_sent += rc; } else { + odp_crypto_packet_result_t result; + rc = odp_crypto_op(&pkt, &out_pkt, ¶ms, 1); if (rc <= 0) { @@ -820,6 +822,12 @@ run_measure_one(crypto_args_t *cargs, } packets_sent += rc; packets_received++; + if (odp_unlikely(odp_crypto_result(&result, out_pkt) != 0) || + odp_unlikely(!result.ok)) { + ODPH_ERR("Crypto operation failed\n"); + odp_packet_free(out_pkt); + return -1; + } if (cargs->debug_packets) { mem = odp_packet_data(out_pkt); print_mem("Immediately encrypted " @@ -849,8 +857,12 @@ run_measure_one(crypto_args_t *cargs, while (ev != ODP_EVENT_INVALID) { out_pkt = odp_crypto_packet_from_event(ev); - odp_crypto_result(&result, out_pkt); - + if (odp_unlikely(odp_crypto_result(&result, out_pkt) != 0) || + odp_unlikely(!result.ok)) { + ODPH_ERR("Crypto operation failed\n"); + odp_packet_free(out_pkt); + return -1; + } if (cargs->debug_packets) { mem = odp_packet_data(out_pkt); print_mem("Received encrypted packet", -- cgit v1.2.3 From 472b63b896364ce2c4e786025a77d10b6f135eee Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 9 Dec 2022 14:44:52 +0200 Subject: example: sysinfo: add missing crypto algorithm names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add missing crypto, auth and hash algorithm names so that sysinfo can print a name for every algorithm defined in the ODP API. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen Reviewed-by: Tuomas Taipale --- example/sysinfo/odp_sysinfo.c | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/example/sysinfo/odp_sysinfo.c b/example/sysinfo/odp_sysinfo.c index 523f70496..093b2f1bf 100644 --- a/example/sysinfo/odp_sysinfo.c +++ b/example/sysinfo/odp_sysinfo.c @@ -134,12 +134,26 @@ static const char *cipher_alg_name(odp_cipher_alg_t cipher) return "aes_cbc"; case ODP_CIPHER_ALG_AES_CTR: return "aes_ctr"; + case ODP_CIPHER_ALG_AES_ECB: + return "aes_ecb"; + case ODP_CIPHER_ALG_AES_CFB128: + return "aes_cfb128"; + case ODP_CIPHER_ALG_AES_XTS: + return "aes_xts"; case ODP_CIPHER_ALG_AES_GCM: return "aes_gcm"; case ODP_CIPHER_ALG_AES_CCM: return "aes_ccm"; case ODP_CIPHER_ALG_CHACHA20_POLY1305: return "chacha20_poly1305"; + case ODP_CIPHER_ALG_KASUMI_F8: + return "kasumi_f8"; + case ODP_CIPHER_ALG_SNOW3G_UEA2: + return "snow3g_uea2"; + case ODP_CIPHER_ALG_AES_EEA2: + return "aes_eea2"; + case ODP_CIPHER_ALG_ZUC_EEA3: + return "zuc_eea3"; default: return "Unknown"; } @@ -154,14 +168,14 @@ static const char *auth_alg_name(odp_auth_alg_t auth) return "md5_hmac"; case ODP_AUTH_ALG_SHA1_HMAC: return "sha1_hmac"; + case ODP_AUTH_ALG_SHA224_HMAC: + return "sha224_hmac"; case ODP_AUTH_ALG_SHA256_HMAC: return "sha256_hmac"; case ODP_AUTH_ALG_SHA384_HMAC: return "sha384_hmac"; case ODP_AUTH_ALG_SHA512_HMAC: return "sha512_hmac"; - case ODP_AUTH_ALG_AES_XCBC_MAC: - return "aes_xcbc_mac"; case ODP_AUTH_ALG_AES_GCM: return "aes_gcm"; case ODP_AUTH_ALG_AES_GMAC: @@ -170,8 +184,30 @@ static const char *auth_alg_name(odp_auth_alg_t auth) return "aes_ccm"; case ODP_AUTH_ALG_AES_CMAC: return "aes_cmac"; + case ODP_AUTH_ALG_AES_XCBC_MAC: + return "aes_xcbc_mac"; case ODP_AUTH_ALG_CHACHA20_POLY1305: return "chacha20_poly1305"; + case ODP_AUTH_ALG_KASUMI_F9: + return "kasumi_f9"; + case ODP_AUTH_ALG_SNOW3G_UIA2: + return "snow3g_uia2"; + case ODP_AUTH_ALG_AES_EIA2: + return "aes_eia2"; + case ODP_AUTH_ALG_ZUC_EIA3: + return "zuc_eia3"; + case ODP_AUTH_ALG_MD5: + return "md5"; + case ODP_AUTH_ALG_SHA1: + return "sha1"; + case ODP_AUTH_ALG_SHA224: + return "sha224"; + case ODP_AUTH_ALG_SHA256: + return "sha256"; + case ODP_AUTH_ALG_SHA384: + return "sha384"; + case ODP_AUTH_ALG_SHA512: + return "sha512"; default: return "Unknown"; } -- cgit v1.2.3 From 894ea21921e015ab4ff0880a67fc6302df361d71 Mon Sep 17 00:00:00 2001 From: Tianyu Li Date: Fri, 16 Dec 2022 16:51:45 +0800 Subject: dependencies: update IPsec MB library version Bump IPsec MB library version to SECLIB-IPSEC-2022.12.13, which adds some missing imb_set_errno values. Signed-off-by: Tianyu Li Reviewed-by: Matias Elo --- DEPENDENCIES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPENDENCIES b/DEPENDENCIES index 34a6a0aa6..816639bee 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -139,7 +139,7 @@ Prerequisites for building the OpenDataPlane (ODP) API # Checkout and build Arm code $ git clone https://git.gitlab.arm.com/arm-reference-solutions/ipsec-mb.git $ cd ipsec-mb/ - $ git checkout SECLIB-IPSEC-2022.05.25 + $ git checkout SECLIB-IPSEC-2022.12.13 $ make $ sudo make install -- cgit v1.2.3 From 6d7df9b105263c8d4f10acc62ea1e2b768b6e17e Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Mon, 19 Dec 2022 10:45:47 +0200 Subject: validation: system: relax CPU cycle count resolution tests CPU cycle count may advance in steps that are not integer values. For example, implementation may measure time and convert it to CPU cycles, and multiplier between time units and CPU frequency may have decimals. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/validation/api/system/system.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/validation/api/system/system.c b/test/validation/api/system/system.c index f0cf48738..a98ebb634 100644 --- a/test/validation/api/system/system.c +++ b/test/validation/api/system/system.c @@ -135,6 +135,7 @@ static void system_test_cpu_cycles_resolution(void) int i; uint64_t res; uint64_t c2, c1, max; + uint64_t test_cycles = odp_cpu_hz() / 100; /* CPU cycles in 10 msec */ max = odp_cpu_cycles_max(); @@ -144,11 +145,16 @@ static void system_test_cpu_cycles_resolution(void) for (i = 0; i < RES_TRY_NUM; i++) { c1 = odp_cpu_cycles(); - odp_time_wait_ns(100 * ODP_TIME_MSEC_IN_NS + i); + odp_time_wait_ns(10 * ODP_TIME_MSEC_IN_NS + i); c2 = odp_cpu_cycles(); - CU_ASSERT(c1 % res == 0); - CU_ASSERT(c2 % res == 0); + /* Diff may be zero with low resolution */ + if (test_cycles && test_cycles > res) { + uint64_t diff = odp_cpu_cycles_diff(c2, c1); + + CU_ASSERT(diff >= res); + } + } } @@ -185,13 +191,11 @@ static void system_test_cpu_cycles_diff(void) tmp = c2 + (max - c1) + res; diff = odp_cpu_cycles_diff(c2, c1); CU_ASSERT(diff == tmp); - CU_ASSERT(diff % res == 0); /* no wrap, revert args */ tmp = c1 - c2; diff = odp_cpu_cycles_diff(c1, c2); CU_ASSERT(diff == tmp); - CU_ASSERT(diff % res == 0); } static void system_test_cpu_cycles_long_period(void) @@ -221,8 +225,6 @@ static void system_test_cpu_cycles_long_period(void) c2 = odp_cpu_cycles(); CU_ASSERT(c2 != c1); - CU_ASSERT(c1 % res == 0); - CU_ASSERT(c2 % res == 0); CU_ASSERT(c1 <= max && c2 <= max); if (c2 > c1) @@ -232,7 +234,6 @@ static void system_test_cpu_cycles_long_period(void) diff = odp_cpu_cycles_diff(c2, c1); CU_ASSERT(diff == tmp); - CU_ASSERT(diff % res == 0); /* wrap is detected and verified */ if (c2 < c1) -- cgit v1.2.3 From 8936a2f35dbbbbdbb8e7a694ab99161cacc93adf Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Tue, 20 Dec 2022 08:10:39 +0000 Subject: linux-gen: pktio: fix input queue count with classifier When classifier is in use, only one packet I/O input queue should be configured. Currently, one packet I/O level input queue is configured but driver level input queue count will be whatever is passed via `odp_pktin_queue_param_t::num_queues` in `odp_pktin_queue_config()`. Fix this by always overriding the parameter to correct value when applicable. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- platform/linux-generic/odp_packet_io.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 22b6bc916..663b31759 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -2166,8 +2166,7 @@ void odp_pktio_extra_stats_print(odp_pktio_t pktio) _ODP_PRINT("\n"); } -int odp_pktin_queue_config(odp_pktio_t pktio, - const odp_pktin_queue_param_t *param) +int odp_pktin_queue_config(odp_pktio_t pktio, const odp_pktin_queue_param_t *param) { pktio_entry_t *entry; odp_pktin_mode_t mode; @@ -2175,7 +2174,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio, uint32_t num_queues, i; int rc; odp_queue_t queue; - odp_pktin_queue_param_t default_param; + odp_pktin_queue_param_t default_param, local_param; if (param == NULL) { odp_pktin_queue_param_init(&default_param); @@ -2204,7 +2203,19 @@ int odp_pktin_queue_config(odp_pktio_t pktio, return -1; } - num_queues = param->classifier_enable ? 1 : param->num_queues; + if (param->classifier_enable) { + num_queues = 1; + + if (param->num_queues != num_queues) { + /* When classifier is enabled, ensure that only one input queue will be + * configured by driver. */ + memcpy(&local_param, param, sizeof(odp_pktin_queue_param_t)); + local_param.num_queues = num_queues; + param = &local_param; + } + } else { + num_queues = param->num_queues; + } rc = odp_pktio_capability(pktio, &capa); if (rc) { -- cgit v1.2.3 From 7f850ba0a85e94094cd4226dee6bbd1ec7798398 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 20 Dec 2022 17:49:17 +0200 Subject: linux-gen: crypto: arm: use macro for the constant AES-GCM IV length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The only supported IV length for AES-GCM is 12 bytes. Define it as a macro to benefit from constant expressions and to get rid of some magic numbers. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- .../linux-generic/arch/aarch64/odp_crypto_armv8.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index 11fadc971..df9d6442f 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -38,6 +38,10 @@ #define ARM_CRYPTO_MAX_DATA_LENGTH 65536 #define ARM_CRYPTO_MAX_DIGEST_LENGTH 16 +#define AES_GCM_IV_LEN 12 +ODP_STATIC_ASSERT(AES_GCM_IV_LEN <= ARM_CRYPTO_MAX_IV_LENGTH, + "AES_GCM_IV_LEN exceeds ARM_CRYPTO_MAX_IV_LENGTH"); + /* * ARM crypto library may read up to 15 bytes past the end of input * data and AAD and write up to 15 bytes past the end of output data. @@ -70,9 +74,9 @@ static const odp_crypto_cipher_capability_t cipher_capa_null[] = { #ifdef __ARM_FEATURE_AES static const odp_crypto_cipher_capability_t cipher_capa_aes_gcm[] = { -{.key_len = 16, .iv_len = 12}, -{.key_len = 24, .iv_len = 12}, -{.key_len = 32, .iv_len = 12} }; +{.key_len = 16, .iv_len = AES_GCM_IV_LEN}, +{.key_len = 24, .iv_len = AES_GCM_IV_LEN}, +{.key_len = 32, .iv_len = AES_GCM_IV_LEN} }; #endif /* @@ -236,7 +240,7 @@ void aes_gcm_encrypt(odp_packet_t pkt, } }; uint8_t *iv_ptr; - uint64_t iv_bit_length = session->p.cipher_iv_len * 8; + uint64_t iv_bit_length = AES_GCM_IV_LEN * 8; uint64_t plaintext_bit_length = param->cipher_range.length * 8; uint64_t aad_bit_length = session->p.auth_aad_len * 8; uint32_t in_pos = param->cipher_range.offset; @@ -261,7 +265,7 @@ void aes_gcm_encrypt(odp_packet_t pkt, goto err; #else iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(iv_ptr != NULL); #endif cs.constants = &session->cc; @@ -337,7 +341,7 @@ void aes_gcm_decrypt(odp_packet_t pkt, }; uint8_t *iv_ptr; uint8_t tag[AES_GCM_TAG_LEN]; - uint64_t iv_bit_length = session->p.cipher_iv_len * 8; + uint64_t iv_bit_length = AES_GCM_IV_LEN * 8; uint64_t plaintext_bit_length = param->cipher_range.length * 8; uint64_t aad_bit_length = session->p.auth_aad_len * 8; uint32_t in_pos = param->cipher_range.offset; @@ -361,7 +365,7 @@ void aes_gcm_decrypt(odp_packet_t pkt, goto err; #else iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); + _ODP_ASSERT(iv_ptr != NULL); #endif cs.constants = &session->cc; @@ -433,7 +437,7 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session) return -1; /* Verify IV len is correct */ - if (12 != session->p.cipher_iv_len) + if (session->p.cipher_iv_len != AES_GCM_IV_LEN) return -1; if (ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH < session->p.cipher_key.length) -- cgit v1.2.3 From 1898b096ac0251584f89ddb547b13ebdd4927f61 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 20 Dec 2022 17:49:22 +0200 Subject: linux-gen: crypto: arm: fix out-of-bounds IV read access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Arm crypto library may read up to 16 bytes of data when it is reading a 12 byte IV. If the IV provided by the caller ends at a page boundary and the next page is not mapped to readable memory, bad things can happen. Fix the problem by always copying the user provided IV to a bigger temporary buffer that is passed to the crypto library. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- platform/linux-generic/arch/aarch64/odp_crypto_armv8.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index df9d6442f..0a73d4868 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -239,6 +239,7 @@ void aes_gcm_encrypt(odp_packet_t pkt, .d = {0, 0} } }; + uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; uint8_t *iv_ptr; uint64_t iv_bit_length = AES_GCM_IV_LEN * 8; uint64_t plaintext_bit_length = param->cipher_range.length * 8; @@ -267,6 +268,9 @@ void aes_gcm_encrypt(odp_packet_t pkt, iv_ptr = param->cipher_iv_ptr; _ODP_ASSERT(iv_ptr != NULL); #endif + /* The crypto lib may read 16 bytes. Copy to a big enough buffer */ + memcpy(iv_data, iv_ptr, AES_GCM_IV_LEN); + iv_ptr = iv_data; cs.constants = &session->cc; @@ -339,6 +343,7 @@ void aes_gcm_decrypt(odp_packet_t pkt, .d = {0, 0} } }; + uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; uint8_t *iv_ptr; uint8_t tag[AES_GCM_TAG_LEN]; uint64_t iv_bit_length = AES_GCM_IV_LEN * 8; @@ -367,6 +372,9 @@ void aes_gcm_decrypt(odp_packet_t pkt, iv_ptr = param->cipher_iv_ptr; _ODP_ASSERT(iv_ptr != NULL); #endif + /* The crypto lib may read 16 bytes. Copy to a big enough buffer */ + memcpy(iv_data, iv_ptr, AES_GCM_IV_LEN); + iv_ptr = iv_data; cs.constants = &session->cc; -- cgit v1.2.3 From 7f42e20b0715ff3719ccfc2ecf9d09db51a03ca3 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 15 Dec 2022 16:00:13 +0200 Subject: linux-gen: sched: add config option for adjusting ordered queue stash size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add new configuration option 'sched_basic.order_stash_size', which can be used to adjust the size of implementation internal per thread ordered queue reorder stash. If 'order_stash_size' > 0, events may be dropped by the implementation if the target queue is full. To prevent this set 'order_stash_size' to 0. Signed-off-by: Matias Elo Reported-and-tested-by: Carl Wallen Reviewed-by: Jere Leppänen --- config/odp-linux-generic.conf | 11 ++++++++++- platform/linux-generic/m4/odp_libconfig.m4 | 2 +- platform/linux-generic/odp_schedule_basic.c | 22 ++++++++++++++++++---- platform/linux-generic/test/inline-timer.conf | 2 +- platform/linux-generic/test/packet_align.conf | 2 +- platform/linux-generic/test/process-mode.conf | 2 +- platform/linux-generic/test/sched-basic.conf | 3 ++- 7 files changed, 34 insertions(+), 10 deletions(-) diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index f8accd07f..814035d61 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.22" +config_file_version = "0.1.23" # System options system: { @@ -234,6 +234,15 @@ sched_basic: { worker = 1 control = 1 } + + # Ordered queue reorder stash size + # + # Number of events each thread can stash internally before having to + # wait for the right order context. Reorder stash can improve + # performance if threads process events in bursts. If 'order_stash_size' + # > 0, events may be dropped by the implementation if the target queue + # is full. To prevent this set 'order_stash_size' to 0. + order_stash_size = 512 } timer: { diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4 index 03dbc929d..658138c31 100644 --- a/platform/linux-generic/m4/odp_libconfig.m4 +++ b/platform/linux-generic/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [22]) +m4_define([_odp_config_version_minor], [23]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 594360326..3fc9c1c17 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -238,13 +238,13 @@ typedef struct { struct { uint8_t burst_default[NUM_SCHED_SYNC][NUM_PRIO]; uint8_t burst_max[NUM_SCHED_SYNC][NUM_PRIO]; + uint16_t order_stash_size; uint8_t num_spread; uint8_t prefer_ratio; } config; - - uint8_t load_balance; - uint16_t max_spread; uint32_t ring_mask; + uint16_t max_spread; + uint8_t load_balance; odp_atomic_u32_t grp_epoch; odp_shm_t shm; odp_ticketlock_t mask_lock[NUM_SCHED_GRPS]; @@ -437,6 +437,20 @@ static int read_config_file(sched_global_t *sched) if (val == 0 || sched->config.num_spread == 1) sched->load_balance = 0; + str = "sched_basic.order_stash_size"; + if (!_odp_libconfig_lookup_int(str, &val)) { + _ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + if (val > MAX_ORDERED_STASH || val < 0) { + _ODP_ERR("Bad value %s = %i [min: 0, max: %u]\n", str, val, MAX_ORDERED_STASH); + return -1; + } + + sched->config.order_stash_size = val; + _ODP_PRINT(" %s: %i\n", str, val); + /* Initialize default values for all queue types */ str = "sched_basic.burst_size_default"; if (read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_ATOMIC], str, 1, @@ -1204,7 +1218,7 @@ static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *event_hdr[], /* Pktout may drop packets, so the operation cannot be stashed. */ if (dst_qentry->pktout.pktio != ODP_PKTIO_INVALID || - odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { + odp_unlikely(stash_num >= sched->config.order_stash_size)) { /* If the local stash is full, wait until it is our turn and * then release the stash and do enqueue directly. */ wait_for_order(src_queue); diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index 261aa0141..5afcad2cb 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.22" +config_file_version = "0.1.23" timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf index 8d2d00e63..9aa6109a6 100644 --- a/platform/linux-generic/test/packet_align.conf +++ b/platform/linux-generic/test/packet_align.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.22" +config_file_version = "0.1.23" pool: { pkt: { diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index 1e0e7cc95..cf32fb86d 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.22" +config_file_version = "0.1.23" # Shared memory options shm: { diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf index e63ffa2f3..5710c43c4 100644 --- a/platform/linux-generic/test/sched-basic.conf +++ b/platform/linux-generic/test/sched-basic.conf @@ -1,9 +1,10 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.22" +config_file_version = "0.1.23" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { prio_spread = 3 load_balance = 0 + order_stash_size = 0 } -- cgit v1.2.3 From 9eccdaa332be8aa81331c352a59cf874ef920f59 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Tue, 20 Dec 2022 17:32:34 +0200 Subject: test: bench_buffer: fix test round counting Fix round counter reset in packet and buffer bench test applications. It counts from 1 to N. Signed-off-by: Petri Savolainen Reviewed-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_bench_buffer.c | 4 ++-- test/performance/odp_bench_packet.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/performance/odp_bench_buffer.c b/test/performance/odp_bench_buffer.c index e7029f57d..e0e7c85cd 100644 --- a/test/performance/odp_bench_buffer.c +++ b/test/performance/odp_bench_buffer.c @@ -199,7 +199,7 @@ static int run_benchmarks(void *arg) /* Skip unsupported tests */ if (args->bench[j].cond != NULL && !args->bench[j].cond()) { j++; - k = 0; + k = 1; if (i > 0) printf("[%02d] odp_%-26s: n/a\n", j, desc); continue; @@ -236,7 +236,7 @@ static int run_benchmarks(void *arg) printf("[%02d] odp_%-26s: %8.1f\n", j + 1, desc, cycles); j++; - k = 0; + k = 1; tot_cycles = 0; } } diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c index b44e92b72..aaf725b57 100644 --- a/test/performance/odp_bench_packet.c +++ b/test/performance/odp_bench_packet.c @@ -275,7 +275,7 @@ static int run_benchmarks(void *arg) printf("odp_%-26s: %8.1f\n", desc, cycles); j++; - k = 0; + k = 1; tot_cycles = 0; } } -- cgit v1.2.3 From eaab512f1f8df3d9615e2b0862b0c10ede4722e8 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Mon, 19 Dec 2022 12:59:55 +0200 Subject: test: bench_misc: add new perf test for misc API calls Test overhead of miscellaneous set of API functions. For example, time and CPU cycle count APIs may be called often in fast path. Signed-off-by: Petri Savolainen Reviewed-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/.gitignore | 1 + test/performance/Makefile.am | 2 + test/performance/odp_bench_misc.c | 1069 +++++++++++++++++++++++++++++++++++++ 3 files changed, 1072 insertions(+) create mode 100644 test/performance/odp_bench_misc.c diff --git a/test/performance/.gitignore b/test/performance/.gitignore index ec1915bba..b86699c91 100644 --- a/test/performance/.gitignore +++ b/test/performance/.gitignore @@ -3,6 +3,7 @@ odp_atomic odp_atomic_perf odp_bench_buffer +odp_bench_misc odp_bench_packet odp_cpu_bench odp_crc diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am index 2a749ae38..9644b8cb8 100644 --- a/test/performance/Makefile.am +++ b/test/performance/Makefile.am @@ -4,6 +4,7 @@ TESTS_ENVIRONMENT += TEST_DIR=${builddir} EXECUTABLES = odp_atomic_perf \ odp_bench_buffer \ + odp_bench_misc \ odp_bench_packet \ odp_crc \ odp_lock_perf \ @@ -54,6 +55,7 @@ bin_PROGRAMS = $(EXECUTABLES) $(COMPILE_ONLY) odp_atomic_perf_SOURCES = odp_atomic_perf.c odp_bench_buffer_SOURCES = odp_bench_buffer.c +odp_bench_misc_SOURCES = odp_bench_misc.c odp_bench_packet_SOURCES = odp_bench_packet.c odp_cpu_bench_SOURCES = odp_cpu_bench.c odp_crc_SOURCES = odp_crc.c diff --git a/test/performance/odp_bench_misc.c b/test/performance/odp_bench_misc.c new file mode 100644 index 000000000..8b9e27bf2 --- /dev/null +++ b/test/performance/odp_bench_misc.c @@ -0,0 +1,1069 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* Needed for sigaction */ +#endif + +#include +#include + +#include +#include +#include +#include +#include + +/* Number of API function calls per test case */ +#define REPEAT_COUNT 1000 + +/* Default number of rounds per test case */ +#define ROUNDS 1000u + +#define BENCH_INFO(run, init, max, name) \ + {#run, run, init, max, name} + +typedef struct { + /* Measure time vs CPU cycles */ + int time; + + /* Benchmark index to run indefinitely */ + int bench_idx; + + /* Rounds per test case */ + uint32_t rounds; + +} appl_args_t; + +/* Initialize benchmark resources */ +typedef void (*bench_init_fn_t)(void); + +/* Run benchmark, returns >0 on success */ +typedef int (*bench_run_fn_t)(void); + +/* Benchmark data */ +typedef struct { + /* Default test name */ + const char *name; + + /* Test function to run */ + bench_run_fn_t run; + + /* Test init function */ + bench_init_fn_t init; + + /* Test specific limit for rounds (tuning for slow implementation) */ + uint32_t max_rounds; + + /* Override default test name */ + const char *desc; + +} bench_info_t; + +/* Global data */ +typedef struct { + appl_args_t appl; + + /* Benchmark functions */ + bench_info_t *bench; + + /* Number of benchmark functions */ + int num_bench; + + /* Break worker loop if set to 1 */ + odp_atomic_u32_t exit_thread; + + /* Test case input / output data */ + odp_time_t t1[REPEAT_COUNT]; + odp_time_t t2[REPEAT_COUNT]; + odp_time_t t3[REPEAT_COUNT]; + uint64_t a1[REPEAT_COUNT]; + uint64_t a2[REPEAT_COUNT]; + uint32_t b1[REPEAT_COUNT]; + uint32_t b2[REPEAT_COUNT]; + uint16_t c1[REPEAT_COUNT]; + uint16_t c2[REPEAT_COUNT]; + + /* Dummy result */ + uint64_t dummy; + + /* Benchmark run failed */ + int bench_failed; + + /* CPU mask as string */ + char cpumask_str[ODP_CPUMASK_STR_SIZE]; + +} gbl_args_t; + +static gbl_args_t *gbl_args; + +static void sig_handler(int signo ODP_UNUSED) +{ + if (gbl_args == NULL) + return; + odp_atomic_store_u32(&gbl_args->exit_thread, 1); +} + +static int setup_sig_handler(void) +{ + struct sigaction action; + + memset(&action, 0, sizeof(action)); + action.sa_handler = sig_handler; + + /* No additional signals blocked. By default, the signal which triggered + * the handler is blocked. */ + if (sigemptyset(&action.sa_mask)) + return -1; + + if (sigaction(SIGINT, &action, NULL)) + return -1; + + return 0; +} + +/* Run given benchmark indefinitely */ +static void run_indef(gbl_args_t *args, int idx) +{ + const char *desc; + const bench_info_t *bench = &args->bench[idx]; + + desc = bench->desc != NULL ? bench->desc : bench->name; + + printf("Running odp_%s test indefinitely\n", desc); + + while (!odp_atomic_load_u32(&gbl_args->exit_thread)) { + int ret; + + if (bench->init != NULL) + bench->init(); + + ret = bench->run(); + + if (!ret) + ODPH_ABORT("Benchmark %s failed\n", desc); + } +} + +static int run_benchmarks(void *arg) +{ + int i, j; + uint64_t c1, c2; + odp_time_t t1, t2; + gbl_args_t *args = arg; + const int meas_time = args->appl.time; + + printf("\nAverage %s per function call\n", meas_time ? "time (nsec)" : "CPU cycles"); + printf("-------------------------------------------------\n"); + + /* Run each test twice. Results from the first warm-up round are ignored. */ + for (i = 0; i < 2; i++) { + uint64_t total = 0; + uint32_t round = 1; + + for (j = 0; j < gbl_args->num_bench; round++) { + int ret; + const char *desc; + const bench_info_t *bench = &args->bench[j]; + uint32_t max_rounds = args->appl.rounds; + + if (bench->max_rounds && max_rounds > bench->max_rounds) + max_rounds = bench->max_rounds; + + /* Run selected test indefinitely */ + if (args->appl.bench_idx) { + if ((j + 1) != args->appl.bench_idx) { + j++; + continue; + } + + run_indef(args, j); + return 0; + } + + desc = bench->desc != NULL ? bench->desc : bench->name; + + if (bench->init != NULL) + bench->init(); + + if (meas_time) + t1 = odp_time_local(); + else + c1 = odp_cpu_cycles(); + + ret = bench->run(); + + if (meas_time) + t2 = odp_time_local(); + else + c2 = odp_cpu_cycles(); + + if (!ret) { + ODPH_ERR("Benchmark odp_%s failed\n", desc); + args->bench_failed = -1; + return -1; + } + + if (meas_time) + total += odp_time_diff_ns(t2, t1); + else + total += odp_cpu_cycles_diff(c2, c1); + + if (round >= max_rounds) { + double result; + + /* Each benchmark runs internally REPEAT_COUNT times. */ + result = ((double)total) / (max_rounds * REPEAT_COUNT); + + /* No print from warm-up round */ + if (i > 0) + printf("[%02d] odp_%-26s: %12.2f\n", j + 1, desc, result); + + j++; + total = 0; + round = 1; + } + } + } + + /* Print dummy result to prevent compiler to optimize it away*/ + printf("\n(dummy result: 0x%" PRIx64 ")\n", args->dummy); + + printf("\n"); + + return 0; +} + +static void init_time_global(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + odp_time_t *t2 = gbl_args->t2; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_global(); + + for (i = 0; i < REPEAT_COUNT; i++) + t2[i] = odp_time_global(); + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_global_ns(); +} + +static void init_time_local(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + odp_time_t *t2 = gbl_args->t2; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_local(); + + for (i = 0; i < REPEAT_COUNT; i++) + t2[i] = odp_time_local(); + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_local_ns(); +} + +static void init_cpu_cycles(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + uint64_t *a2 = gbl_args->a2; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_cycles(); + + for (i = 0; i < REPEAT_COUNT; i++) + a2[i] = odp_cpu_cycles(); +} + +static int time_local(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_local(); + + return i; +} + +static int time_local_strict(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_local_strict(); + + return i; +} + +static int time_local_ns(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_local_ns(); + + return i; +} + +static int time_local_strict_ns(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_local_strict_ns(); + + return i; +} + +static int time_global(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_global(); + + return i; +} + +static int time_global_strict(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_global_strict(); + + return i; +} + +static int time_global_ns(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_global_ns(); + + return i; +} + +static int time_global_strict_ns(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_global_strict_ns(); + + return i; +} + +static int time_diff(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + odp_time_t *t2 = gbl_args->t2; + odp_time_t *t3 = gbl_args->t3; + + for (i = 0; i < REPEAT_COUNT; i++) + t3[i] = odp_time_diff(t2[i], t1[i]); + + return i; +} + +static int time_diff_ns(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + odp_time_t *t2 = gbl_args->t2; + uint64_t res = 0; + + for (i = 0; i < REPEAT_COUNT; i++) + res += odp_time_diff_ns(t2[i], t1[i]); + + gbl_args->dummy += res; + + return i; +} + +static int time_sum(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + odp_time_t *t2 = gbl_args->t2; + odp_time_t *t3 = gbl_args->t3; + + for (i = 0; i < REPEAT_COUNT; i++) + t3[i] = odp_time_sum(t1[i], t2[i]); + + return i; +} + +static int time_to_ns(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + uint64_t res = 0; + + for (i = 0; i < REPEAT_COUNT; i++) + res += odp_time_to_ns(t1[i]); + + gbl_args->dummy += res; + + return i; +} + +static int time_local_from_ns(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_local_from_ns(a1[i]); + + return i; +} + +static int time_global_from_ns(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + t1[i] = odp_time_global_from_ns(a1[i]); + + return i; +} + +static int time_cmp(void) +{ + int i; + odp_time_t *t1 = gbl_args->t1; + odp_time_t *t2 = gbl_args->t2; + int res = 0; + + for (i = 0; i < REPEAT_COUNT; i++) + res += odp_time_cmp(t1[i], t2[i]); + + gbl_args->dummy += res; + + return i; +} + +static int time_local_res(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_local_res(); + + return i; +} + +static int time_global_res(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_time_global_res(); + + return i; +} + +static int cpu_id(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_id(); + + return i; +} + +static int cpu_count(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_count(); + + return i; +} + +static int cpu_hz(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_hz(); + + return i; +} + +static int cpu_hz_max(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_hz_max(); + + return i; +} + +static int cpu_cycles(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_cycles(); + + return i; +} + +static int cpu_cycles_diff(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + uint64_t *a2 = gbl_args->a2; + uint64_t res = 0; + + for (i = 0; i < REPEAT_COUNT; i++) + res += odp_cpu_cycles_diff(a2[i], a1[i]); + + gbl_args->dummy += res; + + return i; +} + +static int cpu_cycles_max(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_cpu_cycles_max(); + + return i; +} + +static int cpu_pause(void) +{ + int i; + + for (i = 0; i < REPEAT_COUNT; i++) + odp_cpu_pause(); + + return i; +} + +static int thread_id(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_thread_id(); + + return i; +} + +static int thread_count(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_thread_count(); + + return i; +} + +static int thread_count_max(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = odp_thread_count_max(); + + return i; +} + +static int thread_type(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + + for (i = 0; i < REPEAT_COUNT; i++) + a1[i] = (int)odp_thread_type(); + + return i; +} + +static int be_to_cpu_64(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + uint64_t *a2 = gbl_args->a2; + + for (i = 0; i < REPEAT_COUNT; i++) + a2[i] = odp_be_to_cpu_64(a1[i]); + + return i; +} + +static int be_to_cpu_32(void) +{ + int i; + uint32_t *b1 = gbl_args->b1; + uint32_t *b2 = gbl_args->b2; + + for (i = 0; i < REPEAT_COUNT; i++) + b2[i] = odp_be_to_cpu_32(b1[i]); + + return i; +} + +static int be_to_cpu_16(void) +{ + int i; + uint16_t *c1 = gbl_args->c1; + uint16_t *c2 = gbl_args->c2; + + for (i = 0; i < REPEAT_COUNT; i++) + c2[i] = odp_be_to_cpu_16(c1[i]); + + return i; +} + +static int cpu_to_be_64(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + uint64_t *a2 = gbl_args->a2; + + for (i = 0; i < REPEAT_COUNT; i++) + a2[i] = odp_cpu_to_be_64(a1[i]); + + return i; +} + +static int cpu_to_be_32(void) +{ + int i; + uint32_t *b1 = gbl_args->b1; + uint32_t *b2 = gbl_args->b2; + + for (i = 0; i < REPEAT_COUNT; i++) + b2[i] = odp_cpu_to_be_32(b1[i]); + + return i; +} + +static int cpu_to_be_16(void) +{ + int i; + uint16_t *c1 = gbl_args->c1; + uint16_t *c2 = gbl_args->c2; + + for (i = 0; i < REPEAT_COUNT; i++) + c2[i] = odp_cpu_to_be_16(c1[i]); + + return i; +} + +static int le_to_cpu_64(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + uint64_t *a2 = gbl_args->a2; + + for (i = 0; i < REPEAT_COUNT; i++) + a2[i] = odp_le_to_cpu_64(a1[i]); + + return i; +} + +static int le_to_cpu_32(void) +{ + int i; + uint32_t *b1 = gbl_args->b1; + uint32_t *b2 = gbl_args->b2; + + for (i = 0; i < REPEAT_COUNT; i++) + b2[i] = odp_le_to_cpu_32(b1[i]); + + return i; +} + +static int le_to_cpu_16(void) +{ + int i; + uint16_t *c1 = gbl_args->c1; + uint16_t *c2 = gbl_args->c2; + + for (i = 0; i < REPEAT_COUNT; i++) + c2[i] = odp_le_to_cpu_16(c1[i]); + + return i; +} + +static int cpu_to_le_64(void) +{ + int i; + uint64_t *a1 = gbl_args->a1; + uint64_t *a2 = gbl_args->a2; + + for (i = 0; i < REPEAT_COUNT; i++) + a2[i] = odp_cpu_to_le_64(a1[i]); + + return i; +} + +static int cpu_to_le_32(void) +{ + int i; + uint32_t *b1 = gbl_args->b1; + uint32_t *b2 = gbl_args->b2; + + for (i = 0; i < REPEAT_COUNT; i++) + b2[i] = odp_cpu_to_le_32(b1[i]); + + return i; +} + +static int cpu_to_le_16(void) +{ + int i; + uint16_t *c1 = gbl_args->c1; + uint16_t *c2 = gbl_args->c2; + + for (i = 0; i < REPEAT_COUNT; i++) + c2[i] = odp_cpu_to_le_16(c1[i]); + + return i; +} + +static int mb_release(void) +{ + int i; + + for (i = 0; i < REPEAT_COUNT; i++) + odp_mb_release(); + + return i; +} + +static int mb_acquire(void) +{ + int i; + + for (i = 0; i < REPEAT_COUNT; i++) + odp_mb_acquire(); + + return i; +} + +static int mb_full(void) +{ + int i; + + for (i = 0; i < REPEAT_COUNT; i++) + odp_mb_full(); + + return i; +} + +bench_info_t test_suite[] = { + BENCH_INFO(time_local, NULL, 0, NULL), + BENCH_INFO(time_local_strict, NULL, 0, NULL), + BENCH_INFO(time_local_ns, NULL, 0, NULL), + BENCH_INFO(time_local_strict_ns, NULL, 0, NULL), + BENCH_INFO(time_global, NULL, 0, NULL), + BENCH_INFO(time_global_strict, NULL, 0, NULL), + BENCH_INFO(time_global_ns, NULL, 0, NULL), + BENCH_INFO(time_global_strict_ns, NULL, 0, NULL), + BENCH_INFO(time_diff, init_time_global, 0, "time_diff (global)"), + BENCH_INFO(time_diff, init_time_local, 0, "time_diff (local)"), + BENCH_INFO(time_diff_ns, init_time_global, 0, NULL), + BENCH_INFO(time_sum, init_time_global, 0, NULL), + BENCH_INFO(time_to_ns, init_time_global, 0, NULL), + BENCH_INFO(time_local_from_ns, init_time_global, 0, NULL), + BENCH_INFO(time_global_from_ns, init_time_global, 0, NULL), + BENCH_INFO(time_cmp, init_time_global, 0, NULL), + BENCH_INFO(time_local_res, NULL, 0, NULL), + BENCH_INFO(time_global_res, NULL, 0, NULL), + BENCH_INFO(cpu_id, NULL, 0, NULL), + BENCH_INFO(cpu_count, NULL, 0, NULL), + BENCH_INFO(cpu_hz, NULL, 1, NULL), + BENCH_INFO(cpu_hz_max, NULL, 0, NULL), + BENCH_INFO(cpu_cycles, NULL, 0, NULL), + BENCH_INFO(cpu_cycles_diff, init_cpu_cycles, 0, NULL), + BENCH_INFO(cpu_cycles_max, NULL, 0, NULL), + BENCH_INFO(cpu_pause, NULL, 0, NULL), + BENCH_INFO(thread_id, NULL, 0, NULL), + BENCH_INFO(thread_count, NULL, 0, NULL), + BENCH_INFO(thread_count_max, NULL, 0, NULL), + BENCH_INFO(thread_type, NULL, 0, NULL), + BENCH_INFO(be_to_cpu_64, NULL, 0, NULL), + BENCH_INFO(be_to_cpu_32, NULL, 0, NULL), + BENCH_INFO(be_to_cpu_16, NULL, 0, NULL), + BENCH_INFO(cpu_to_be_64, NULL, 0, NULL), + BENCH_INFO(cpu_to_be_32, NULL, 0, NULL), + BENCH_INFO(cpu_to_be_16, NULL, 0, NULL), + BENCH_INFO(le_to_cpu_64, NULL, 0, NULL), + BENCH_INFO(le_to_cpu_32, NULL, 0, NULL), + BENCH_INFO(le_to_cpu_16, NULL, 0, NULL), + BENCH_INFO(cpu_to_le_64, NULL, 0, NULL), + BENCH_INFO(cpu_to_le_32, NULL, 0, NULL), + BENCH_INFO(cpu_to_le_16, NULL, 0, NULL), + BENCH_INFO(mb_release, NULL, 0, NULL), + BENCH_INFO(mb_acquire, NULL, 0, NULL), + BENCH_INFO(mb_full, NULL, 0, NULL), +}; + +/* Print usage information */ +static void usage(void) +{ + printf("\n" + "ODP miscellaneous API micro benchmarks\n" + "\n" + "Options:\n" + " -t, --time Time measurement. 0: measure CPU cycles (default), 1: measure time\n" + " -i, --index Benchmark index to run indefinitely.\n" + " -r, --rounds Run each test case 'num' times (default %u).\n" + " -h, --help Display help and exit.\n\n" + "\n", ROUNDS); +} + +/* Parse command line arguments */ +static int parse_args(int argc, char *argv[]) +{ + int opt; + int long_index; + appl_args_t *appl_args = &gbl_args->appl; + static const struct option longopts[] = { + {"time", required_argument, NULL, 't'}, + {"index", required_argument, NULL, 'i'}, + {"rounds", required_argument, NULL, 'r'}, + {"help", no_argument, NULL, 'h'}, + {NULL, 0, NULL, 0} + }; + + static const char *shortopts = "t:i:r:h"; + + appl_args->time = 0; /* Measure CPU cycles */ + appl_args->bench_idx = 0; /* Run all benchmarks */ + appl_args->rounds = ROUNDS; + + while (1) { + opt = getopt_long(argc, argv, shortopts, longopts, &long_index); + + if (opt == -1) + break; /* No more options */ + + switch (opt) { + case 't': + appl_args->time = atoi(optarg); + break; + case 'i': + appl_args->bench_idx = atoi(optarg); + break; + case 'r': + appl_args->rounds = atoi(optarg); + break; + case 'h': + usage(); + return 1; + default: + ODPH_ERR("Bad option. Use -h for help.\n"); + return -1; + } + } + + if (appl_args->rounds < 1) { + ODPH_ERR("Invalid test cycle repeat count: %u\n", appl_args->rounds); + return -1; + } + + if (appl_args->bench_idx < 0 || appl_args->bench_idx > gbl_args->num_bench) { + ODPH_ERR("Bad bench index %i\n", appl_args->bench_idx); + return -1; + } + + optind = 1; /* Reset 'extern optind' from the getopt lib */ + + return 0; +} + +/* Print system and application info */ +static void print_info(void) +{ + odp_sys_info_print(); + + printf("\n" + "odp_bench_misc options\n" + "----------------------\n"); + + printf("CPU mask: %s\n", gbl_args->cpumask_str); + printf("Measurement unit: %s\n", gbl_args->appl.time ? "nsec" : "CPU cycles"); + printf("Test rounds: %u\n", gbl_args->appl.rounds); + printf("\n"); +} + +int main(int argc, char *argv[]) +{ + odph_helper_options_t helper_options; + odph_thread_t worker_thread; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; + int cpu, i; + odp_shm_t shm; + odp_cpumask_t cpumask, default_mask; + odp_instance_t instance; + odp_init_t init_param; + int ret = 0; + + /* Let helper collect its own arguments (e.g. --odph_proc) */ + argc = odph_parse_options(argc, argv); + if (odph_options(&helper_options)) { + ODPH_ERR("Reading ODP helper options failed\n"); + exit(EXIT_FAILURE); + } + + odp_init_param_init(&init_param); + init_param.mem_model = helper_options.mem_model; + + /* Init ODP before calling anything else */ + if (odp_init_global(&instance, &init_param, NULL)) { + ODPH_ERR("Global init failed\n"); + exit(EXIT_FAILURE); + } + + /* Init this thread */ + if (odp_init_local(instance, ODP_THREAD_CONTROL)) { + ODPH_ERR("Local init failed\n"); + exit(EXIT_FAILURE); + } + + if (setup_sig_handler()) { + ODPH_ERR("Signal handler setup failed\n"); + exit(EXIT_FAILURE); + } + + /* Reserve memory for args from shared mem */ + shm = odp_shm_reserve("shm_args", sizeof(gbl_args_t), ODP_CACHE_LINE_SIZE, 0); + if (shm == ODP_SHM_INVALID) { + ODPH_ERR("Shared mem reserve failed\n"); + exit(EXIT_FAILURE); + } + + gbl_args = odp_shm_addr(shm); + if (gbl_args == NULL) { + ODPH_ERR("Shared mem alloc failed\n"); + exit(EXIT_FAILURE); + } + + memset(gbl_args, 0, sizeof(gbl_args_t)); + odp_atomic_init_u32(&gbl_args->exit_thread, 0); + + gbl_args->bench = test_suite; + gbl_args->num_bench = sizeof(test_suite) / sizeof(test_suite[0]); + + for (i = 0; i < REPEAT_COUNT; i++) { + gbl_args->t1[i] = ODP_TIME_NULL; + gbl_args->t2[i] = ODP_TIME_NULL; + gbl_args->t3[i] = ODP_TIME_NULL; + gbl_args->a1[i] = i; + gbl_args->a2[i] = i; + gbl_args->b1[i] = i; + gbl_args->b2[i] = i; + gbl_args->c1[i] = i; + gbl_args->c2[i] = i; + } + + /* Parse and store the application arguments */ + ret = parse_args(argc, argv); + if (ret) + goto exit; + + /* Get default worker cpumask */ + if (odp_cpumask_default_worker(&default_mask, 1) != 1) { + ODPH_ERR("Unable to allocate worker thread\n"); + ret = -1; + goto exit; + } + + (void)odp_cpumask_to_str(&default_mask, gbl_args->cpumask_str, + sizeof(gbl_args->cpumask_str)); + + print_info(); + + memset(&worker_thread, 0, sizeof(odph_thread_t)); + + /* Create worker thread */ + cpu = odp_cpumask_first(&default_mask); + + odp_cpumask_zero(&cpumask); + odp_cpumask_set(&cpumask, cpu); + + odph_thread_common_param_init(&thr_common); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + thr_common.share_param = 1; + + odph_thread_param_init(&thr_param); + thr_param.start = run_benchmarks; + thr_param.arg = gbl_args; + thr_param.thr_type = ODP_THREAD_WORKER; + + odph_thread_create(&worker_thread, &thr_common, &thr_param, 1); + + odph_thread_join(&worker_thread, 1); + + ret = gbl_args->bench_failed; + +exit: + if (odp_shm_free(shm)) { + ODPH_ERR("Shared mem free failed\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_local()) { + ODPH_ERR("Local term failed\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_global(instance)) { + ODPH_ERR("Global term failed\n"); + exit(EXIT_FAILURE); + } + + if (ret < 0) + return EXIT_FAILURE; + + return EXIT_SUCCESS; +} -- cgit v1.2.3 From e54e66f7f221eed8d0e97114bb86bafad28a655a Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 19 Dec 2022 16:17:10 +0200 Subject: example: timer_accuracy: remove deprecated odp_timer_set_abs() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop using deprecated odp_timer_set_abs() function. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- example/timer/odp_timer_accuracy.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/example/timer/odp_timer_accuracy.c b/example/timer/odp_timer_accuracy.c index 84031b9e9..0952320bb 100644 --- a/example/timer/odp_timer_accuracy.c +++ b/example/timer/odp_timer_accuracy.c @@ -233,7 +233,7 @@ static int start_timers(test_global_t *test_global) odp_timer_t timer; odp_queue_t queue; odp_queue_param_t queue_param; - uint64_t tick, start_tick; + uint64_t start_tick; uint64_t period_ns, res_ns, res_hz, start_ns, nsec, offset_ns; uint64_t max_res_ns, max_res_hz; odp_event_t event; @@ -452,17 +452,17 @@ static int start_timers(test_global_t *test_global) for (i = 0; i < num_tmo; i++) { for (j = 0; j < burst; j++) { - nsec = offset_ns + (i * period_ns) + (j * burst_gap); - tick = start_tick + odp_timer_ns_to_tick(timer_pool, - nsec); - timer_ctx_t *ctx = &test_global->timer_ctx[idx]; + odp_timer_start_t start_param; - timer = ctx->timer; - event = ctx->event; + nsec = offset_ns + (i * period_ns) + (j * burst_gap); ctx->nsec = start_ns + nsec; - ret = odp_timer_set_abs(timer, tick, &event); + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = start_tick + odp_timer_ns_to_tick(timer_pool, nsec); + start_param.tmo_ev = ctx->event; + + ret = odp_timer_start(ctx->timer, &start_param); if (ret != ODP_TIMER_SUCCESS) { printf("Timer[%" PRIu64 "] set failed: %i\n", @@ -647,6 +647,7 @@ static void run_test(test_global_t *test_global) odp_timer_pool_t tp = test_global->timer_pool; unsigned int retries = test_global->opt.early_retry; uint64_t start_ns = test_global->start_ns; + odp_timer_start_t start_param; tim = ctx->timer; @@ -659,18 +660,20 @@ static void run_test(test_global_t *test_global) nsec = ctx->nsec - start_ns; tick = test_global->start_tick + odp_timer_ns_to_tick(tp, nsec); - - ret = odp_timer_set_abs(tim, tick, &ev); + start_param.tick_type = ODP_TIMER_TICK_ABS; } else { /* Relative time */ tick = test_global->period_tick; time = odp_time_local(); time_ns = odp_time_to_ns(time); ctx->nsec = time_ns + period_ns; - - ret = odp_timer_set_rel(tim, tick, &ev); + start_param.tick_type = ODP_TIMER_TICK_REL; } + start_param.tmo_ev = ev; + start_param.tick = tick; + + ret = odp_timer_start(tim, &start_param); if (ret == ODP_TIMER_TOO_NEAR) stat->num_too_near++; else -- cgit v1.2.3 From 7bc0f147350cf07eb8fd6b70a31955055436a514 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 19 Dec 2022 16:33:41 +0200 Subject: example: timer_simple: remove deprecated odp_timer_set_abs() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop using deprecated odp_timer_set_abs() function. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- example/timer/odp_timer_simple.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/example/timer/odp_timer_simple.c b/example/timer/odp_timer_simple.c index f44977476..4e3120653 100644 --- a/example/timer/odp_timer_simple.c +++ b/example/timer/odp_timer_simple.c @@ -129,13 +129,19 @@ int main(int argc ODP_UNUSED, char *argv[] ODP_UNUSED) for (i = 0; i < 5; i++) { odp_time_t time; + odp_timer_start_t start_param; /* Program timeout action on current tick + period */ tick = odp_timer_current_tick(timer_pool); - rc = odp_timer_set_abs(tim, tick + period, &ev); + + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = tick + period; + start_param.tmo_ev = ev; + + rc = odp_timer_start(tim, &start_param); /* Too early or too late timeout requested */ if (odp_unlikely(rc != ODP_TIMER_SUCCESS)) - ODPH_ABORT("odp_timer_set_abs() failed: %d\n", rc); + ODPH_ABORT("odp_timer_start() failed: %d\n", rc); /* Wait for 2 seconds for timeout action to be generated */ ev = odp_schedule(&queue, sched_tmo); -- cgit v1.2.3 From 9f55955a1d3c3f4c7476318749f0cbea23247657 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 19 Dec 2022 16:33:56 +0200 Subject: example: timer_test: remove deprecated odp_timer_set_abs() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop using deprecated odp_timer_set_abs() function. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- example/timer/odp_timer_test.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/example/timer/odp_timer_test.c b/example/timer/odp_timer_test.c index 277bbf6ae..f298c9fe8 100644 --- a/example/timer/odp_timer_test.c +++ b/example/timer/odp_timer_test.c @@ -121,14 +121,19 @@ static void test_abs_timeouts(int thr, test_globals_t *gbls) int wait = 0; odp_event_t ev; odp_timer_set_t rc; + odp_timer_start_t start_param; if (ttp) { tick += period; - rc = odp_timer_set_abs(ttp->tim, tick, &ttp->ev); + + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = tick; + start_param.tmo_ev = ttp->ev; + + rc = odp_timer_start(ttp->tim, &start_param); if (odp_unlikely(rc != ODP_TIMER_SUCCESS)) { /* Too early or too late timeout requested */ - ODPH_ABORT("odp_timer_set_abs() failed: %s\n", - timerset2str(rc)); + ODPH_ABORT("odp_timer_start() failed: %s\n", timerset2str(rc)); } } -- cgit v1.2.3 From 402a11745b270aa64a24d7f4338d491c7bd25bbf Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 19 Dec 2022 16:38:17 +0200 Subject: test: timer_perf: remove deprecated odp_timer_set_abs() function usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop using deprecated odp_timer_set_abs() function. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- test/performance/odp_timer_perf.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/performance/odp_timer_perf.c b/test/performance/odp_timer_perf.c index 998db628e..b09f9f60d 100644 --- a/test/performance/odp_timer_perf.c +++ b/test/performance/odp_timer_perf.c @@ -438,6 +438,7 @@ static int set_timers(test_global_t *global) odp_event_t ev; int status; timer_ctx_t *ctx = &global->timer_ctx[i][j]; + odp_timer_start_t start_param; /* Set timers backwards, the last timer is set first */ if (j == 0) @@ -456,9 +457,11 @@ static int set_timers(test_global_t *global) tick_ns = odp_timer_ns_to_tick(tp, nsec); nsec = nsec - period_ns; - status = odp_timer_set_abs(timer, tick_cur + tick_ns, - &ev); + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = tick_cur + tick_ns; + start_param.tmo_ev = ev; + status = odp_timer_start(timer, &start_param); if (status != ODP_TIMER_SUCCESS) { ODPH_ERR("Timer set %i/%i (ret %i)\n", i, j, status); return -1; @@ -656,6 +659,7 @@ static int set_cancel_mode_worker(void *arg) odp_timer_t timer; odp_timer_pool_t tp; odp_timeout_t tmo; + odp_timer_start_t start_param; thread_arg_t *thread_arg = arg; test_global_t *global = thread_arg->global; test_options_t *test_options = &global->test_options; @@ -746,7 +750,11 @@ static int set_cancel_mode_worker(void *arg) if (status < 0) continue; - status = odp_timer_set_abs(timer, tick + j * period_tick, &ev); + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = tick + j * period_tick; + start_param.tmo_ev = ev; + + status = odp_timer_start(timer, &start_param); num_set++; if (status != ODP_TIMER_SUCCESS) { -- cgit v1.2.3 From 6f11a982fec155ac069caabd98d5d3c9e9dcc646 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 20 Dec 2022 08:17:18 +0200 Subject: example: debug: remove deprecated odp_timer_set_rel() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop using deprecated odp_timer_set_rel() function. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- example/debug/odp_debug.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/example/debug/odp_debug.c b/example/debug/odp_debug.c index 6b4ea977f..6683e46b4 100644 --- a/example/debug/odp_debug.c +++ b/example/debug/odp_debug.c @@ -372,6 +372,7 @@ static int timer_debug(void) odp_timer_capability_t timer_capa; odp_timer_pool_t timer_pool; odp_timer_pool_param_t timer_param; + odp_timer_start_t start_param; odp_timer_t timer; odp_queue_t queue; odp_queue_param_t queue_param; @@ -452,8 +453,13 @@ static int timer_debug(void) odp_timeout_print(timeout); event = odp_timeout_to_event(timeout); - if (odp_timer_set_rel(timer, tick, &event) != ODP_TIMER_SUCCESS) - ODPH_ERR("Timer set failed.\n"); + + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = tick; + start_param.tmo_ev = event; + + if (odp_timer_start(timer, &start_param) != ODP_TIMER_SUCCESS) + ODPH_ERR("Timer start failed.\n"); printf("\n"); odp_timer_print(timer); -- cgit v1.2.3 From 795aea90c07b14dd4b502fa89ffea252e92fb058 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 20 Dec 2022 10:12:26 +0200 Subject: test: sched_pktio: remove deprecated odp_timer_set_rel() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop using deprecated odp_timer_set_rel() function. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- test/performance/odp_sched_pktio.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index 11be9d367..1553b02b3 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -425,6 +425,7 @@ static int worker_thread_timers(void *arg) pktin_queue_context_t *queue_context; odp_timer_t timer; odp_timer_set_t ret; + odp_timer_start_t start_param; worker_arg_t *worker_arg = arg; test_global_t *test_global = worker_arg->test_global_ptr; int worker_id = worker_arg->worker_id; @@ -460,12 +461,17 @@ static int worker_thread_timers(void *arg) src_pktio = queue_context->src_pktio; src_queue = queue_context->src_queue; timer = test_global->timer.timer[src_pktio][src_queue]; + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = tick; + start_param.tmo_ev = ODP_EVENT_INVALID; for (i = 0; i < num; i++) { if (odp_unlikely(odp_event_type(ev[i]) == ODP_EVENT_TIMEOUT)) { tmos++; - ret = odp_timer_set_rel(timer, tick, &ev[i]); + + start_param.tmo_ev = ev[i]; + ret = odp_timer_start(timer, &start_param); if (odp_unlikely(ret != ODP_TIMER_SUCCESS)) { /* Should never happen. Timeout event @@ -487,7 +493,7 @@ static int worker_thread_timers(void *arg) if (tmos == 0) { /* Reset timer with existing timeout event */ - ret = odp_timer_set_rel(timer, tick, NULL); + ret = odp_timer_restart(timer, &start_param); if (odp_unlikely(ret != ODP_TIMER_SUCCESS && ret != ODP_TIMER_FAIL)) { @@ -1318,10 +1324,10 @@ static int create_timers(test_global_t *test_global) static int start_timers(test_global_t *test_global) { int i, j; - odp_event_t event; odp_timeout_t timeout; odp_timer_t timer; odp_timer_set_t ret; + odp_timer_start_t start_param; uint64_t timeout_tick = test_global->timer.timeout_tick; int num_pktio = test_global->opt.num_pktio; int num_queue = test_global->opt.num_pktio_queue; @@ -1335,6 +1341,9 @@ static int start_timers(test_global_t *test_global) timeout_tick += odp_timer_ns_to_tick(test_global->timer.timer_pool, TIMEOUT_OFFSET_NS); + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = timeout_tick; + for (i = 0; i < num_pktio; i++) { for (j = 0; j < num_queue; j++) { timer = test_global->timer.timer[i][j]; @@ -1345,10 +1354,9 @@ static int start_timers(test_global_t *test_global) return -1; } - event = odp_timeout_to_event(timeout); - - ret = odp_timer_set_rel(timer, timeout_tick, &event); + start_param.tmo_ev = odp_timeout_to_event(timeout); + ret = odp_timer_start(timer, &start_param); if (ret != ODP_TIMER_SUCCESS) { printf("Timer set failed\n"); return -1; -- cgit v1.2.3 From 4a04a0250ea945e2a810db06e934ceab5c1adaef Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 20 Dec 2022 10:18:47 +0200 Subject: test: timer_perf: remove deprecated odp_timer_set_rel() function usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop using deprecated odp_timer_set_rel() function. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- test/performance/odp_timer_perf.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/performance/odp_timer_perf.c b/test/performance/odp_timer_perf.c index b09f9f60d..ccea30a14 100644 --- a/test/performance/odp_timer_perf.c +++ b/test/performance/odp_timer_perf.c @@ -698,7 +698,11 @@ static int set_cancel_mode_worker(void *arg) period_tick = global->timer_pool[i].period_tick; tick = start_tick + j * period_tick; - status = odp_timer_set_rel(timer, tick, &ev); + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = tick; + start_param.tmo_ev = ev; + + status = odp_timer_start(timer, &start_param); num_tmo++; num_set++; -- cgit v1.2.3 From 987af82ac81d4811444221fe38a0f5eff798c608 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 19 Dec 2022 16:48:43 +0200 Subject: validation: timer: remove deprecated odp_timer_set_abs() function usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop using deprecated odp_timer_set_abs() function. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- test/validation/api/timer/timer.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index 1c0a16349..071e710fa 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -1332,6 +1332,7 @@ static void timer_test_queue_type(odp_queue_type_t queue_type, int priv, int exp odp_queue_param_t queue_param; odp_timer_pool_param_t tparam; odp_timer_pool_t tp; + odp_timer_start_t start_param; odp_queue_t queue; odp_timer_t tim; int i, ret, num_tmo; @@ -1411,7 +1412,12 @@ static void timer_test_queue_type(odp_queue_type_t queue_type, int priv, int exp timer[i] = tim; tick = tick_base + ((i + 1) * period_tick); - ret = odp_timer_set_abs(tim, tick, &ev); + + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = tick; + start_param.tmo_ev = ev; + + ret = odp_timer_start(tim, &start_param); target_tick[i] = tick; target_nsec[i] = nsec_base + ((i + 1) * period_ns); @@ -1943,6 +1949,7 @@ static int worker_entrypoint(void *arg ODP_UNUSED) struct timespec ts; uint32_t nstale; odp_timer_set_t timer_rc; + odp_timer_start_t start_param; odp_timer_pool_t tp = global_mem->tp; odp_pool_t tbp = global_mem->tbp; uint32_t num_timers = global_mem->timers_per_thread; @@ -2008,7 +2015,13 @@ static int worker_entrypoint(void *arg ODP_UNUSED) (rand_r(&seed) % RANGE_MS) * 1000000ULL; tck = odp_timer_current_tick(tp) + odp_timer_ns_to_tick(tp, nsec); - timer_rc = odp_timer_set_abs(tt[i].tim, tck, &tt[i].ev); + + start_param.tick_type = ODP_TIMER_TICK_ABS; + start_param.tick = tck; + start_param.tmo_ev = tt[i].ev; + tt[i].ev = ODP_EVENT_INVALID; + + timer_rc = odp_timer_start(tt[i].tim, &start_param); if (timer_rc == ODP_TIMER_TOO_NEAR) { ODPH_ERR("Missed tick, setting timer\n"); } else if (timer_rc != ODP_TIMER_SUCCESS) { -- cgit v1.2.3 From 7bccb570bf6242f4bab409139dd10177629129f7 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 20 Dec 2022 10:55:47 +0200 Subject: validation: timer: remove deprecated odp_timer_set_rel() function usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop using deprecated odp_timer_set_rel() function. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- test/validation/api/timer/timer.c | 49 +++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index 071e710fa..5f57fdaae 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -666,6 +666,7 @@ static void timer_pool_max_res(void) odp_queue_param_t queue_param; odp_timer_pool_t tp; odp_timer_t timer; + odp_timer_start_t start_param; odp_pool_param_t pool_param; odp_pool_t pool; odp_queue_t queue; @@ -729,7 +730,11 @@ static void timer_pool_max_res(void) ev = odp_timeout_to_event(tmo); CU_ASSERT_FATAL(ev != ODP_EVENT_INVALID); - ret = odp_timer_set_rel(timer, tick, &ev); + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = tick; + start_param.tmo_ev = ev; + + ret = odp_timer_start(timer, &start_param); CU_ASSERT(ret == ODP_TIMER_SUCCESS); ev = ODP_EVENT_INVALID; @@ -1145,6 +1150,7 @@ static void timer_test_event_type(odp_queue_type_t queue_type, odp_queue_param_t queue_param; odp_timer_pool_param_t timer_param; odp_timer_pool_t timer_pool; + odp_timer_start_t start_param; odp_queue_t queue; odp_timeout_t tmo; odp_buffer_t buf; @@ -1231,7 +1237,11 @@ static void timer_test_event_type(odp_queue_type_t queue_type, timer[i] = odp_timer_alloc(timer_pool, queue, user_ctx); CU_ASSERT_FATAL(timer[i] != ODP_TIMER_INVALID); - ret = odp_timer_set_rel(timer[i], (i + 1) * period_tick, &ev); + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = (i + 1) * period_tick; + start_param.tmo_ev = ev; + + ret = odp_timer_start(timer[i], &start_param); if (ret == ODP_TIMER_TOO_NEAR) ODPH_DBG("Timer set failed. Too near %i.\n", i); @@ -1547,12 +1557,12 @@ static void timer_test_cancel(void) odp_queue_param_t queue_param; odp_timer_capability_t capa; odp_timer_pool_t tp; + odp_timer_start_t start_param; odp_queue_t queue; odp_timer_t tim; odp_event_t ev; odp_timeout_t tmo; odp_timer_set_t rc; - uint64_t tick; int ret; memset(&capa, 0, sizeof(capa)); @@ -1603,9 +1613,11 @@ static void timer_test_cancel(void) if (ev == ODP_EVENT_INVALID) CU_FAIL_FATAL("Failed to allocate timeout"); - tick = odp_timer_ns_to_tick(tp, tparam.max_tmo / 2); + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = odp_timer_ns_to_tick(tp, tparam.max_tmo / 2); + start_param.tmo_ev = ev; - rc = odp_timer_set_rel(tim, tick, &ev); + rc = odp_timer_start(tim, &start_param); if (rc != ODP_TIMER_SUCCESS) CU_FAIL_FATAL("Failed to set timer (relative time)"); @@ -1651,6 +1663,7 @@ static void timer_test_tmo_limit(odp_queue_type_t queue_type, odp_queue_param_t queue_param; odp_timer_pool_param_t timer_param; odp_timer_pool_t timer_pool; + odp_timer_start_t start_param; odp_queue_t queue; odp_timeout_t tmo; odp_event_t ev; @@ -1737,7 +1750,12 @@ static void timer_test_tmo_limit(odp_queue_type_t queue_type, CU_ASSERT(ev != ODP_EVENT_INVALID); t1 = odp_time_local(); - ret = odp_timer_set_rel(timer[i], tmo_tick, &ev); + + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = tmo_tick; + start_param.tmo_ev = ev; + + ret = odp_timer_start(timer[i], &start_param); if (ret == ODP_TIMER_TOO_NEAR) ODPH_DBG("Timer set failed. Too near %i.\n", i); @@ -2071,23 +2089,34 @@ static int worker_entrypoint(void *arg ODP_UNUSED) odp_timer_set_t rc; uint64_t cur_tick; uint64_t tck; + int reset_timer = 0; - if (tt[i].ev != ODP_EVENT_INVALID) + if (tt[i].ev != ODP_EVENT_INVALID) { /* Timer inactive => set */ nset++; - else if (odp_timer_current_tick(tp) >= tt[i].tick) + } else if (odp_timer_current_tick(tp) >= tt[i].tick) { /* Timer just expired. */ goto sleep; - else + } else { /* Timer active => reset */ nreset++; + reset_timer = 1; + } nsec = min_tmo + THREE_POINT_THREE_MSEC + (rand_r(&seed) % RANGE_MS) * 1000000ULL; tck = odp_timer_ns_to_tick(tp, nsec); cur_tick = odp_timer_current_tick(tp); - rc = odp_timer_set_rel(tt[i].tim, tck, &tt[i].ev); + + start_param.tick_type = ODP_TIMER_TICK_REL; + start_param.tick = tck; + start_param.tmo_ev = tt[i].ev; + + if (reset_timer) + rc = odp_timer_restart(tt[i].tim, &start_param); + else + rc = odp_timer_start(tt[i].tim, &start_param); if (rc == ODP_TIMER_TOO_NEAR) { CU_FAIL("Failed to set timer: TOO NEAR"); -- cgit v1.2.3 From 50f60d94806d6dcaf15b6459642af2e48a431f71 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 22 Dec 2022 11:08:16 +0200 Subject: doc: usersguide: update timer documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove references to deprecated odp_timer_set_abs() and odp_timer_set_rel() functions from user's guide. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen --- doc/images/timeout_fsm.gv | 8 ++------ doc/images/timer_fsm.gv | 15 +++++---------- doc/users-guide/users-guide-timer.adoc | 10 +++++----- 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/doc/images/timeout_fsm.gv b/doc/images/timeout_fsm.gv index dab24d0d6..179434bce 100644 --- a/doc/images/timeout_fsm.gv +++ b/doc/images/timeout_fsm.gv @@ -8,17 +8,13 @@ digraph timer_state_machine { TO_Unalloc -> TO_Alloc [label="odp_timeout_alloc()"]; TO_Alloc -> TO_Unalloc [label="odp_timeout_free()"]; TO_Alloc -> TO_Pending [fontcolor=green, - label="odp_timer_set_abs()"]; - TO_Alloc -> TO_Pending [fontcolor=green, - label="odp_timer_set_rel()"]; + label="odp_timer_start()"]; TO_Pending -> TO_Alloc [fontcolor=green, label="odp_timer_cancel()"]; TO_Pending -> TO_Enqueued [fontcolor=green, label="timer expires"]; TO_Enqueued -> TO_Delivered [label="odp_schedule()"]; TO_Delivered -> TO_Pending [fontcolor=green, - label="odp_timer_set_abs()"]; - TO_Delivered -> TO_Pending [fontcolor=green, - label="odp_timer_set_rel()"]; + label="odp_timer_start()"]; TO_Delivered -> TO_Delivered [label="odp_timeout_from_event()"]; TO_Delivered -> TO_Delivered [label="odp_timeout_timer()"]; TO_Delivered -> TO_Unalloc diff --git a/doc/images/timer_fsm.gv b/doc/images/timer_fsm.gv index 819d0f979..eb1df8ae9 100644 --- a/doc/images/timer_fsm.gv +++ b/doc/images/timer_fsm.gv @@ -3,17 +3,12 @@ digraph timer_state_machine { node [fontsize=28]; edge [fontsize=28]; node [shape=doublecircle]; Timer_Unalloc; - node [shape=circle]; Timer_Alloc Timer_Set Timer_Expired + node [shape=circle]; Timer_Alloc Timer_Active Timer_Expired Timer_Unalloc -> Timer_Alloc [label="odp_timer_alloc()"]; Timer_Alloc -> Timer_Unalloc [label="odp_timer_free()"]; - Timer_Alloc -> Timer_Set [fontcolor=green,label="odp_timer_set_abs()"]; - Timer_Alloc -> Timer_Set [fontcolor=green,label="odp_timer_set_rel()"]; - Timer_Set -> Timer_Alloc [fontcolor=green,label="odp_timer_cancel()"]; - Timer_Set -> Timer_Expired [fontcolor=green,label="timer expires"]; + Timer_Alloc -> Timer_Active [fontcolor=green,label="odp_timer_start()"]; + Timer_Active -> Timer_Alloc [fontcolor=green,label="odp_timer_cancel()"]; + Timer_Active -> Timer_Expired [fontcolor=green,label="timer expires"]; Timer_Expired -> Timer_Unalloc [label="odp_timer_free()"]; - Timer_Expired -> Timer_Set [fontcolor=green, - label="odp_timer_set_abs()"]; - Timer_Expired -> Timer_Set [fontcolor=green, - label="odp_timer_set_rel()"]; - + Timer_Expired -> Timer_Active [fontcolor=green, label="odp_timer_start()"]; } diff --git a/doc/users-guide/users-guide-timer.adoc b/doc/users-guide/users-guide-timer.adoc index 4a8bad74f..e854b5f2d 100644 --- a/doc/users-guide/users-guide-timer.adoc +++ b/doc/users-guide/users-guide-timer.adoc @@ -46,11 +46,11 @@ as an input parameter to enable the pool-specific conversion ratios to be used. Associated with each timer pool is a free running tick counter that can be -sampled at any time via the `odp_timer_current_tick()` API. Timers can be set -to an absolute future tick value via `odp_timer_set_abs()` or to a future tick -value relative to the current tick via `odp_timer_set_rel()`. Implementations -may impose minimum and maximum future values supported by a given timer pool -and timer set operations will fail if the requested value is outside of the +sampled at any time via the `odp_timer_current_tick()` API. Timers are started +with `odp_timer_start()` and the expiration time can be an absolute future tick +value or a future tick value relative to the current tick. Implementations may +impose minimum and maximum future values supported by a given timer pool and +timer start operations will fail if the requested value is outside of the supported range. Before a set timer expires, it can be canceled via the `odp_timer_cancel()` -- cgit v1.2.3 From ccb7b35bb75366968e316bbadd6a030a976e5b21 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 28 Dec 2022 16:42:43 +0200 Subject: linux-gen: dpdk: fix zero-copy mode with vfio-pci driver For DPDK zero-copy pktio to work with VFIO the packet pool memory has to be mapped for DMA access. Fixes: https://github.com/OpenDataPlane/odp/issues/1718 Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-generic/pktio/dpdk.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 7e9db90ce..fa40d1bde 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -49,6 +49,7 @@ #include #include #include +#include /* NUMA is not supported on all platforms */ #ifdef _ODP_HAVE_NUMA_LIBRARY @@ -438,6 +439,15 @@ static struct rte_mempool *mbuf_pool_create(const char *name, goto fail; } + /* Map pages for DMA access to enable VFIO usage */ + for (uint64_t i = 0; i < pool_entry->shm_size; i += page_size) { + addr = pool_entry->base_addr + i; + + rte_vfio_container_dma_map(RTE_VFIO_DEFAULT_CONTAINER_FD, + (uint64_t)(uintptr_t)addr, + rte_mem_virt2iova(addr), page_size); + } + rte_mempool_obj_iter(mp, pktmbuf_init, NULL); return mp; -- cgit v1.2.3 From 648c27bfcfdf4c19e4418aedf75e803eb914bcfd Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 28 Oct 2022 17:32:56 +0300 Subject: api: crypto: split header files Split crypto API into separate header files for functions and types. This enables inline implementation of API functions. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- include/Makefile.am | 9 + include/odp/api/abi-default/crypto.h | 19 +- include/odp/api/abi-default/crypto_types.h | 37 + include/odp/api/crypto.h | 12 - include/odp/api/crypto_types.h | 28 + include/odp/api/spec/crypto.h | 1060 +------------------ include/odp/api/spec/crypto_types.h | 1095 ++++++++++++++++++++ include/odp/api/spec/ipsec.h | 2 +- include/odp/api/spec/ipsec_types.h | 2 +- .../arch/arm32-linux/odp/api/abi/crypto_types.h | 7 + .../arch/arm64-linux/odp/api/abi/crypto_types.h | 7 + .../arch/default-linux/odp/api/abi/crypto_types.h | 7 + .../arch/power64-linux/odp/api/abi/crypto_types.h | 7 + .../arch/x86_32-linux/odp/api/abi/crypto_types.h | 7 + .../arch/x86_64-linux/odp/api/abi/crypto_types.h | 7 + platform/linux-generic/Makefile.am | 1 + .../linux-generic/include-abi/odp/api/abi/crypto.h | 14 +- .../include-abi/odp/api/abi/crypto_types.h | 42 + platform/linux-generic/odp_ipsec_sad.c | 1 + 19 files changed, 1263 insertions(+), 1101 deletions(-) create mode 100644 include/odp/api/abi-default/crypto_types.h create mode 100644 include/odp/api/crypto_types.h create mode 100644 include/odp/api/spec/crypto_types.h create mode 100644 include/odp/arch/arm32-linux/odp/api/abi/crypto_types.h create mode 100644 include/odp/arch/arm64-linux/odp/api/abi/crypto_types.h create mode 100644 include/odp/arch/default-linux/odp/api/abi/crypto_types.h create mode 100644 include/odp/arch/power64-linux/odp/api/abi/crypto_types.h create mode 100644 include/odp/arch/x86_32-linux/odp/api/abi/crypto_types.h create mode 100644 include/odp/arch/x86_64-linux/odp/api/abi/crypto_types.h create mode 100644 platform/linux-generic/include-abi/odp/api/abi/crypto_types.h diff --git a/include/Makefile.am b/include/Makefile.am index 49ccf552e..7e9b8fedf 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -20,6 +20,7 @@ odpapiinclude_HEADERS = \ odp/api/cpu.h \ odp/api/cpumask.h \ odp/api/crypto.h \ + odp/api/crypto_types.h \ odp/api/debug.h \ odp/api/deprecated.h \ odp/api/dma.h \ @@ -84,6 +85,7 @@ odpapispecinclude_HEADERS = \ odp/api/spec/cpu.h \ odp/api/spec/cpumask.h \ odp/api/spec/crypto.h \ + odp/api/spec/crypto_types.h \ odp/api/spec/debug.h \ odp/api/spec/dma.h \ odp/api/spec/dma_types.h \ @@ -150,6 +152,7 @@ odpapiabidefaultinclude_HEADERS = \ odp/api/abi-default/cpu.h \ odp/api/abi-default/cpumask.h \ odp/api/abi-default/crypto.h \ + odp/api/abi-default/crypto_types.h \ odp/api/abi-default/debug.h \ odp/api/abi-default/dma_types.h \ odp/api/abi-default/errno.h \ @@ -208,6 +211,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm32-linux/odp/api/abi/cpu.h \ odp/arch/arm32-linux/odp/api/abi/cpumask.h \ odp/arch/arm32-linux/odp/api/abi/crypto.h \ + odp/arch/arm32-linux/odp/api/abi/crypto_types.h \ odp/arch/arm32-linux/odp/api/abi/debug.h \ odp/arch/arm32-linux/odp/api/abi/dma_types.h \ odp/arch/arm32-linux/odp/api/abi/errno.h \ @@ -262,6 +266,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm64-linux/odp/api/abi/cpu.h \ odp/arch/arm64-linux/odp/api/abi/cpumask.h \ odp/arch/arm64-linux/odp/api/abi/crypto.h \ + odp/arch/arm64-linux/odp/api/abi/crypto_types.h \ odp/arch/arm64-linux/odp/api/abi/debug.h \ odp/arch/arm64-linux/odp/api/abi/dma_types.h \ odp/arch/arm64-linux/odp/api/abi/errno.h \ @@ -316,6 +321,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/default-linux/odp/api/abi/cpu.h \ odp/arch/default-linux/odp/api/abi/cpumask.h \ odp/arch/default-linux/odp/api/abi/crypto.h \ + odp/arch/default-linux/odp/api/abi/crypto_types.h \ odp/arch/default-linux/odp/api/abi/debug.h \ odp/arch/default-linux/odp/api/abi/dma_types.h \ odp/arch/default-linux/odp/api/abi/errno.h \ @@ -370,6 +376,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/power64-linux/odp/api/abi/cpu.h \ odp/arch/power64-linux/odp/api/abi/cpumask.h \ odp/arch/power64-linux/odp/api/abi/crypto.h \ + odp/arch/power64-linux/odp/api/abi/crypto_types.h \ odp/arch/power64-linux/odp/api/abi/debug.h \ odp/arch/power64-linux/odp/api/abi/dma_types.h \ odp/arch/power64-linux/odp/api/abi/errno.h \ @@ -424,6 +431,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_32-linux/odp/api/abi/cpu.h \ odp/arch/x86_32-linux/odp/api/abi/cpumask.h \ odp/arch/x86_32-linux/odp/api/abi/crypto.h \ + odp/arch/x86_32-linux/odp/api/abi/crypto_types.h \ odp/arch/x86_32-linux/odp/api/abi/debug.h \ odp/arch/x86_32-linux/odp/api/abi/dma_types.h \ odp/arch/x86_32-linux/odp/api/abi/errno.h \ @@ -478,6 +486,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_64-linux/odp/api/abi/cpu.h \ odp/arch/x86_64-linux/odp/api/abi/cpumask.h \ odp/arch/x86_64-linux/odp/api/abi/crypto.h \ + odp/arch/x86_64-linux/odp/api/abi/crypto_types.h \ odp/arch/x86_64-linux/odp/api/abi/debug.h \ odp/arch/x86_64-linux/odp/api/abi/dma_types.h \ odp/arch/x86_64-linux/odp/api/abi/errno.h \ diff --git a/include/odp/api/abi-default/crypto.h b/include/odp/api/abi-default/crypto.h index e7c0a651c..0e9e8506a 100644 --- a/include/odp/api/abi-default/crypto.h +++ b/include/odp/api/abi-default/crypto.h @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -11,23 +12,7 @@ extern "C" { #endif -#include - -/** @internal Dummy type for strong typing */ -typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_crypto_compl_t; - -/** @ingroup odp_crypto - * @{ - */ - -#define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL) - -typedef uint64_t odp_crypto_session_t; -typedef _odp_abi_crypto_compl_t *odp_crypto_compl_t; - -/** - * @} - */ +/* Empty header required due to the inline functions */ #ifdef __cplusplus } diff --git a/include/odp/api/abi-default/crypto_types.h b/include/odp/api/abi-default/crypto_types.h new file mode 100644 index 000000000..cc9155bfa --- /dev/null +++ b/include/odp/api/abi-default/crypto_types.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ABI_CRYPTO_TYPES_H_ +#define ODP_ABI_CRYPTO_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/** @internal Dummy type for strong typing */ +typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_crypto_compl_t; + +/** @ingroup odp_crypto + * @{ + */ + +#define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL) + +typedef uint64_t odp_crypto_session_t; +typedef _odp_abi_crypto_compl_t *odp_crypto_compl_t; + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/crypto.h b/include/odp/api/crypto.h index ec7c8370b..336304553 100644 --- a/include/odp/api/crypto.h +++ b/include/odp/api/crypto.h @@ -17,19 +17,7 @@ extern "C" { #endif -#include -#include #include -#include -#include - -/** @ingroup odp_crypto - * @{ - */ - -/** - * @} - */ #include diff --git a/include/odp/api/crypto_types.h b/include/odp/api/crypto_types.h new file mode 100644 index 000000000..225ff7df0 --- /dev/null +++ b/include/odp/api/crypto_types.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP crypto + */ + +#ifndef ODP_API_CRYPTO_TYPES_H_ +#define ODP_API_CRYPTO_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index a79a05ad3..2118cb589 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -15,9 +15,9 @@ #define ODP_API_SPEC_CRYPTO_H_ #include +#include #include #include -#include #include #ifdef __cplusplus @@ -29,1062 +29,6 @@ extern "C" { * @{ */ -/** - * @def ODP_CRYPTO_SESSION_INVALID - * Invalid session handle - */ - -/** - * @typedef odp_crypto_session_t - * Crypto API opaque session handle - */ - -/** - * @typedef odp_crypto_compl_t -* @deprecated Crypto API completion event (platform dependent). -*/ - -/** - * Crypto API operation mode - */ -typedef enum { - /** Synchronous, return results immediately */ - ODP_CRYPTO_SYNC, - /** Asynchronous, return results via posted event */ - ODP_CRYPTO_ASYNC, -} odp_crypto_op_mode_t; - -/** - * Crypto API operation type - */ -typedef enum { - /** Encrypt and/or compute authentication ICV */ - ODP_CRYPTO_OP_ENCODE, - /** Decrypt and/or verify authentication ICV */ - ODP_CRYPTO_OP_DECODE, -} odp_crypto_op_t; - -/** - * Crypto API cipher algorithm - */ -typedef enum { - /** No cipher algorithm specified */ - ODP_CIPHER_ALG_NULL, - - /** DES */ - ODP_CIPHER_ALG_DES, - - /** Triple DES with cipher block chaining */ - ODP_CIPHER_ALG_3DES_CBC, - - /** Triple DES with Electronic Codebook */ - ODP_CIPHER_ALG_3DES_ECB, - - /** AES with cipher block chaining */ - ODP_CIPHER_ALG_AES_CBC, - - /** AES with counter mode */ - ODP_CIPHER_ALG_AES_CTR, - - /** AES with electronic codebook */ - ODP_CIPHER_ALG_AES_ECB, - - /** AES with 128-bit cipher feedback */ - ODP_CIPHER_ALG_AES_CFB128, - - /** AES with XEX-based tweaked-codebook mode with ciphertext stealing - * (XTS) */ - ODP_CIPHER_ALG_AES_XTS, - - /** AES-GCM - * - * AES in Galois/Counter Mode (GCM) algorithm. GCM provides both - * authentication and ciphering of data (authenticated encryption) - * in the same operation. Hence this algorithm must be paired always - * with ODP_AUTH_ALG_AES_GCM authentication. - */ - ODP_CIPHER_ALG_AES_GCM, - - /** AES-CCM - * - * AES in Counter with CBC-MAC (CCM) mode algorithm. CCM provides both - * authentication and ciphering of data (authenticated encryption) - * in the same operation. Hence this algorithm must be paired always - * with ODP_AUTH_ALG_AES_CCM authentication. - */ - ODP_CIPHER_ALG_AES_CCM, - - /** ChaCha20-Poly1305 - * - * ChaCha20 with Poly1305 provide both authentication and ciphering of - * data (authenticated encryption) in the same operation. Hence this - * algorithm must be paired always with ODP_AUTH_ALG_CHACHA20_POLY1305 - * authentication. - */ - ODP_CIPHER_ALG_CHACHA20_POLY1305, - - /** Confidentiality F8 algorithm (UEA1) - * - * KASUMI-based F8 algorithm (also known as UEA1). - * - * IV should be formatted according to the 3GPP TS 35.201: - * COUNT || BEARER || DIRECTION || 0...0 - */ - ODP_CIPHER_ALG_KASUMI_F8, - - /** Confidentiality UEA2 algorithm (128-EEA1) - * - * SNOW 3G-based UEA2 algorithm (also known as 128-EEA1). - * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * UEA2 & UIA2 specification: - * COUNT || BEARER || DIRECTION || 0...0 || - * COUNT || BEARER || DIRECTION || 0...0 || - */ - ODP_CIPHER_ALG_SNOW3G_UEA2, - - /** Confidentiality 128-EEA2 algorithm - * - * AES-CTR-based 128-EEA2 algorithm. - * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * 128-EA2 & 128-EIA2 specification: - * COUNT || BEARER || - * DIRECTION || 0....0 - */ - ODP_CIPHER_ALG_AES_EEA2, - - /** ZUC based confidentiality algorithm - * - * 128-EEA3/128-NEA3 algorithm when key length is 128 bits. - * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * 128-EEA3 & 128-EIA3 specification: - * COUNT || BEARER || DIRECTION || 0...0 || - * COUNT || BEARER || DIRECTION || 0...0 || - * - * 256-bit key length support is experimental and subject to - * change. The following variants may be supported: - * - * - ZUC-256 with 25 byte IV (of which 184 bits are variable) - * as specified in "The ZUC-256 Stream Cipher". - * - ZUC-256 with 16 byte IV as specified in - * "An Addendum to the ZUC-256 Stream Cipher", - * https://eprint.iacr.org/2021/1439 - */ - ODP_CIPHER_ALG_ZUC_EEA3, - -} odp_cipher_alg_t; - -/** - * Crypto API authentication algorithm - */ -typedef enum { - /** No authentication algorithm specified */ - ODP_AUTH_ALG_NULL, - - /** HMAC-MD5 - * - * MD5 algorithm in HMAC mode - */ - ODP_AUTH_ALG_MD5_HMAC, - - /** HMAC-SHA-1 - * - * SHA-1 algorithm in HMAC mode - */ - ODP_AUTH_ALG_SHA1_HMAC, - - /** HMAC-SHA-224 - * - * SHA-224 algorithm in HMAC mode - */ - ODP_AUTH_ALG_SHA224_HMAC, - - /** HMAC-SHA-256 - * - * SHA-256 algorithm in HMAC mode - */ - ODP_AUTH_ALG_SHA256_HMAC, - - /** HMAC-SHA-384 - * - * SHA-384 algorithm in HMAC mode - */ - ODP_AUTH_ALG_SHA384_HMAC, - - /** HMAC-SHA-512 - * - * SHA-512 algorithm in HMAC mode - */ - ODP_AUTH_ALG_SHA512_HMAC, - - /** AES-GCM - * - * AES in Galois/Counter Mode (GCM) algorithm. GCM provides both - * authentication and ciphering of data (authenticated encryption) - * in the same operation. Hence this algorithm must be paired always - * with ODP_CIPHER_ALG_AES_GCM cipher. - */ - ODP_AUTH_ALG_AES_GCM, - - /** AES-GMAC - * - * AES Galois Message Authentication Code (GMAC) algorithm. AES-GMAC - * is based on AES-GCM operation, but provides authentication only. - * Hence this algorithm can be paired only with ODP_CIPHER_ALG_NULL - * cipher. - * - * NIST and RFC specifications of GMAC refer to all data to be - * authenticated as AAD. In ODP the data to be authenticated, i.e. - * AAD, is ODP packet data and specified using the auth_range - * parameter. The aad_length and aad_ptr parameters, which would - * require the data to be contiguous in memory, are ignored with - * AES-GMAC. - * - * GMAC needs an initialization vector, which can be passed via - * session (auth_iv) or packet (auth_iv_ptr) level parameters. - */ - ODP_AUTH_ALG_AES_GMAC, - - /** AES-CCM - * - * AES in Counter with CBC-MAC (CCM) mode algorithm. CCM provides both - * authentication and ciphering of data (authenticated encryption) - * in the same operation. Hence this algorithm must be paired always - * with ODP_CIPHER_ALG_AES_CCM cipher. - */ - ODP_AUTH_ALG_AES_CCM, - - /** AES-CMAC - * - * AES Cipher-based Message Authentication Code (CMAC) algorithm. CMAC - * is a keyed hash function that is based on a symmetric key block - * cipher, such as the AES. - */ - ODP_AUTH_ALG_AES_CMAC, - - /** AES-XCBC-MAC - * - * AES CBC MAC for arbitrary-length messages (XCBC-MAC). - * - */ - ODP_AUTH_ALG_AES_XCBC_MAC, - - /** ChaCha20-Poly1305 AEAD - * - * ChaCha20 with Poly1305 provide both authentication and ciphering of - * data (authenticated encryption) in the same operation. Hence this - * algorithm must be paired always with - * ODP_CIPHER_ALG_CHACHA20_POLY1305 cipher. - */ - ODP_AUTH_ALG_CHACHA20_POLY1305, - - /** Integrity F9 algorithm (UIA1) - * - * KASUMI-based F9 algorithm (also known as UIA1). - * - * IV (9 bytes) is a concatenation of COUNT (32b), FRESH (32b) and - * DIRECTION (LSB-aligned, 1b). - * IV (8 bytes) is a concatenation of COUNT (32b) and FRESH (32b) - * DIRECTION (1b) and padding should come at the end of message. - */ - ODP_AUTH_ALG_KASUMI_F9, - - /** Integrity UIA2 algorithm (128-EIA1) - * - * SNOW 3G-based UIA2 algorithm (also known as 128-EIA1). - * IV (128 bit) should be formatted according to the ETSI/SAGE - * UEA2 & UIA2 specification: - * COUNT || FRESH || - * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 || - * FRESH0 .. FRESH15 || FRESH16 XOR DIRECTION || FRESH17 .. FRESH31 - */ - ODP_AUTH_ALG_SNOW3G_UIA2, - - /** Integrity 128-EIA2 algorithm - * - * AES_CMAC-based 128-EIA2 algorithm. - * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * 128-EA2 & 128-EIA2 specification: - * COUNT || BEARER || - * DIRECTION || 0....0 - */ - ODP_AUTH_ALG_AES_EIA2, - - /** ZUC-based integrity algorithm. - * - * 128-EIA3/128-NIA3 algorithm when key length is 128 bits. - * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * 128-EA3 & 128-EIA2 specification: - * COUNT || BEARER || - * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 || - * BEARER || 0...0 || DIRECTION || 0...0 - * - * 256-bit key length support is experimental and subject to - * change. The following variants may be supported: - * - * - ZUC-256 with 25 byte IV (of which 184 bits are variable) and - * 32/64/128 bit MAC as specified in "The ZUC-256 Stream Cipher". - * - ZUC-256 with 16 byte IV and 32/64/128 bit MAC as specified in - * "An Addendum to the ZUC-256 Stream Cipher", - * https://eprint.iacr.org/2021/1439 - */ - ODP_AUTH_ALG_ZUC_EIA3, - - /** MD5 algorithm */ - ODP_AUTH_ALG_MD5, - - /** SHA1 algorithm */ - ODP_AUTH_ALG_SHA1, - - /** 224 bit SHA2 algorithm */ - ODP_AUTH_ALG_SHA224, - - /** 256 bit SHA2 algorithm */ - ODP_AUTH_ALG_SHA256, - - /** 384 bit SHA2 algorithm */ - ODP_AUTH_ALG_SHA384, - - /** 512 bit SHA2 algorithm */ - ODP_AUTH_ALG_SHA512, - -} odp_auth_alg_t; - -/** - * Cipher algorithms in a bit field structure - */ -typedef union odp_crypto_cipher_algos_t { - /** Cipher algorithms */ - struct { - /** ODP_CIPHER_ALG_NULL */ - uint32_t null : 1; - - /** ODP_CIPHER_ALG_DES */ - uint32_t des : 1; - - /** ODP_CIPHER_ALG_3DES_CBC */ - uint32_t trides_cbc : 1; - - /** ODP_CIPHER_ALG_3DES_ECB */ - uint32_t trides_ecb : 1; - - /** ODP_CIPHER_ALG_AES_CBC */ - uint32_t aes_cbc : 1; - - /** ODP_CIPHER_ALG_AES_CTR */ - uint32_t aes_ctr : 1; - - /** ODP_CIPHER_ALG_AES_ECB */ - uint32_t aes_ecb : 1; - - /** ODP_CIPHER_ALG_AES_CFB128 */ - uint32_t aes_cfb128 : 1; - - /** ODP_CIPHER_ALG_AES_XTS */ - uint32_t aes_xts : 1; - - /** ODP_CIPHER_ALG_AES_GCM */ - uint32_t aes_gcm : 1; - - /** ODP_CIPHER_ALG_AES_CCM */ - uint32_t aes_ccm : 1; - - /** ODP_CIPHER_ALG_CHACHA20_POLY1305 */ - uint32_t chacha20_poly1305 : 1; - - /** ODP_CIPHER_ALG_KASUMI_F8 */ - uint32_t kasumi_f8 : 1; - - /** ODP_CIPHER_ALG_SNOW3G_UEA2 */ - uint32_t snow3g_uea2 : 1; - - /** ODP_CIPHER_ALG_AES_EEA2 */ - uint32_t aes_eea2 : 1; - - /** ODP_CIPHER_ALG_ZUC_EEA3 */ - uint32_t zuc_eea3 : 1; - - } bit; - - /** All bits of the bit field structure - * - * This field can be used to set/clear all flags, or bitwise - * operations over the entire structure. */ - uint32_t all_bits; -} odp_crypto_cipher_algos_t; - -/** - * Authentication algorithms in a bit field structure - */ -typedef union odp_crypto_auth_algos_t { - /** Authentication algorithms */ - struct { - /** ODP_AUTH_ALG_NULL */ - uint32_t null : 1; - - /** ODP_AUTH_ALG_MD5_HMAC */ - uint32_t md5_hmac : 1; - - /** ODP_AUTH_ALG_SHA1_HMAC */ - uint32_t sha1_hmac : 1; - - /** ODP_AUTH_ALG_SHA224_HMAC */ - uint32_t sha224_hmac : 1; - - /** ODP_AUTH_ALG_SHA256_HMAC */ - uint32_t sha256_hmac : 1; - - /** ODP_AUTH_ALG_SHA384_HMAC */ - uint32_t sha384_hmac : 1; - - /** ODP_AUTH_ALG_SHA512_HMAC */ - uint32_t sha512_hmac : 1; - - /** ODP_AUTH_ALG_AES_GCM */ - uint32_t aes_gcm : 1; - - /** ODP_AUTH_ALG_AES_GMAC*/ - uint32_t aes_gmac : 1; - - /** ODP_AUTH_ALG_AES_CCM */ - uint32_t aes_ccm : 1; - - /** ODP_AUTH_ALG_AES_CMAC*/ - uint32_t aes_cmac : 1; - - /** ODP_AUTH_ALG_AES_XCBC_MAC*/ - uint32_t aes_xcbc_mac : 1; - - /** ODP_AUTH_ALG_CHACHA20_POLY1305 */ - uint32_t chacha20_poly1305 : 1; - - /** ODP_AUTH_ALG_KASUMI_F9 */ - uint32_t kasumi_f9 : 1; - - /** ODP_AUTH_ALG_SNOW3G_UIA2 */ - uint32_t snow3g_uia2 : 1; - - /** ODP_AUTH_ALG_AES_EIA2 */ - uint32_t aes_eia2 : 1; - - /** ODP_AUTH_ALG_ZUC_EIA3 */ - uint32_t zuc_eia3 : 1; - - /** ODP_AUTH_ALG_MD5 */ - uint32_t md5 : 1; - - /** ODP_AUTH_ALG_SHA1 */ - uint32_t sha1 : 1; - - /** ODP_AUTH_ALG_SHA224 */ - uint32_t sha224 : 1; - - /** ODP_AUTH_ALG_SHA256 */ - uint32_t sha256 : 1; - - /** ODP_AUTH_ALG_SHA384 */ - uint32_t sha384 : 1; - - /** ODP_AUTH_ALG_SHA512 */ - uint32_t sha512 : 1; - - } bit; - - /** All bits of the bit field structure - * - * This field can be used to set/clear all flags, or bitwise - * operations over the entire structure. */ - uint32_t all_bits; -} odp_crypto_auth_algos_t; - -/** - * Crypto API key structure - */ -typedef struct odp_crypto_key { - /** Key data */ - uint8_t *data; - - /** Key length in bytes */ - uint32_t length; - -} odp_crypto_key_t; - -/** - * Crypto API IV structure - * - * @deprecated Use per-packet IV in crypto operation parameters - */ -typedef struct odp_crypto_iv { - /** IV data - * - * Ignored when length is zero. Null value indicates that an - * IV will be provided for each packet through the crypto - * operation parameters. In that case the per-operation - * IV parameter must always point to a valid IV. - * - * Default value is NULL. - */ - uint8_t *data; - - /** IV length in bytes. Default value is zero. */ - uint32_t length; - -} ODP_DEPRECATE(odp_crypto_iv_t); - -/** - * Crypto API session creation parameters - */ -typedef struct odp_crypto_session_param_t { - /** Encode vs. decode operation - * - * The default value is ODP_CRYPTO_OP_ENCODE. - */ - odp_crypto_op_t op; - - /** Authenticate cipher vs. plain text - * - * Controls ordering of authentication and cipher operations, - * and is relative to the operation (encode vs decode). When encoding, - * TRUE indicates the authentication operation should be performed - * after the cipher operation else before. When decoding, TRUE - * indicates the reverse order of operation. - * - * The value is ignored with authenticated encryption algorithms - * such as AES-GCM. - * - * true: Authenticate cipher text - * false: Authenticate plain text - * - * The default value is false. - */ - odp_bool_t auth_cipher_text; - - /** Hash result location may overlap authentication range - * - * This flag indicates that the hash result location may (but is - * not required to) overlap authentication range. Setting this - * flag may reduce performance. - * - * Default value is false. - */ - odp_bool_t hash_result_in_auth_range; - - /** Preferred sync vs. async for odp_crypto_operation() - * - * The default value is ODP_CRYPTO_SYNC. - * - * @deprecated Used only with deprecated odp_crypto_operation() - */ - odp_crypto_op_mode_t ODP_DEPRECATE(pref_mode); - - /** Operation mode when using packet interface: sync or async - * - * The default value is ODP_CRYPTO_SYNC. - */ - odp_crypto_op_mode_t op_mode; - - /** Cipher algorithm - * - * Select cipher algorithm to be used. ODP_CIPHER_ALG_NULL indicates - * that ciphering is disabled. Use odp_crypto_capability() for - * supported algorithms. Note that some algorithms restrict choice of - * the pairing authentication algorithm. When ciphering is enabled - * cipher key and IV need to be set. The default value is - * ODP_CIPHER_ALG_NULL. - */ - odp_cipher_alg_t cipher_alg; - - /** Cipher key - * - * Use odp_crypto_cipher_capa() for supported key and IV lengths. - */ - odp_crypto_key_t cipher_key; - - /** Cipher Initialization Vector (IV) - * - * Unless using the deprecated API, this specifies the length of - * the IV only. The actual IV must then be provided in per-packet - * parameters of crypto operations. - */ - union { -#if ODP_DEPRECATED_API - /** @deprecated Cipher IV */ - odp_crypto_iv_t ODP_DEPRECATE(cipher_iv); -#endif - /** Cipher IV length */ - struct { -#if ODP_DEPRECATED_API - /** @cond - * Unused padding field - */ - uint8_t *dummy_padding_0; - /** @endcond */ -#endif - /** Length of cipher initialization vector. - * Default value is zero. - */ - uint32_t cipher_iv_len; - }; - }; - - /** Authentication algorithm - * - * Select authentication algorithm to be used. ODP_AUTH_ALG_NULL - * indicates that authentication is disabled. Use - * odp_crypto_capability() for supported algorithms. Note that some - * algorithms restrict choice of the pairing cipher algorithm. When - * single algorithm provides both ciphering and authentication - * (i.e. Authenticated Encryption), authentication side key - * (auth_key) and IV (auth_iv) are ignored, and cipher side values are - * used instead. These algorithms ignore authentication side key - * and IV: ODP_AUTH_ALG_AES_GCM, ODP_AUTH_ALG_AES_CCM and - * ODP_AUTH_ALG_CHACHA20_POLY1305. Otherwise, all authentication side - * parameters must be set when authentication is enabled. The default - * value is ODP_AUTH_ALG_NULL. - */ - odp_auth_alg_t auth_alg; - - /** Authentication key - * - * Use odp_crypto_auth_capability() for supported key lengths. - */ - odp_crypto_key_t auth_key; - - /** Authentication Initialization Vector (IV) - * - * Unless using the deprecated API, this specifies the length of - * the IV only. The actual IV must then be provided in per-packet - * parameters of crypto operations. - */ - union { -#if ODP_DEPRECATED_API - /** @deprecated Authentication IV */ - odp_crypto_iv_t ODP_DEPRECATE(auth_iv); -#endif - /** Authentication IV length */ - struct { -#if ODP_DEPRECATED_API - /** @cond - * Unused padding field - */ - uint8_t *dummy_padding_1; - /** @endcond */ -#endif - /** Length of authentication initialization vector. - * Default value is zero. - */ - uint32_t auth_iv_len; - }; - }; - - /** Authentication digest length in bytes - * - * Use odp_crypto_auth_capability() for supported digest lengths. - */ - uint32_t auth_digest_len; - - /** Additional Authenticated Data (AAD) length in bytes - * - * AAD length is constant for all operations (packets) of the session. - * Set to zero when AAD is not used. Use odp_crypto_auth_capability() - * for supported AAD lengths. The default value is zero. - */ - uint32_t auth_aad_len; - - /** Async mode completion event queue - * - * The completion queue is used to return completions from - * odp_crypto_op_enq() (and the deprecated odp_crypto_operation()) - * to the application. - */ - odp_queue_t compl_queue; - - /** Output pool - * - * When the output packet is not specified during the call to - * crypto operation, the output packet will be allocated - * from this pool. - */ - odp_pool_t output_pool; - -} odp_crypto_session_param_t; - -/** - * Crypto API per packet operation parameters - * - * @deprecated Use odp_crypto_packet_op_param_t instead. - */ -typedef struct odp_crypto_op_param_t { - /** Session handle from creation */ - odp_crypto_session_t session; - - /** User context */ - void *ctx; - - /** Input packet - * - * Specifies the input packet for the crypto operation. When the - * 'out_pkt' variable is set to ODP_PACKET_INVALID (indicating a new - * packet should be allocated for the resulting packet). - */ - odp_packet_t pkt; - - /** Output packet - * - * Both "in place" (the original packet 'pkt' is modified) and - * "copy" (the packet is replicated to a new packet which contains - * the modified data) modes are supported. The "in place" mode of - * operation is indicated by setting 'out_pkt' equal to 'pkt'. - * For the copy mode of operation, setting 'out_pkt' to a valid packet - * value indicates the caller wishes to specify the destination packet. - * Setting 'out_pkt' to ODP_PACKET_INVALID indicates the caller wishes - * the destination packet be allocated from the output pool specified - * during session creation. - */ - odp_packet_t out_pkt; - - /** IV pointer for cipher */ - uint8_t *cipher_iv_ptr; - - /** Authentication IV pointer */ - uint8_t *auth_iv_ptr; - - /** Offset from start of packet for hash result - * - * In case of decode sessions, the expected hash will be read from - * this offset and compared with the calculated hash. After the - * operation the hash bytes will have undefined values. - * - * In case of encode sessions the calculated hash will be stored in - * this offset. - * - * If the hash_result_in_auth_range session parameter is true, - * the hash result location may overlap auth_range. In that case - * the result location will be zeroed in decode sessions before - * hash calculation. Zeroing is not done in encode sessions. - */ - uint32_t hash_result_offset; - - /** Pointer to AAD. AAD length is defined by 'auth_aad_len' - * session parameter. - */ - uint8_t *aad_ptr; - - /** Data range to be ciphered */ - odp_packet_data_range_t cipher_range; - - /** Data range to be authenticated - * - * The value is ignored with authenticated encryption algorithms, - * such as AES-GCM, which authenticate data in the cipher range - * and the AAD. - * - * As a special case AES-GMAC uses this field instead of aad_ptr - * for the data bytes to be authenticated. - */ - odp_packet_data_range_t auth_range; - -} ODP_DEPRECATE(odp_crypto_op_param_t); - -/** - * Crypto packet API per packet operation parameters - */ -typedef struct odp_crypto_packet_op_param_t { - /** Session handle from creation */ - odp_crypto_session_t session; - - /** IV pointer for cipher */ - uint8_t *cipher_iv_ptr; - - /** IV pointer for authentication */ - uint8_t *auth_iv_ptr; - - /** Offset from start of packet for hash result - * - * In case of decode sessions, the expected hash will be read from - * this offset and compared with the calculated hash. After the - * operation the hash bytes will have undefined values. - * - * In case of encode sessions the calculated hash will be stored in - * this offset. - * - * If the hash_result_in_auth_range session parameter is true, - * the hash result location may overlap auth_range. In that case the - * result location will be zeroed in decode sessions before hash - * calculation. Zeroing is not done in encode sessions. - */ - uint32_t hash_result_offset; - - /** Pointer to AAD. AAD length is defined by 'auth_aad_len' - * session parameter. - */ - uint8_t *aad_ptr; - - /** Data range to apply cipher */ - odp_packet_data_range_t cipher_range; - - /** Data range to authenticate */ - odp_packet_data_range_t auth_range; - -} odp_crypto_packet_op_param_t; - -/** - * Crypto API session creation return code - */ -typedef enum { - /** Session created */ - ODP_CRYPTO_SES_ERR_NONE, - /** Creation failed, no resources */ - ODP_CRYPTO_SES_ERR_ENOMEM, - /** Creation failed, bad cipher params */ - ODP_CRYPTO_SES_ERR_CIPHER, - /** Creation failed, bad auth params */ - ODP_CRYPTO_SES_ERR_AUTH, - - /** Unsupported combination of algorithms - * - * The combination of cipher and auth algorithms with their - * specific parameters is not supported even if the algorithms - * appear in capabilities and are supported in combination with - * other algorithms or other algorithm specific parameters. - */ - ODP_CRYPTO_SES_ERR_ALG_COMBO, - - /** Unsupported order of cipher and auth - * - * The requested mutual order of ciphering and authentication - * is not supported with the chosen individual cipher and - * authentication algorithms. - */ - ODP_CRYPTO_SES_ERR_ALG_ORDER, - - /** Unsupported combination of session creation parameters - * - * The combination of provided session creation parameters is not - * supported. This error can occur when there are limitations that - * are not expressible through crypto capabilities or other error - * status values. - */ - ODP_CRYPTO_SES_ERR_PARAMS, -} odp_crypto_ses_create_err_t; - -/** This synonym for backward compatibility will be deprecated later */ -#define ODP_CRYPTO_SES_CREATE_ERR_NONE ODP_CRYPTO_SES_ERR_NONE -/** This synonym for backward compatibility will be deprecated later */ -#define ODP_CRYPTO_SES_CREATE_ERR_ENOMEM ODP_CRYPTO_SES_ERR_ENOMEM -/** This synonym for backward compatibility will be deprecated later */ -#define ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER ODP_CRYPTO_SES_ERR_CIPHER -/** This synonym for backward compatibility will be deprecated later */ -#define ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH ODP_CRYPTO_SES_ERR_AUTH - -/** - * Crypto API algorithm return code - */ -typedef enum { - /** Algorithm successful */ - ODP_CRYPTO_ALG_ERR_NONE, - /** Invalid data block size */ - ODP_CRYPTO_ALG_ERR_DATA_SIZE, - /** Key size invalid for algorithm */ - ODP_CRYPTO_ALG_ERR_KEY_SIZE, - /** Computed ICV value mismatch */ - ODP_CRYPTO_ALG_ERR_ICV_CHECK, - /** IV value not specified */ - ODP_CRYPTO_ALG_ERR_IV_INVALID, -} odp_crypto_alg_err_t; - -/** - * Crypto API hardware centric return code - */ -typedef enum { - /** Operation completed successfully */ - ODP_CRYPTO_HW_ERR_NONE, - /** Error detected during DMA of data */ - ODP_CRYPTO_HW_ERR_DMA, - /** Operation failed due to pool depletion */ - ODP_CRYPTO_HW_ERR_BP_DEPLETED, -} odp_crypto_hw_err_t; - -/** - * Cryto API per packet operation completion status - */ -typedef struct odp_crypto_op_status { - /** Algorithm specific return code */ - odp_crypto_alg_err_t alg_err; - - /** Hardware specific return code */ - odp_crypto_hw_err_t hw_err; - -} odp_crypto_op_status_t; - -/** - * Crypto API operation result - * - * @deprecated Use odp_crypto_packet_result_t instead. - */ -typedef struct odp_crypto_op_result { - /** Request completed successfully */ - odp_bool_t ok; - - /** User context from request */ - void *ctx; - - /** Output packet */ - odp_packet_t pkt; - - /** Cipher status */ - odp_crypto_op_status_t cipher_status; - - /** Authentication status */ - odp_crypto_op_status_t auth_status; - -} ODP_DEPRECATE(odp_crypto_op_result_t); - -/** - * Crypto packet API operation result - */ -typedef struct odp_crypto_packet_result_t { - /** Request completed successfully */ - odp_bool_t ok; - - /** Cipher status */ - odp_crypto_op_status_t cipher_status; - - /** Authentication status */ - odp_crypto_op_status_t auth_status; - -} odp_crypto_packet_result_t; - -/** - * Crypto capabilities - */ -typedef struct odp_crypto_capability_t { - /** Maximum number of crypto sessions */ - uint32_t max_sessions; - - /** Supported packet operation in SYNC mode */ - odp_support_t sync_mode; - - /** Supported packet operation in ASYNC mode */ - odp_support_t async_mode; - - /** Supported cipher algorithms */ - odp_crypto_cipher_algos_t ciphers; - - /** Cipher algorithms implemented with HW offload */ - odp_crypto_cipher_algos_t hw_ciphers; - - /** Supported authentication algorithms */ - odp_crypto_auth_algos_t auths; - - /** Authentication algorithms implemented with HW offload */ - odp_crypto_auth_algos_t hw_auths; - - /** - * Scheduled crypto completion queue support - * - * This defines whether scheduled queues are supported as crypto - * compl_queue. - * 0: Scheduled queues are not supported as crypto completion queues - * 1: Scheduled queues are supported as crypto completion queues - * @see odp_crypto_session_param_t - */ - odp_bool_t queue_type_sched; - - /** - * Plain crypto completion queue support - * - * This defines whether plain queues are supported as crypto - * compl_queue. - * 0: Plain queues are not supported as crypto completion queues - * 1: Plain queues are supported as crypto completion queues - * @see odp_crypto_session_param_t - */ - odp_bool_t queue_type_plain; -} odp_crypto_capability_t; - -/** - * Cipher algorithm capabilities - */ -typedef struct odp_crypto_cipher_capability_t { - /** Key length in bytes */ - uint32_t key_len; - - /** IV length in bytes */ - uint32_t iv_len; - - /** Cipher is operating in bitwise mode - * - * This cipher works on series of bits, rather than sequences of bytes: - * cipher_range in odp_crypto_op_param_t and - * odp_crypto_packet_op_param_t will use bits, rather than bytes. - * - * Note: data buffer MUST start on the byte boundary, using offset - * which is not divisible by 8 is unsupported and will result in - * unspecified behaviour. - * - * Note2: If the data length is not a multiple of 8, the remaining - * bits of the data in the last byte of the input/output will be the - * most significant bits, i.e. the most significant bit is considered - * to be the first bit of a byte for the purpose of input and output - * data range. The output bits that fall out of the output range are - * undefined. - */ - odp_bool_t bit_mode; - -} odp_crypto_cipher_capability_t; - -/** - * Authentication algorithm capabilities - */ -typedef struct odp_crypto_auth_capability_t { - /** Digest length in bytes */ - uint32_t digest_len; - - /** Key length in bytes */ - uint32_t key_len; - - /** IV length in bytes */ - uint32_t iv_len; - - /** Additional Authenticated Data (AAD) lengths */ - struct { - /** Minimum AAD length in bytes */ - uint32_t min; - - /** Maximum AAD length in bytes */ - uint32_t max; - - /** Increment of supported lengths between min and max - * (in bytes) */ - uint32_t inc; - } aad_len; - - /** Auth is operating in bitstring mode - * - * This auth works on series of bits, rather than sequences of bytes: - * auth_range in odp_crypto_op_param_t and - * odp_crypto_packet_op_param_t will use bits, rather than bytes. - * - * Note: data buffer MUST start on the byte boundary, using offset - * which is not divisible by 8 is unsupported and will result in - * unpredictable behaviour. - * - * Note2: If the data length is not a multiple of 8, the remaining - * bits of the data in the last byte of the input/output will be the - * most significant bits, i.e. the most significant bit is considered - * to be the first bit of a byte for the purpose of input and output - * data range. The output bits that fall out of the output range are - * undefined. - */ - odp_bool_t bit_mode; - -} odp_crypto_auth_capability_t; - /** * Query crypto capabilities * @@ -1336,7 +280,7 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, * from provided pool. All arrays should be of num_pkt size. * * @param pkt_in Packets to be processed - * @param[in,out] pkt_out Packet handle array specifyint resulting packets + * @param[in,out] pkt_out Packet handle array specifying resulting packets * @param param Operation parameters array * @param num_pkt Number of packets to be processed * diff --git a/include/odp/api/spec/crypto_types.h b/include/odp/api/spec/crypto_types.h new file mode 100644 index 000000000..7f2c49a7d --- /dev/null +++ b/include/odp/api/spec/crypto_types.h @@ -0,0 +1,1095 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP crypto types */ + +#ifndef ODP_API_SPEC_CRYPTO_TYPES_H_ +#define ODP_API_SPEC_CRYPTO_TYPES_H_ +#include + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup odp_crypto + * @{ + */ + +/** + * @def ODP_CRYPTO_SESSION_INVALID + * Invalid session handle + */ + +/** + * @typedef odp_crypto_session_t + * Crypto API opaque session handle + */ + +/** + * @typedef odp_crypto_compl_t +* @deprecated Crypto API completion event (platform dependent). +*/ + +/** + * Crypto API operation mode + */ +typedef enum { + /** Synchronous, return results immediately */ + ODP_CRYPTO_SYNC, + /** Asynchronous, return results via posted event */ + ODP_CRYPTO_ASYNC, +} odp_crypto_op_mode_t; + +/** + * Crypto API operation type + */ +typedef enum { + /** Encrypt and/or compute authentication ICV */ + ODP_CRYPTO_OP_ENCODE, + /** Decrypt and/or verify authentication ICV */ + ODP_CRYPTO_OP_DECODE, +} odp_crypto_op_t; + +/** + * Crypto API cipher algorithm + */ +typedef enum { + /** No cipher algorithm specified */ + ODP_CIPHER_ALG_NULL, + + /** DES */ + ODP_CIPHER_ALG_DES, + + /** Triple DES with cipher block chaining */ + ODP_CIPHER_ALG_3DES_CBC, + + /** Triple DES with Electronic Codebook */ + ODP_CIPHER_ALG_3DES_ECB, + + /** AES with cipher block chaining */ + ODP_CIPHER_ALG_AES_CBC, + + /** AES with counter mode */ + ODP_CIPHER_ALG_AES_CTR, + + /** AES with electronic codebook */ + ODP_CIPHER_ALG_AES_ECB, + + /** AES with 128-bit cipher feedback */ + ODP_CIPHER_ALG_AES_CFB128, + + /** AES with XEX-based tweaked-codebook mode with ciphertext stealing + * (XTS) */ + ODP_CIPHER_ALG_AES_XTS, + + /** AES-GCM + * + * AES in Galois/Counter Mode (GCM) algorithm. GCM provides both + * authentication and ciphering of data (authenticated encryption) + * in the same operation. Hence this algorithm must be paired always + * with ODP_AUTH_ALG_AES_GCM authentication. + */ + ODP_CIPHER_ALG_AES_GCM, + + /** AES-CCM + * + * AES in Counter with CBC-MAC (CCM) mode algorithm. CCM provides both + * authentication and ciphering of data (authenticated encryption) + * in the same operation. Hence this algorithm must be paired always + * with ODP_AUTH_ALG_AES_CCM authentication. + */ + ODP_CIPHER_ALG_AES_CCM, + + /** ChaCha20-Poly1305 + * + * ChaCha20 with Poly1305 provide both authentication and ciphering of + * data (authenticated encryption) in the same operation. Hence this + * algorithm must be paired always with ODP_AUTH_ALG_CHACHA20_POLY1305 + * authentication. + */ + ODP_CIPHER_ALG_CHACHA20_POLY1305, + + /** Confidentiality F8 algorithm (UEA1) + * + * KASUMI-based F8 algorithm (also known as UEA1). + * + * IV should be formatted according to the 3GPP TS 35.201: + * COUNT || BEARER || DIRECTION || 0...0 + */ + ODP_CIPHER_ALG_KASUMI_F8, + + /** Confidentiality UEA2 algorithm (128-EEA1) + * + * SNOW 3G-based UEA2 algorithm (also known as 128-EEA1). + * + * IV (128 bit) should be formatted according to the ETSI/SAGE + * UEA2 & UIA2 specification: + * COUNT || BEARER || DIRECTION || 0...0 || + * COUNT || BEARER || DIRECTION || 0...0 || + */ + ODP_CIPHER_ALG_SNOW3G_UEA2, + + /** Confidentiality 128-EEA2 algorithm + * + * AES-CTR-based 128-EEA2 algorithm. + * + * IV (128 bit) should be formatted according to the ETSI/SAGE + * 128-EA2 & 128-EIA2 specification: + * COUNT || BEARER || + * DIRECTION || 0....0 + */ + ODP_CIPHER_ALG_AES_EEA2, + + /** ZUC based confidentiality algorithm + * + * 128-EEA3/128-NEA3 algorithm when key length is 128 bits. + * + * IV (128 bit) should be formatted according to the ETSI/SAGE + * 128-EEA3 & 128-EIA3 specification: + * COUNT || BEARER || DIRECTION || 0...0 || + * COUNT || BEARER || DIRECTION || 0...0 || + * + * 256-bit key length support is experimental and subject to + * change. The following variants may be supported: + * + * - ZUC-256 with 25 byte IV (of which 184 bits are variable) + * as specified in "The ZUC-256 Stream Cipher". + * - ZUC-256 with 16 byte IV as specified in + * "An Addendum to the ZUC-256 Stream Cipher", + * https://eprint.iacr.org/2021/1439 + */ + ODP_CIPHER_ALG_ZUC_EEA3, + +} odp_cipher_alg_t; + +/** + * Crypto API authentication algorithm + */ +typedef enum { + /** No authentication algorithm specified */ + ODP_AUTH_ALG_NULL, + + /** HMAC-MD5 + * + * MD5 algorithm in HMAC mode + */ + ODP_AUTH_ALG_MD5_HMAC, + + /** HMAC-SHA-1 + * + * SHA-1 algorithm in HMAC mode + */ + ODP_AUTH_ALG_SHA1_HMAC, + + /** HMAC-SHA-224 + * + * SHA-224 algorithm in HMAC mode + */ + ODP_AUTH_ALG_SHA224_HMAC, + + /** HMAC-SHA-256 + * + * SHA-256 algorithm in HMAC mode + */ + ODP_AUTH_ALG_SHA256_HMAC, + + /** HMAC-SHA-384 + * + * SHA-384 algorithm in HMAC mode + */ + ODP_AUTH_ALG_SHA384_HMAC, + + /** HMAC-SHA-512 + * + * SHA-512 algorithm in HMAC mode + */ + ODP_AUTH_ALG_SHA512_HMAC, + + /** AES-GCM + * + * AES in Galois/Counter Mode (GCM) algorithm. GCM provides both + * authentication and ciphering of data (authenticated encryption) + * in the same operation. Hence this algorithm must be paired always + * with ODP_CIPHER_ALG_AES_GCM cipher. + */ + ODP_AUTH_ALG_AES_GCM, + + /** AES-GMAC + * + * AES Galois Message Authentication Code (GMAC) algorithm. AES-GMAC + * is based on AES-GCM operation, but provides authentication only. + * Hence this algorithm can be paired only with ODP_CIPHER_ALG_NULL + * cipher. + * + * NIST and RFC specifications of GMAC refer to all data to be + * authenticated as AAD. In ODP the data to be authenticated, i.e. + * AAD, is ODP packet data and specified using the auth_range + * parameter. The aad_length and aad_ptr parameters, which would + * require the data to be contiguous in memory, are ignored with + * AES-GMAC. + * + * GMAC needs an initialization vector, which can be passed via + * session (auth_iv) or packet (auth_iv_ptr) level parameters. + */ + ODP_AUTH_ALG_AES_GMAC, + + /** AES-CCM + * + * AES in Counter with CBC-MAC (CCM) mode algorithm. CCM provides both + * authentication and ciphering of data (authenticated encryption) + * in the same operation. Hence this algorithm must be paired always + * with ODP_CIPHER_ALG_AES_CCM cipher. + */ + ODP_AUTH_ALG_AES_CCM, + + /** AES-CMAC + * + * AES Cipher-based Message Authentication Code (CMAC) algorithm. CMAC + * is a keyed hash function that is based on a symmetric key block + * cipher, such as the AES. + */ + ODP_AUTH_ALG_AES_CMAC, + + /** AES-XCBC-MAC + * + * AES CBC MAC for arbitrary-length messages (XCBC-MAC). + * + */ + ODP_AUTH_ALG_AES_XCBC_MAC, + + /** ChaCha20-Poly1305 AEAD + * + * ChaCha20 with Poly1305 provide both authentication and ciphering of + * data (authenticated encryption) in the same operation. Hence this + * algorithm must be paired always with + * ODP_CIPHER_ALG_CHACHA20_POLY1305 cipher. + */ + ODP_AUTH_ALG_CHACHA20_POLY1305, + + /** Integrity F9 algorithm (UIA1) + * + * KASUMI-based F9 algorithm (also known as UIA1). + * + * IV (9 bytes) is a concatenation of COUNT (32b), FRESH (32b) and + * DIRECTION (LSB-aligned, 1b). + * IV (8 bytes) is a concatenation of COUNT (32b) and FRESH (32b) + * DIRECTION (1b) and padding should come at the end of message. + */ + ODP_AUTH_ALG_KASUMI_F9, + + /** Integrity UIA2 algorithm (128-EIA1) + * + * SNOW 3G-based UIA2 algorithm (also known as 128-EIA1). + * IV (128 bit) should be formatted according to the ETSI/SAGE + * UEA2 & UIA2 specification: + * COUNT || FRESH || + * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 || + * FRESH0 .. FRESH15 || FRESH16 XOR DIRECTION || FRESH17 .. FRESH31 + */ + ODP_AUTH_ALG_SNOW3G_UIA2, + + /** Integrity 128-EIA2 algorithm + * + * AES_CMAC-based 128-EIA2 algorithm. + * + * IV (128 bit) should be formatted according to the ETSI/SAGE + * 128-EA2 & 128-EIA2 specification: + * COUNT || BEARER || + * DIRECTION || 0....0 + */ + ODP_AUTH_ALG_AES_EIA2, + + /** ZUC-based integrity algorithm. + * + * 128-EIA3/128-NIA3 algorithm when key length is 128 bits. + * + * IV (128 bit) should be formatted according to the ETSI/SAGE + * 128-EA3 & 128-EIA2 specification: + * COUNT || BEARER || + * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 || + * BEARER || 0...0 || DIRECTION || 0...0 + * + * 256-bit key length support is experimental and subject to + * change. The following variants may be supported: + * + * - ZUC-256 with 25 byte IV (of which 184 bits are variable) and + * 32/64/128 bit MAC as specified in "The ZUC-256 Stream Cipher". + * - ZUC-256 with 16 byte IV and 32/64/128 bit MAC as specified in + * "An Addendum to the ZUC-256 Stream Cipher", + * https://eprint.iacr.org/2021/1439 + */ + ODP_AUTH_ALG_ZUC_EIA3, + + /** MD5 algorithm */ + ODP_AUTH_ALG_MD5, + + /** SHA1 algorithm */ + ODP_AUTH_ALG_SHA1, + + /** 224 bit SHA2 algorithm */ + ODP_AUTH_ALG_SHA224, + + /** 256 bit SHA2 algorithm */ + ODP_AUTH_ALG_SHA256, + + /** 384 bit SHA2 algorithm */ + ODP_AUTH_ALG_SHA384, + + /** 512 bit SHA2 algorithm */ + ODP_AUTH_ALG_SHA512, + +} odp_auth_alg_t; + +/** + * Cipher algorithms in a bit field structure + */ +typedef union odp_crypto_cipher_algos_t { + /** Cipher algorithms */ + struct { + /** ODP_CIPHER_ALG_NULL */ + uint32_t null : 1; + + /** ODP_CIPHER_ALG_DES */ + uint32_t des : 1; + + /** ODP_CIPHER_ALG_3DES_CBC */ + uint32_t trides_cbc : 1; + + /** ODP_CIPHER_ALG_3DES_ECB */ + uint32_t trides_ecb : 1; + + /** ODP_CIPHER_ALG_AES_CBC */ + uint32_t aes_cbc : 1; + + /** ODP_CIPHER_ALG_AES_CTR */ + uint32_t aes_ctr : 1; + + /** ODP_CIPHER_ALG_AES_ECB */ + uint32_t aes_ecb : 1; + + /** ODP_CIPHER_ALG_AES_CFB128 */ + uint32_t aes_cfb128 : 1; + + /** ODP_CIPHER_ALG_AES_XTS */ + uint32_t aes_xts : 1; + + /** ODP_CIPHER_ALG_AES_GCM */ + uint32_t aes_gcm : 1; + + /** ODP_CIPHER_ALG_AES_CCM */ + uint32_t aes_ccm : 1; + + /** ODP_CIPHER_ALG_CHACHA20_POLY1305 */ + uint32_t chacha20_poly1305 : 1; + + /** ODP_CIPHER_ALG_KASUMI_F8 */ + uint32_t kasumi_f8 : 1; + + /** ODP_CIPHER_ALG_SNOW3G_UEA2 */ + uint32_t snow3g_uea2 : 1; + + /** ODP_CIPHER_ALG_AES_EEA2 */ + uint32_t aes_eea2 : 1; + + /** ODP_CIPHER_ALG_ZUC_EEA3 */ + uint32_t zuc_eea3 : 1; + + } bit; + + /** All bits of the bit field structure + * + * This field can be used to set/clear all flags, or bitwise + * operations over the entire structure. */ + uint32_t all_bits; +} odp_crypto_cipher_algos_t; + +/** + * Authentication algorithms in a bit field structure + */ +typedef union odp_crypto_auth_algos_t { + /** Authentication algorithms */ + struct { + /** ODP_AUTH_ALG_NULL */ + uint32_t null : 1; + + /** ODP_AUTH_ALG_MD5_HMAC */ + uint32_t md5_hmac : 1; + + /** ODP_AUTH_ALG_SHA1_HMAC */ + uint32_t sha1_hmac : 1; + + /** ODP_AUTH_ALG_SHA224_HMAC */ + uint32_t sha224_hmac : 1; + + /** ODP_AUTH_ALG_SHA256_HMAC */ + uint32_t sha256_hmac : 1; + + /** ODP_AUTH_ALG_SHA384_HMAC */ + uint32_t sha384_hmac : 1; + + /** ODP_AUTH_ALG_SHA512_HMAC */ + uint32_t sha512_hmac : 1; + + /** ODP_AUTH_ALG_AES_GCM */ + uint32_t aes_gcm : 1; + + /** ODP_AUTH_ALG_AES_GMAC*/ + uint32_t aes_gmac : 1; + + /** ODP_AUTH_ALG_AES_CCM */ + uint32_t aes_ccm : 1; + + /** ODP_AUTH_ALG_AES_CMAC*/ + uint32_t aes_cmac : 1; + + /** ODP_AUTH_ALG_AES_XCBC_MAC*/ + uint32_t aes_xcbc_mac : 1; + + /** ODP_AUTH_ALG_CHACHA20_POLY1305 */ + uint32_t chacha20_poly1305 : 1; + + /** ODP_AUTH_ALG_KASUMI_F9 */ + uint32_t kasumi_f9 : 1; + + /** ODP_AUTH_ALG_SNOW3G_UIA2 */ + uint32_t snow3g_uia2 : 1; + + /** ODP_AUTH_ALG_AES_EIA2 */ + uint32_t aes_eia2 : 1; + + /** ODP_AUTH_ALG_ZUC_EIA3 */ + uint32_t zuc_eia3 : 1; + + /** ODP_AUTH_ALG_MD5 */ + uint32_t md5 : 1; + + /** ODP_AUTH_ALG_SHA1 */ + uint32_t sha1 : 1; + + /** ODP_AUTH_ALG_SHA224 */ + uint32_t sha224 : 1; + + /** ODP_AUTH_ALG_SHA256 */ + uint32_t sha256 : 1; + + /** ODP_AUTH_ALG_SHA384 */ + uint32_t sha384 : 1; + + /** ODP_AUTH_ALG_SHA512 */ + uint32_t sha512 : 1; + + } bit; + + /** All bits of the bit field structure + * + * This field can be used to set/clear all flags, or bitwise + * operations over the entire structure. */ + uint32_t all_bits; +} odp_crypto_auth_algos_t; + +/** + * Crypto API key structure + */ +typedef struct odp_crypto_key { + /** Key data */ + uint8_t *data; + + /** Key length in bytes */ + uint32_t length; + +} odp_crypto_key_t; + +/** + * Crypto API IV structure + * + * @deprecated Use per-packet IV in crypto operation parameters + */ +typedef struct odp_crypto_iv { + /** IV data + * + * Ignored when length is zero. Null value indicates that an + * IV will be provided for each packet through the crypto + * operation parameters. In that case the per-operation + * IV parameter must always point to a valid IV. + * + * Default value is NULL. + */ + uint8_t *data; + + /** IV length in bytes. Default value is zero. */ + uint32_t length; + +} ODP_DEPRECATE(odp_crypto_iv_t); + +/** + * Crypto API session creation parameters + */ +typedef struct odp_crypto_session_param_t { + /** Encode vs. decode operation + * + * The default value is ODP_CRYPTO_OP_ENCODE. + */ + odp_crypto_op_t op; + + /** Authenticate cipher vs. plain text + * + * Controls ordering of authentication and cipher operations, + * and is relative to the operation (encode vs decode). When encoding, + * TRUE indicates the authentication operation should be performed + * after the cipher operation else before. When decoding, TRUE + * indicates the reverse order of operation. + * + * The value is ignored with authenticated encryption algorithms + * such as AES-GCM. + * + * true: Authenticate cipher text + * false: Authenticate plain text + * + * The default value is false. + */ + odp_bool_t auth_cipher_text; + + /** Hash result location may overlap authentication range + * + * This flag indicates that the hash result location may (but is + * not required to) overlap authentication range. Setting this + * flag may reduce performance. + * + * Default value is false. + */ + odp_bool_t hash_result_in_auth_range; + + /** Preferred sync vs. async for odp_crypto_operation() + * + * The default value is ODP_CRYPTO_SYNC. + * + * @deprecated Used only with deprecated odp_crypto_operation() + */ + odp_crypto_op_mode_t ODP_DEPRECATE(pref_mode); + + /** Operation mode when using packet interface: sync or async + * + * The default value is ODP_CRYPTO_SYNC. + */ + odp_crypto_op_mode_t op_mode; + + /** Cipher algorithm + * + * Select cipher algorithm to be used. ODP_CIPHER_ALG_NULL indicates + * that ciphering is disabled. Use odp_crypto_capability() for + * supported algorithms. Note that some algorithms restrict choice of + * the pairing authentication algorithm. When ciphering is enabled + * cipher key and IV need to be set. The default value is + * ODP_CIPHER_ALG_NULL. + */ + odp_cipher_alg_t cipher_alg; + + /** Cipher key + * + * Use odp_crypto_cipher_capa() for supported key and IV lengths. + */ + odp_crypto_key_t cipher_key; + + /** Cipher Initialization Vector (IV) + * + * Unless using the deprecated API, this specifies the length of + * the IV only. The actual IV must then be provided in per-packet + * parameters of crypto operations. + */ + union { +#if ODP_DEPRECATED_API + /** @deprecated Cipher IV */ + odp_crypto_iv_t ODP_DEPRECATE(cipher_iv); +#endif + /** Cipher IV length */ + struct { +#if ODP_DEPRECATED_API + /** @cond + * Unused padding field + */ + uint8_t *dummy_padding_0; + /** @endcond */ +#endif + /** Length of cipher initialization vector. + * Default value is zero. + */ + uint32_t cipher_iv_len; + }; + }; + + /** Authentication algorithm + * + * Select authentication algorithm to be used. ODP_AUTH_ALG_NULL + * indicates that authentication is disabled. Use + * odp_crypto_capability() for supported algorithms. Note that some + * algorithms restrict choice of the pairing cipher algorithm. When + * single algorithm provides both ciphering and authentication + * (i.e. Authenticated Encryption), authentication side key + * (auth_key) and IV (auth_iv) are ignored, and cipher side values are + * used instead. These algorithms ignore authentication side key + * and IV: ODP_AUTH_ALG_AES_GCM, ODP_AUTH_ALG_AES_CCM and + * ODP_AUTH_ALG_CHACHA20_POLY1305. Otherwise, all authentication side + * parameters must be set when authentication is enabled. The default + * value is ODP_AUTH_ALG_NULL. + */ + odp_auth_alg_t auth_alg; + + /** Authentication key + * + * Use odp_crypto_auth_capability() for supported key lengths. + */ + odp_crypto_key_t auth_key; + + /** Authentication Initialization Vector (IV) + * + * Unless using the deprecated API, this specifies the length of + * the IV only. The actual IV must then be provided in per-packet + * parameters of crypto operations. + */ + union { +#if ODP_DEPRECATED_API + /** @deprecated Authentication IV */ + odp_crypto_iv_t ODP_DEPRECATE(auth_iv); +#endif + /** Authentication IV length */ + struct { +#if ODP_DEPRECATED_API + /** @cond + * Unused padding field + */ + uint8_t *dummy_padding_1; + /** @endcond */ +#endif + /** Length of authentication initialization vector. + * Default value is zero. + */ + uint32_t auth_iv_len; + }; + }; + + /** Authentication digest length in bytes + * + * Use odp_crypto_auth_capability() for supported digest lengths. + */ + uint32_t auth_digest_len; + + /** Additional Authenticated Data (AAD) length in bytes + * + * AAD length is constant for all operations (packets) of the session. + * Set to zero when AAD is not used. Use odp_crypto_auth_capability() + * for supported AAD lengths. The default value is zero. + */ + uint32_t auth_aad_len; + + /** Async mode completion event queue + * + * The completion queue is used to return completions from + * odp_crypto_op_enq() (and the deprecated odp_crypto_operation()) + * to the application. + */ + odp_queue_t compl_queue; + + /** Output pool + * + * When the output packet is not specified during the call to + * crypto operation, the output packet will be allocated + * from this pool. + */ + odp_pool_t output_pool; + +} odp_crypto_session_param_t; + +/** + * Crypto API per packet operation parameters + * + * @deprecated Use odp_crypto_packet_op_param_t instead. + */ +typedef struct odp_crypto_op_param_t { + /** Session handle from creation */ + odp_crypto_session_t session; + + /** User context */ + void *ctx; + + /** Input packet + * + * Specifies the input packet for the crypto operation. When the + * 'out_pkt' variable is set to ODP_PACKET_INVALID (indicating a new + * packet should be allocated for the resulting packet). + */ + odp_packet_t pkt; + + /** Output packet + * + * Both "in place" (the original packet 'pkt' is modified) and + * "copy" (the packet is replicated to a new packet which contains + * the modified data) modes are supported. The "in place" mode of + * operation is indicated by setting 'out_pkt' equal to 'pkt'. + * For the copy mode of operation, setting 'out_pkt' to a valid packet + * value indicates the caller wishes to specify the destination packet. + * Setting 'out_pkt' to ODP_PACKET_INVALID indicates the caller wishes + * the destination packet be allocated from the output pool specified + * during session creation. + */ + odp_packet_t out_pkt; + + /** IV pointer for cipher */ + uint8_t *cipher_iv_ptr; + + /** Authentication IV pointer */ + uint8_t *auth_iv_ptr; + + /** Offset from start of packet for hash result + * + * In case of decode sessions, the expected hash will be read from + * this offset and compared with the calculated hash. After the + * operation the hash bytes will have undefined values. + * + * In case of encode sessions the calculated hash will be stored in + * this offset. + * + * If the hash_result_in_auth_range session parameter is true, + * the hash result location may overlap auth_range. In that case + * the result location will be zeroed in decode sessions before + * hash calculation. Zeroing is not done in encode sessions. + */ + uint32_t hash_result_offset; + + /** Pointer to AAD. AAD length is defined by 'auth_aad_len' + * session parameter. + */ + uint8_t *aad_ptr; + + /** Data range to be ciphered */ + odp_packet_data_range_t cipher_range; + + /** Data range to be authenticated + * + * The value is ignored with authenticated encryption algorithms, + * such as AES-GCM, which authenticate data in the cipher range + * and the AAD. + * + * As a special case AES-GMAC uses this field instead of aad_ptr + * for the data bytes to be authenticated. + */ + odp_packet_data_range_t auth_range; + +} ODP_DEPRECATE(odp_crypto_op_param_t); + +/** + * Crypto packet API per packet operation parameters + */ +typedef struct odp_crypto_packet_op_param_t { + /** Session handle from creation */ + odp_crypto_session_t session; + + /** IV pointer for cipher */ + uint8_t *cipher_iv_ptr; + + /** IV pointer for authentication */ + uint8_t *auth_iv_ptr; + + /** Offset from start of packet for hash result + * + * In case of decode sessions, the expected hash will be read from + * this offset and compared with the calculated hash. After the + * operation the hash bytes will have undefined values. + * + * In case of encode sessions the calculated hash will be stored in + * this offset. + * + * If the hash_result_in_auth_range session parameter is true, + * the hash result location may overlap auth_range. In that case the + * result location will be zeroed in decode sessions before hash + * calculation. Zeroing is not done in encode sessions. + */ + uint32_t hash_result_offset; + + /** Pointer to AAD. AAD length is defined by 'auth_aad_len' + * session parameter. + */ + uint8_t *aad_ptr; + + /** Data range to apply cipher */ + odp_packet_data_range_t cipher_range; + + /** Data range to authenticate */ + odp_packet_data_range_t auth_range; + +} odp_crypto_packet_op_param_t; + +/** + * Crypto API session creation return code + */ +typedef enum { + /** Session created */ + ODP_CRYPTO_SES_ERR_NONE, + /** Creation failed, no resources */ + ODP_CRYPTO_SES_ERR_ENOMEM, + /** Creation failed, bad cipher params */ + ODP_CRYPTO_SES_ERR_CIPHER, + /** Creation failed, bad auth params */ + ODP_CRYPTO_SES_ERR_AUTH, + + /** Unsupported combination of algorithms + * + * The combination of cipher and auth algorithms with their + * specific parameters is not supported even if the algorithms + * appear in capabilities and are supported in combination with + * other algorithms or other algorithm specific parameters. + */ + ODP_CRYPTO_SES_ERR_ALG_COMBO, + + /** Unsupported order of cipher and auth + * + * The requested mutual order of ciphering and authentication + * is not supported with the chosen individual cipher and + * authentication algorithms. + */ + ODP_CRYPTO_SES_ERR_ALG_ORDER, + + /** Unsupported combination of session creation parameters + * + * The combination of provided session creation parameters is not + * supported. This error can occur when there are limitations that + * are not expressible through crypto capabilities or other error + * status values. + */ + ODP_CRYPTO_SES_ERR_PARAMS, +} odp_crypto_ses_create_err_t; + +/** This synonym for backward compatibility will be deprecated later */ +#define ODP_CRYPTO_SES_CREATE_ERR_NONE ODP_CRYPTO_SES_ERR_NONE +/** This synonym for backward compatibility will be deprecated later */ +#define ODP_CRYPTO_SES_CREATE_ERR_ENOMEM ODP_CRYPTO_SES_ERR_ENOMEM +/** This synonym for backward compatibility will be deprecated later */ +#define ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER ODP_CRYPTO_SES_ERR_CIPHER +/** This synonym for backward compatibility will be deprecated later */ +#define ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH ODP_CRYPTO_SES_ERR_AUTH + +/** + * Crypto API algorithm return code + */ +typedef enum { + /** Algorithm successful */ + ODP_CRYPTO_ALG_ERR_NONE, + /** Invalid data block size */ + ODP_CRYPTO_ALG_ERR_DATA_SIZE, + /** Key size invalid for algorithm */ + ODP_CRYPTO_ALG_ERR_KEY_SIZE, + /** Computed ICV value mismatch */ + ODP_CRYPTO_ALG_ERR_ICV_CHECK, + /** IV value not specified */ + ODP_CRYPTO_ALG_ERR_IV_INVALID, +} odp_crypto_alg_err_t; + +/** + * Crypto API hardware centric return code + */ +typedef enum { + /** Operation completed successfully */ + ODP_CRYPTO_HW_ERR_NONE, + /** Error detected during DMA of data */ + ODP_CRYPTO_HW_ERR_DMA, + /** Operation failed due to pool depletion */ + ODP_CRYPTO_HW_ERR_BP_DEPLETED, +} odp_crypto_hw_err_t; + +/** + * Cryto API per packet operation completion status + */ +typedef struct odp_crypto_op_status { + /** Algorithm specific return code */ + odp_crypto_alg_err_t alg_err; + + /** Hardware specific return code */ + odp_crypto_hw_err_t hw_err; + +} odp_crypto_op_status_t; + +/** + * Crypto API operation result + * + * @deprecated Use odp_crypto_packet_result_t instead. + */ +typedef struct odp_crypto_op_result { + /** Request completed successfully */ + odp_bool_t ok; + + /** User context from request */ + void *ctx; + + /** Output packet */ + odp_packet_t pkt; + + /** Cipher status */ + odp_crypto_op_status_t cipher_status; + + /** Authentication status */ + odp_crypto_op_status_t auth_status; + +} ODP_DEPRECATE(odp_crypto_op_result_t); + +/** + * Crypto packet API operation result + */ +typedef struct odp_crypto_packet_result_t { + /** Request completed successfully */ + odp_bool_t ok; + + /** Cipher status */ + odp_crypto_op_status_t cipher_status; + + /** Authentication status */ + odp_crypto_op_status_t auth_status; + +} odp_crypto_packet_result_t; + +/** + * Crypto capabilities + */ +typedef struct odp_crypto_capability_t { + /** Maximum number of crypto sessions */ + uint32_t max_sessions; + + /** Supported packet operation in SYNC mode */ + odp_support_t sync_mode; + + /** Supported packet operation in ASYNC mode */ + odp_support_t async_mode; + + /** Supported cipher algorithms */ + odp_crypto_cipher_algos_t ciphers; + + /** Cipher algorithms implemented with HW offload */ + odp_crypto_cipher_algos_t hw_ciphers; + + /** Supported authentication algorithms */ + odp_crypto_auth_algos_t auths; + + /** Authentication algorithms implemented with HW offload */ + odp_crypto_auth_algos_t hw_auths; + + /** + * Scheduled crypto completion queue support + * + * This defines whether scheduled queues are supported as crypto + * compl_queue. + * 0: Scheduled queues are not supported as crypto completion queues + * 1: Scheduled queues are supported as crypto completion queues + * @see odp_crypto_session_param_t + */ + odp_bool_t queue_type_sched; + + /** + * Plain crypto completion queue support + * + * This defines whether plain queues are supported as crypto + * compl_queue. + * 0: Plain queues are not supported as crypto completion queues + * 1: Plain queues are supported as crypto completion queues + * @see odp_crypto_session_param_t + */ + odp_bool_t queue_type_plain; +} odp_crypto_capability_t; + +/** + * Cipher algorithm capabilities + */ +typedef struct odp_crypto_cipher_capability_t { + /** Key length in bytes */ + uint32_t key_len; + + /** IV length in bytes */ + uint32_t iv_len; + + /** Cipher is operating in bitwise mode + * + * This cipher works on series of bits, rather than sequences of bytes: + * cipher_range in odp_crypto_op_param_t and + * odp_crypto_packet_op_param_t will use bits, rather than bytes. + * + * Note: data buffer MUST start on the byte boundary, using offset + * which is not divisible by 8 is unsupported and will result in + * unspecified behaviour. + * + * Note2: If the data length is not a multiple of 8, the remaining + * bits of the data in the last byte of the input/output will be the + * most significant bits, i.e. the most significant bit is considered + * to be the first bit of a byte for the purpose of input and output + * data range. The output bits that fall out of the output range are + * undefined. + */ + odp_bool_t bit_mode; + +} odp_crypto_cipher_capability_t; + +/** + * Authentication algorithm capabilities + */ +typedef struct odp_crypto_auth_capability_t { + /** Digest length in bytes */ + uint32_t digest_len; + + /** Key length in bytes */ + uint32_t key_len; + + /** IV length in bytes */ + uint32_t iv_len; + + /** Additional Authenticated Data (AAD) lengths */ + struct { + /** Minimum AAD length in bytes */ + uint32_t min; + + /** Maximum AAD length in bytes */ + uint32_t max; + + /** Increment of supported lengths between min and max + * (in bytes) */ + uint32_t inc; + } aad_len; + + /** Auth is operating in bitstring mode + * + * This auth works on series of bits, rather than sequences of bytes: + * auth_range in odp_crypto_op_param_t and + * odp_crypto_packet_op_param_t will use bits, rather than bytes. + * + * Note: data buffer MUST start on the byte boundary, using offset + * which is not divisible by 8 is unsupported and will result in + * unpredictable behaviour. + * + * Note2: If the data length is not a multiple of 8, the remaining + * bits of the data in the last byte of the input/output will be the + * most significant bits, i.e. the most significant bit is considered + * to be the first bit of a byte for the purpose of input and output + * data range. The output bits that fall out of the output range are + * undefined. + */ + odp_bool_t bit_mode; + +} odp_crypto_auth_capability_t; + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include +#endif diff --git a/include/odp/api/spec/ipsec.h b/include/odp/api/spec/ipsec.h index b091961cc..603bf3aef 100644 --- a/include/odp/api/spec/ipsec.h +++ b/include/odp/api/spec/ipsec.h @@ -19,7 +19,7 @@ extern "C" { #endif -#include +#include #include #include #include diff --git a/include/odp/api/spec/ipsec_types.h b/include/odp/api/spec/ipsec_types.h index b74d26d4f..b34fb5982 100644 --- a/include/odp/api/spec/ipsec_types.h +++ b/include/odp/api/spec/ipsec_types.h @@ -20,7 +20,7 @@ extern "C" { #endif #include -#include +#include #include #include #include diff --git a/include/odp/arch/arm32-linux/odp/api/abi/crypto_types.h b/include/odp/arch/arm32-linux/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..ddadec1c1 --- /dev/null +++ b/include/odp/arch/arm32-linux/odp/api/abi/crypto_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/include/odp/arch/arm64-linux/odp/api/abi/crypto_types.h b/include/odp/arch/arm64-linux/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..ddadec1c1 --- /dev/null +++ b/include/odp/arch/arm64-linux/odp/api/abi/crypto_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/include/odp/arch/default-linux/odp/api/abi/crypto_types.h b/include/odp/arch/default-linux/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..ddadec1c1 --- /dev/null +++ b/include/odp/arch/default-linux/odp/api/abi/crypto_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/include/odp/arch/power64-linux/odp/api/abi/crypto_types.h b/include/odp/arch/power64-linux/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..ddadec1c1 --- /dev/null +++ b/include/odp/arch/power64-linux/odp/api/abi/crypto_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/crypto_types.h b/include/odp/arch/x86_32-linux/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..ddadec1c1 --- /dev/null +++ b/include/odp/arch/x86_32-linux/odp/api/abi/crypto_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/crypto_types.h b/include/odp/arch/x86_64-linux/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..ddadec1c1 --- /dev/null +++ b/include/odp/arch/x86_64-linux/odp/api/abi/crypto_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 3310e9603..7bc48edd6 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -75,6 +75,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/comp.h \ include-abi/odp/api/abi/cpumask.h \ include-abi/odp/api/abi/crypto.h \ + include-abi/odp/api/abi/crypto_types.h \ include-abi/odp/api/abi/debug.h \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ diff --git a/platform/linux-generic/include-abi/odp/api/abi/crypto.h b/platform/linux-generic/include-abi/odp/api/abi/crypto.h index b57667e76..24019db12 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/crypto.h +++ b/platform/linux-generic/include-abi/odp/api/abi/crypto.h @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,18 +19,7 @@ extern "C" { #endif -/** @ingroup odp_crypto - * @{ - */ - -#define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL) - -typedef uint64_t odp_crypto_session_t; -typedef ODP_HANDLE_T(odp_crypto_compl_t); - -/** - * @} - */ +/* Placeholder for inlined API functions */ #ifdef __cplusplus } diff --git a/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h b/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h new file mode 100644 index 000000000..a5cb43c5d --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h @@ -0,0 +1,42 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP crypto + */ + +#ifndef ODP_API_ABI_CRYPTO_TYPES_H_ +#define ODP_API_ABI_CRYPTO_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include + +/** @ingroup odp_crypto + * @{ + */ + +#define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL) + +typedef uint64_t odp_crypto_session_t; +typedef ODP_HANDLE_T(odp_crypto_compl_t); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 1b3a90e6a..9bb89ad8b 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include -- cgit v1.2.3 From e2eb952cc47843f453edd01bc3b153feeff5ef68 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 28 Oct 2022 18:02:29 +0300 Subject: linux-gen: crypto: inline event conversion functions Inline odp_crypto_packet_from_event() and odp_crypto_packet_to_event() function implementations. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-generic/Makefile.am | 2 + .../linux-generic/arch/aarch64/odp_crypto_armv8.c | 14 ------- .../linux-generic/include-abi/odp/api/abi/crypto.h | 3 +- .../include/odp/api/plat/crypto_inlines.h | 49 ++++++++++++++++++++++ platform/linux-generic/odp_crypto_api.c | 11 +++++ platform/linux-generic/odp_crypto_ipsecmb.c | 14 ------- platform/linux-generic/odp_crypto_null.c | 14 ------- platform/linux-generic/odp_crypto_openssl.c | 14 ------- 8 files changed, 64 insertions(+), 57 deletions(-) create mode 100644 platform/linux-generic/include/odp/api/plat/crypto_inlines.h create mode 100644 platform/linux-generic/odp_crypto_api.c diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 7bc48edd6..7d312d0b7 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -34,6 +34,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/buffer_inline_types.h \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ + include/odp/api/plat/crypto_inlines.h \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ @@ -294,6 +295,7 @@ __LIB__libodp_linux_la_SOURCES += \ odp_buffer_api.c \ odp_byteorder_api.c \ odp_cpu_api.c \ + odp_crypto_api.c \ odp_event_api.c \ odp_hash_api.c \ odp_ipsec_api.c \ diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index 0a73d4868..58e566c22 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -889,20 +889,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); - - return odp_packet_from_event(ev); -} - -odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) -{ - return odp_packet_to_event(pkt); -} - int odp_crypto_result(odp_crypto_packet_result_t *result, odp_packet_t packet) { diff --git a/platform/linux-generic/include-abi/odp/api/abi/crypto.h b/platform/linux-generic/include-abi/odp/api/abi/crypto.h index 24019db12..bef725c28 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/crypto.h +++ b/platform/linux-generic/include-abi/odp/api/abi/crypto.h @@ -19,7 +19,8 @@ extern "C" { #endif -/* Placeholder for inlined API functions */ +/* Inlined API functions */ +#include #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp/api/plat/crypto_inlines.h b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h new file mode 100644 index 000000000..cfea67283 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h @@ -0,0 +1,49 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CRYPTO_INLINES_H_ +#define ODP_PLAT_CRYPTO_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#include + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_crypto_packet_from_event __odp_crypto_packet_from_event + #define odp_crypto_packet_to_event __odp_crypto_packet_to_event +#else + #define _ODP_INLINE +#endif + +_ODP_INLINE odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) +{ + _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); + _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); + + return odp_packet_from_event(ev); +} + +_ODP_INLINE odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) +{ + return odp_packet_to_event(pkt); +} + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/odp_crypto_api.c b/platform/linux-generic/odp_crypto_api.c new file mode 100644 index 000000000..646472e2e --- /dev/null +++ b/platform/linux-generic/odp_crypto_api.c @@ -0,0 +1,11 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +/* Non-inlined versions of API functions */ +#define _ODP_NO_INLINE +#include diff --git a/platform/linux-generic/odp_crypto_ipsecmb.c b/platform/linux-generic/odp_crypto_ipsecmb.c index d291a7067..34b62fdee 100644 --- a/platform/linux-generic/odp_crypto_ipsecmb.c +++ b/platform/linux-generic/odp_crypto_ipsecmb.c @@ -790,20 +790,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); - - return odp_packet_from_event(ev); -} - -odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) -{ - return odp_packet_to_event(pkt); -} - int odp_crypto_result(odp_crypto_packet_result_t *result, odp_packet_t packet) { diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index b9d319861..aeb27ab5b 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -449,20 +449,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); - - return odp_packet_from_event(ev); -} - -odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) -{ - return odp_packet_to_event(pkt); -} - static odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) { diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 0f637850f..210cad38d 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -2778,20 +2778,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); - - return odp_packet_from_event(ev); -} - -odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) -{ - return odp_packet_to_event(pkt); -} - static odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) { -- cgit v1.2.3 From 76e9963229adb40410d7e4439bac35ec92e42c46 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 28 Oct 2022 18:15:36 +0300 Subject: linux-gen: crypto: inline odp_crypto_result() implementation Inline odp_crypto_result() function implementation. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../linux-generic/arch/aarch64/odp_crypto_armv8.c | 25 +--------------------- .../include/odp/api/plat/crypto_inlines.h | 16 ++++++++++++++ .../include/odp/api/plat/packet_inline_types.h | 1 + platform/linux-generic/odp_crypto_ipsecmb.c | 25 +--------------------- platform/linux-generic/odp_crypto_null.c | 25 +--------------------- platform/linux-generic/odp_crypto_openssl.c | 25 +--------------------- platform/linux-generic/odp_packet.c | 1 + 7 files changed, 22 insertions(+), 96 deletions(-) diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index 58e566c22..a1c2c96d2 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -180,21 +180,13 @@ void free_session(odp_crypto_generic_session_t *session) odp_spinlock_unlock(&global->lock); } -static -odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) -{ - odp_packet_hdr_t *hdr = packet_hdr(pkt); - - return &hdr->crypto_op_result; -} - static inline void set_crypto_op_result(odp_packet_t pkt, odp_crypto_alg_err_t cipher_err, odp_crypto_alg_err_t auth_err) { odp_crypto_packet_result_t *op_result; - op_result = get_op_result_from_packet(pkt); + op_result = &packet_hdr(pkt)->crypto_op_result; op_result->cipher_status.alg_err = cipher_err; op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = auth_err; @@ -889,21 +881,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -int odp_crypto_result(odp_crypto_packet_result_t *result, - odp_packet_t packet) -{ - odp_crypto_packet_result_t *op_result; - - _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == - ODP_EVENT_PACKET_CRYPTO); - - op_result = get_op_result_from_packet(packet); - - memcpy(result, op_result, sizeof(*result)); - - return 0; -} - static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) { int md_copy; diff --git a/platform/linux-generic/include/odp/api/plat/crypto_inlines.h b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h index cfea67283..8e98d8580 100644 --- a/platform/linux-generic/include/odp/api/plat/crypto_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h @@ -11,10 +11,12 @@ extern "C" { #endif +#include #include #include #include +#include /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ @@ -23,6 +25,7 @@ extern "C" { #define _ODP_INLINE static inline #define odp_crypto_packet_from_event __odp_crypto_packet_from_event #define odp_crypto_packet_to_event __odp_crypto_packet_to_event + #define odp_crypto_result __odp_crypto_result #else #define _ODP_INLINE #endif @@ -40,6 +43,19 @@ _ODP_INLINE odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) return odp_packet_to_event(pkt); } +_ODP_INLINE int odp_crypto_result(odp_crypto_packet_result_t *result, odp_packet_t pkt) +{ + odp_crypto_packet_result_t *op_result; + + _ODP_ASSERT(odp_packet_subtype(pkt) == ODP_EVENT_PACKET_CRYPTO); + + op_result = _odp_pkt_get_ptr(pkt, odp_crypto_packet_result_t, crypto_op); + + *result = *op_result; + + return 0; +} + /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index ae03457f9..6773b73ad 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -53,6 +53,7 @@ typedef struct _odp_packet_inline_offset_t { uint16_t subtype; uint16_t cls_mark; uint16_t ipsec_ctx; + uint16_t crypto_op; } _odp_packet_inline_offset_t; diff --git a/platform/linux-generic/odp_crypto_ipsecmb.c b/platform/linux-generic/odp_crypto_ipsecmb.c index 34b62fdee..89e964a83 100644 --- a/platform/linux-generic/odp_crypto_ipsecmb.c +++ b/platform/linux-generic/odp_crypto_ipsecmb.c @@ -151,14 +151,6 @@ void free_session(odp_crypto_generic_session_t *session) odp_spinlock_unlock(&global->lock); } -static -odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) -{ - odp_packet_hdr_t *hdr = packet_hdr(pkt); - - return &hdr->crypto_op_result; -} - static odp_crypto_alg_err_t null_crypto_routine(odp_packet_t pkt ODP_UNUSED, const odp_crypto_packet_op_param_t *param ODP_UNUSED, @@ -790,21 +782,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -int odp_crypto_result(odp_crypto_packet_result_t *result, - odp_packet_t packet) -{ - odp_crypto_packet_result_t *op_result; - - _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == - ODP_EVENT_PACKET_CRYPTO); - - op_result = get_op_result_from_packet(packet); - - memcpy(result, op_result, sizeof(*result)); - - return 0; -} - static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) { int md_copy; @@ -884,7 +861,7 @@ int crypto_int(odp_packet_t pkt_in, } packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); - op_result = get_op_result_from_packet(out_pkt); + op_result = &packet_hdr(out_pkt)->crypto_op_result; op_result->cipher_status.alg_err = rc_cipher; op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = rc_auth; diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index aeb27ab5b..50d4b5c6b 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -449,29 +449,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -static -odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) -{ - odp_packet_hdr_t *hdr = packet_hdr(pkt); - - return &hdr->crypto_op_result; -} - -int odp_crypto_result(odp_crypto_packet_result_t *result, - odp_packet_t packet) -{ - odp_crypto_packet_result_t *op_result; - - _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == - ODP_EVENT_PACKET_CRYPTO); - - op_result = get_op_result_from_packet(packet); - - memcpy(result, op_result, sizeof(*result)); - - return 0; -} - static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) { int md_copy; @@ -541,7 +518,7 @@ int crypto_int(odp_packet_t pkt_in, /* Fill in result */ packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); - op_result = get_op_result_from_packet(out_pkt); + op_result = &packet_hdr(out_pkt)->crypto_op_result; op_result->cipher_status.alg_err = ODP_CRYPTO_ALG_ERR_NONE; op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = ODP_CRYPTO_ALG_ERR_NONE; diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 210cad38d..88b9e5800 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -2778,29 +2778,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -static -odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) -{ - odp_packet_hdr_t *hdr = packet_hdr(pkt); - - return &hdr->crypto_op_result; -} - -int odp_crypto_result(odp_crypto_packet_result_t *result, - odp_packet_t packet) -{ - odp_crypto_packet_result_t *op_result; - - _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == - ODP_EVENT_PACKET_CRYPTO); - - op_result = get_op_result_from_packet(packet); - - memcpy(result, op_result, sizeof(*result)); - - return 0; -} - static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) { int md_copy; @@ -2895,7 +2872,7 @@ int crypto_int(odp_packet_t pkt_in, out: /* Fill in result */ packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); - op_result = get_op_result_from_packet(out_pkt); + op_result = &packet_hdr(out_pkt)->crypto_op_result; op_result->cipher_status.alg_err = rc_cipher; op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = rc_auth; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 212e48de0..1ce475625 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -69,6 +69,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .subtype = offsetof(odp_packet_hdr_t, subtype), .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), + .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result), }; #include -- cgit v1.2.3 From 63818da71b24777be9fb4abfb1cb1311a52ccca8 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 4 Jan 2023 09:39:07 +0200 Subject: validation: pool: test that newly allocated packet vectors are empty Validate that newly allocated packet vectors don't contain any packets. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- test/validation/api/pool/pool.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c index e8bf38c9d..cded05650 100644 --- a/test/validation/api/pool/pool.c +++ b/test/validation/api/pool/pool.c @@ -1,6 +1,6 @@ /* Copyright (c) 2014-2018, Linaro Limited * Copyright (c) 2020, Marvell - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -326,13 +326,19 @@ static void alloc_packet_vector(uint32_t cache_size) num = 0; for (i = 0; i < max_num; i++) { - pkt_vec[num] = odp_packet_vector_alloc(pool); - CU_ASSERT(pkt_vec[num] != ODP_PACKET_VECTOR_INVALID); - CU_ASSERT(odp_packet_vector_valid(pkt_vec[num]) == 1); - CU_ASSERT(odp_event_is_valid(odp_packet_vector_to_event(pkt_vec[num])) == 1); + odp_packet_vector_t pktv = odp_packet_vector_alloc(pool); - if (pkt_vec[num] != ODP_PACKET_VECTOR_INVALID) - num++; + CU_ASSERT(pktv != ODP_PACKET_VECTOR_INVALID); + + if (pktv == ODP_PACKET_VECTOR_INVALID) + continue; + + CU_ASSERT(odp_packet_vector_valid(pktv) == 1); + CU_ASSERT(odp_event_is_valid(odp_packet_vector_to_event(pktv)) == 1); + CU_ASSERT(odp_packet_vector_size(pktv) == 0); + + pkt_vec[num] = pktv; + num++; } for (i = 0; i < num; i++) -- cgit v1.2.3 From 39a034bf661d364d8696302dc939df7c7bbe6973 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 4 Jan 2023 10:20:23 +0200 Subject: validation: packet: improve packet vector testing Validate that newly allocated packet vectors are initialized correctly to be of size zero and that odp_packet_vector_free() doesn't free packets inside a packet vector. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- test/validation/api/packet/packet.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index 2810cc6b3..545ecb5c0 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * Copyright (c) 2020, Marvell * All rights reserved. * @@ -3104,17 +3104,31 @@ static void packet_vector_test_alloc_free(void) CU_ASSERT(odp_packet_vector_to_u64(pktv) != odp_packet_vector_to_u64(ODP_PACKET_VECTOR_INVALID)); - /* User flag should be initially zero */ + /* Vector size and user flag should be initially zero */ + CU_ASSERT(odp_packet_vector_size(pktv) == 0); CU_ASSERT(odp_packet_vector_user_flag(pktv) == 0); odp_packet_vector_user_flag_set(pktv, 1); CU_ASSERT(odp_packet_vector_user_flag(pktv) != 0); odp_packet_vector_user_flag_set(pktv, 0); CU_ASSERT(odp_packet_vector_user_flag(pktv) == 0); + /* Included packet should not be freed by odp_packet_vector_free() */ + pkt = odp_packet_alloc(default_pool, default_param.pkt.len); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + + CU_ASSERT(odp_packet_vector_tbl(pktv, &pkts_tbl) == 0); + pkts_tbl[0] = pkt; + odp_packet_vector_size_set(pktv, 1); + /* Free with flag still set, alloc should clear it. */ odp_packet_vector_user_flag_set(pktv, 1); odp_packet_vector_free(pktv); + + /* Check that included packet is still valid */ + CU_ASSERT(odp_packet_is_valid(pkt)); + pktv = odp_packet_vector_alloc(pool); + CU_ASSERT(odp_packet_vector_size(pktv) == 0); CU_ASSERT(odp_packet_vector_user_flag(pktv) == 0); /* Since it was only one buffer pool, more vector packets can't be @@ -3130,10 +3144,7 @@ static void packet_vector_test_alloc_free(void) CU_ASSERT_FATAL(pktv != ODP_PACKET_VECTOR_INVALID); CU_ASSERT(odp_packet_vector_size(pktv) == 0); - /* Free packet vector using odp_event_free() */ - pkt = odp_packet_alloc(default_pool, default_param.pkt.len); - CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); - + /* Free packet vector and included packet using odp_event_free() */ CU_ASSERT(odp_packet_vector_tbl(pktv, &pkts_tbl) == 0); pkts_tbl[0] = pkt; odp_packet_vector_size_set(pktv, 1); -- cgit v1.2.3 From 20cedc12f7ac2431bbedfc037c92b795cde2dd55 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 3 Oct 2022 17:17:39 +0300 Subject: test: performance: fix gcc-7 lto build failures Fix invalid 'alloc-size-larger-than' errors in GCC-7 LTO build. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- test/performance/odp_ipsec.c | 4 ++-- test/performance/odp_sched_pktio.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c index c464ba0d7..50e26b1c9 100644 --- a/test/performance/odp_ipsec.c +++ b/test/performance/odp_ipsec.c @@ -862,7 +862,7 @@ run_measure_one_async(ipsec_args_t *cargs, packets_sent += rc; } else { odp_packet_t pkt_out[max_in_flight]; - uint32_t i = 0; + int i = 0; /* * Dequeue packets until we can enqueue the next burst @@ -895,7 +895,7 @@ run_measure_one_async(ipsec_args_t *cargs, } debug_packets(debug, pkt_out, i); - if (i) + if (i > 0) odp_packet_free_sp(pkt_out, i); } } diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index 1553b02b3..5cc07d010 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -173,7 +173,7 @@ static inline void send_packets(test_global_t *test_global, drop = num_pkt - sent; - if (odp_unlikely(drop)) + if (odp_unlikely(drop > 0)) odp_packet_free_multi(&pkt[sent], drop); if (odp_unlikely(test_global->opt.collect_stat)) { -- cgit v1.2.3 From 8782c99a6cb53af57316e66271e0a1c6b951fa2b Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 6 Jul 2022 16:38:41 +0300 Subject: linux-gen: queue: header include clean-up Add missing, remove unnecessary, and group existing includes. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-generic/odp_queue_basic.c | 45 +++++++++++++++++--------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 360f907bf..eebfb92ab 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -5,39 +5,42 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include #include -#include +#include #include -#include + #include -#include -#include +#include +#include + +#include +#include #include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include + +#include +#include -#include #define LOCK(queue_ptr) odp_ticketlock_lock(&((queue_ptr)->lock)) #define UNLOCK(queue_ptr) odp_ticketlock_unlock(&((queue_ptr)->lock)) #define LOCK_INIT(queue_ptr) odp_ticketlock_init(&((queue_ptr)->lock)) -#include -#include - #define MIN_QUEUE_SIZE 32 #define MAX_QUEUE_SIZE (1 * 1024 * 1024) -- cgit v1.2.3 From a735a7131bceadb87c25b553446e4829923bcc1c Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 29 Jun 2022 16:36:22 +0300 Subject: linux-gen: event: add event validation support Add support for runtime event validation (initially buffer endmark checking). Event validation can be enabled during configure with '--enable-event-validation' [warn/abort] or with --enabled-debug=full. When event validation is enabled, endmarks are checked in: - odp_buffer_free() / odp_buffer_free_multi() - odp_buffer_is_valid() - odp_event_free() / odp_event_free_multi() / odp_event_free_sp() - odp_event_is_valid() - odp_packet_free() / odp_packet_free_multi() / odp_packet_free_sp() - odp_packet_is_valid() Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- include/odp/autoheader_external.h.in | 3 + platform/linux-generic/Makefile.am | 3 + .../odp/api/plat/event_validation_external.h | 93 ++++++++ .../include/odp/api/plat/packet_inlines.h | 3 +- .../include/odp/api/plat/pool_inline_types.h | 1 + .../linux-generic/include/odp_event_internal.h | 7 +- .../include/odp_event_validation_internal.h | 52 +++++ platform/linux-generic/include/odp_init_internal.h | 3 + platform/linux-generic/include/odp_pool_internal.h | 1 + platform/linux-generic/m4/configure.m4 | 3 + platform/linux-generic/m4/odp_event_validation.m4 | 23 ++ platform/linux-generic/odp_event.c | 25 ++- platform/linux-generic/odp_event_validation.c | 234 +++++++++++++++++++++ platform/linux-generic/odp_init.c | 16 +- platform/linux-generic/odp_packet.c | 20 +- platform/linux-generic/odp_pool.c | 31 ++- 16 files changed, 497 insertions(+), 21 deletions(-) create mode 100644 platform/linux-generic/include/odp/api/plat/event_validation_external.h create mode 100644 platform/linux-generic/include/odp_event_validation_internal.h create mode 100644 platform/linux-generic/m4/odp_event_validation.m4 create mode 100644 platform/linux-generic/odp_event_validation.c diff --git a/include/odp/autoheader_external.h.in b/include/odp/autoheader_external.h.in index dd60fa0b2..74662b2e0 100644 --- a/include/odp/autoheader_external.h.in +++ b/include/odp/autoheader_external.h.in @@ -11,4 +11,7 @@ /* Define cache line size */ #undef _ODP_CACHE_LINE_SIZE +/* Define to 1 or 2 to enable event validation */ +#undef _ODP_EVENT_VALIDATION + #endif diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 7d312d0b7..43fcfa950 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -38,6 +38,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ + include/odp/api/plat/event_validation_external.h \ include/odp/api/plat/event_vector_inline_types.h \ include/odp/api/plat/hash_inlines.h \ include/odp/api/plat/ipsec_inlines.h \ @@ -131,6 +132,7 @@ noinst_HEADERS = \ include/odp_debug_internal.h \ include/odp_errno_define.h \ include/odp_event_internal.h \ + include/odp_event_validation_internal.h \ include/odp_fdserver_internal.h \ include/odp_forward_typedefs_internal.h \ include/odp_global_data.h \ @@ -207,6 +209,7 @@ __LIB__libodp_linux_la_SOURCES = \ odp_dma.c \ odp_errno.c \ odp_event.c \ + odp_event_validation.c \ odp_fdserver.c \ odp_hash_crc_gen.c \ odp_impl.c \ diff --git a/platform/linux-generic/include/odp/api/plat/event_validation_external.h b/platform/linux-generic/include/odp/api/plat/event_validation_external.h new file mode 100644 index 000000000..2d6c8613a --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/event_validation_external.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP event validation + * + * @warning These definitions are not part of ODP API, they are for + * implementation internal use only. + */ + +#ifndef ODP_EVENT_VALIDATION_EXTERNAL_H_ +#define ODP_EVENT_VALIDATION_EXTERNAL_H_ + +#include + +#include +#include +#include +#include + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** Enumerations for identifying ODP API functions */ +typedef enum { + _ODP_EV_BUFFER_FREE = 0, + _ODP_EV_BUFFER_FREE_MULTI, + _ODP_EV_BUFFER_IS_VALID, + _ODP_EV_EVENT_FREE, + _ODP_EV_EVENT_FREE_MULTI, + _ODP_EV_EVENT_FREE_SP, + _ODP_EV_EVENT_IS_VALID, + _ODP_EV_PACKET_FREE, + _ODP_EV_PACKET_FREE_MULTI, + _ODP_EV_PACKET_FREE_SP, + _ODP_EV_PACKET_IS_VALID, + _ODP_EV_MAX +} _odp_ev_id_t; + +/* Implementation internal event validation functions */ +#if _ODP_EVENT_VALIDATION + +int _odp_buffer_validate(odp_buffer_t buf, _odp_ev_id_t ev_id); + +int _odp_buffer_validate_multi(const odp_buffer_t buf[], int num, _odp_ev_id_t ev_id); + +int _odp_packet_validate(odp_packet_t pkt, _odp_ev_id_t ev_id); + +int _odp_packet_validate_multi(const odp_packet_t pkt[], int num, _odp_ev_id_t ev_id); + +#else + +static inline int _odp_buffer_validate(odp_buffer_t buf ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_buffer_validate_multi(const odp_buffer_t buf[] ODP_UNUSED, + int num ODP_UNUSED, + _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_packet_validate(odp_packet_t pkt ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_packet_validate_multi(const odp_packet_t pkt[] ODP_UNUSED, + int num ODP_UNUSED, + _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +#endif /* _ODP_EVENT_VALIDATION */ + +#ifdef __cplusplus +} +#endif + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 01d47d837..93e95e21c 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -632,7 +632,8 @@ _ODP_INLINE uint32_t odp_packet_buf_size(odp_packet_buf_t pkt_buf) odp_pool_t pool = _odp_pkt_get(pkt_buf, odp_pool_t, pool); return _odp_pool_get(pool, uint32_t, ext_pkt_buf_size) - - _odp_pool_get(pool, uint32_t, ext_head_offset); + _odp_pool_get(pool, uint32_t, ext_head_offset) - + _odp_pool_get(pool, uint32_t, trailer_size); } _ODP_INLINE void *odp_packet_buf_head(odp_packet_buf_t pkt_buf) diff --git a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h index 02f59f982..fbff7eda7 100644 --- a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h @@ -30,6 +30,7 @@ typedef struct _odp_pool_inline_offset_t { uint16_t index; uint16_t seg_len; uint16_t uarea_size; + uint16_t trailer_size; uint16_t ext_head_offset; uint16_t ext_pkt_buf_size; diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index 5a29e926e..4bc28d708 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -53,7 +53,7 @@ typedef struct _odp_event_hdr_t { /* --- Mostly read only data --- */ - /* Initial buffer tail pointer */ + /* Initial buffer tail pointer and endmark location (if enabled) */ uint8_t *buf_end; /* Combined pool and event index */ @@ -85,6 +85,11 @@ static inline void _odp_event_type_set(odp_event_t event, int ev) _odp_event_hdr(event)->event_type = ev; } +static inline uint64_t *_odp_event_endmark_get_ptr(odp_event_t event) +{ + return (uint64_t *)(uintptr_t)_odp_event_hdr(event)->buf_end; +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_event_validation_internal.h b/platform/linux-generic/include/odp_event_validation_internal.h new file mode 100644 index 000000000..f4ac16f31 --- /dev/null +++ b/platform/linux-generic/include/odp_event_validation_internal.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_EVENT_VALIDATION_INTERNAL_H_ +#define ODP_EVENT_VALIDATION_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include +#include + +#include + +#include + +#include + +#if _ODP_EVENT_VALIDATION + +#define _ODP_EV_ENDMARK_VAL 0xDEADBEEFDEADBEEF +#define _ODP_EV_ENDMARK_SIZE (sizeof(uint64_t)) + +static inline void _odp_event_endmark_set(odp_event_t event) +{ + uint64_t *endmark_ptr; + + endmark_ptr = _odp_event_endmark_get_ptr(event); + *endmark_ptr = _ODP_EV_ENDMARK_VAL; +} + +#else + +#define _ODP_EV_ENDMARK_VAL 0 +#define _ODP_EV_ENDMARK_SIZE 0 + +static inline void _odp_event_endmark_set(odp_event_t event ODP_UNUSED) +{ +} + +#endif + +#ifdef __cplusplus +} +#endif +#endif diff --git a/platform/linux-generic/include/odp_init_internal.h b/platform/linux-generic/include/odp_init_internal.h index 2a1039854..24e8346ad 100644 --- a/platform/linux-generic/include/odp_init_internal.h +++ b/platform/linux-generic/include/odp_init_internal.h @@ -33,6 +33,9 @@ int _odp_pool_init_local(void); int _odp_pool_term_global(void); int _odp_pool_term_local(void); +int _odp_event_validation_init_global(void); +int _odp_event_validation_term_global(void); + int _odp_queue_init_global(void); int _odp_queue_term_global(void); diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index 1c5b51c3d..c8d2168f3 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -87,6 +87,7 @@ typedef struct pool_t { uint32_t block_size; uint32_t block_offset; uint32_t num_populated; + uint32_t trailer_size; uint8_t *base_addr; uint8_t *max_addr; uint8_t *uarea_base_addr; diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 61d57634f..f018d2b24 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -7,6 +7,7 @@ ODP_ATOMIC ODP_PTHREAD ODP_TIMER m4_include([platform/linux-generic/m4/odp_cpu.m4]) +m4_include([platform/linux-generic/m4/odp_event_validation.m4]) m4_include([platform/linux-generic/m4/odp_pcap.m4]) m4_include([platform/linux-generic/m4/odp_scheduler.m4]) @@ -30,6 +31,7 @@ m4_include([platform/linux-generic/m4/odp_pcapng.m4]) m4_include([platform/linux-generic/m4/odp_netmap.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) m4_include([platform/linux-generic/m4/odp_xdp.m4]) +ODP_EVENT_VALIDATION ODP_SCHEDULER AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${IPSEC_MB_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS}"]) @@ -37,6 +39,7 @@ AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFI # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. AS_VAR_APPEND([PLAT_CFG_TEXT], [" + event_validation: ${enable_event_validation} openssl: ${with_openssl} openssl_rand: ${openssl_rand} crypto: ${with_crypto} diff --git a/platform/linux-generic/m4/odp_event_validation.m4 b/platform/linux-generic/m4/odp_event_validation.m4 new file mode 100644 index 000000000..08bb8902e --- /dev/null +++ b/platform/linux-generic/m4/odp_event_validation.m4 @@ -0,0 +1,23 @@ +# ODP_EVENT_VALIDATION +# -------------------- +# Select event validation level +AC_DEFUN([ODP_EVENT_VALIDATION], [dnl +AC_ARG_ENABLE([event-validation], + [AS_HELP_STRING([--enable-event-validation], + [enable event validation (warn/abort) + [default=disabled] (linux-generic)])], + [], [AS_IF([test "x$enable_debug" = "xfull"], + [enable_event_validation=yes], [enable_event_validation=no])]) + +# Default to abort mode if validation is enabled +AS_IF([test "x$enable_event_validation" = "xyes"], + [enable_event_validation="abort"]) + +validation_level=0 +AS_IF([test "x$enable_event_validation" = "xwarn"], [validation_level=1]) +AS_IF([test "x$enable_event_validation" = "xyes" -o "x$enable_event_validation" = "xabort"], + [validation_level=2]) + +AC_DEFINE_UNQUOTED([_ODP_EVENT_VALIDATION], [$validation_level], + [Define to 1 or 2 to enable event validation]) +]) # ODP_EVENT_VALIDATION diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c index c4e0f2c9d..edf77e2dc 100644 --- a/platform/linux-generic/odp_event.c +++ b/platform/linux-generic/odp_event.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,6 +18,7 @@ #include #include #include +#include #include /* Inlined API functions */ @@ -41,13 +42,15 @@ _odp_event_inline_offset ODP_ALIGNED_CACHE = { #include -void odp_event_free(odp_event_t event) +static inline void event_free(odp_event_t event, _odp_ev_id_t id) { switch (odp_event_type(event)) { case ODP_EVENT_BUFFER: + _odp_buffer_validate(odp_buffer_from_event(event), id); odp_buffer_free(odp_buffer_from_event(event)); break; case ODP_EVENT_PACKET: + _odp_packet_validate(odp_packet_from_event(event), id); odp_packet_free(odp_packet_from_event(event)); break; case ODP_EVENT_PACKET_VECTOR: @@ -75,17 +78,21 @@ void odp_event_free(odp_event_t event) } } -void odp_event_free_multi(const odp_event_t event[], int num) +void odp_event_free(odp_event_t event) { - int i; + event_free(event, _ODP_EV_EVENT_FREE); +} - for (i = 0; i < num; i++) - odp_event_free(event[i]); +void odp_event_free_multi(const odp_event_t event[], int num) +{ + for (int i = 0; i < num; i++) + event_free(event[i], _ODP_EV_EVENT_FREE_MULTI); } void odp_event_free_sp(const odp_event_t event[], int num) { - odp_event_free_multi(event, num); + for (int i = 0; i < num; i++) + event_free(event[i], _ODP_EV_EVENT_FREE_SP); } uint64_t odp_event_to_u64(odp_event_t hdl) @@ -103,9 +110,9 @@ int odp_event_is_valid(odp_event_t event) switch (odp_event_type(event)) { case ODP_EVENT_BUFFER: - /* Fall through */ + return !_odp_buffer_validate(odp_buffer_from_event(event), _ODP_EV_EVENT_IS_VALID); case ODP_EVENT_PACKET: - /* Fall through */ + return !_odp_packet_validate(odp_packet_from_event(event), _ODP_EV_EVENT_IS_VALID); case ODP_EVENT_TIMEOUT: /* Fall through */ #if ODP_DEPRECATED_API diff --git a/platform/linux-generic/odp_event_validation.c b/platform/linux-generic/odp_event_validation.c new file mode 100644 index 000000000..8dd8d7b8e --- /dev/null +++ b/platform/linux-generic/odp_event_validation.c @@ -0,0 +1,234 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define EVENT_VALIDATION_NONE 0 +#define EVENT_VALIDATION_WARN 1 +#define EVENT_VALIDATION_ABORT 2 + +#define EVENT_DATA_PRINT_MAX_LEN 128 + +typedef struct { + odp_atomic_u64_t err_count[_ODP_EV_MAX]; + odp_shm_t shm; + +} event_validation_global_t; + +typedef struct { + const char *str; +} _odp_ev_info_t; + +static event_validation_global_t *_odp_ev_glb; + +#if _ODP_EVENT_VALIDATION + +/* Table for mapping function IDs to API function names */ +static const _odp_ev_info_t ev_info_tbl[] = { + [_ODP_EV_BUFFER_FREE] = {.str = "odp_buffer_free()"}, + [_ODP_EV_BUFFER_FREE_MULTI] = {.str = "odp_buffer_free_multi()"}, + [_ODP_EV_BUFFER_IS_VALID] = {.str = "odp_buffer_is_valid()"}, + [_ODP_EV_EVENT_FREE] = {.str = "odp_event_free()"}, + [_ODP_EV_EVENT_FREE_MULTI] = {.str = "odp_event_free_multi()"}, + [_ODP_EV_EVENT_FREE_SP] = {.str = "odp_event_free()_sp"}, + [_ODP_EV_EVENT_IS_VALID] = {.str = "odp_event_is_valid()"}, + [_ODP_EV_PACKET_FREE] = {.str = "odp_packet_free()"}, + [_ODP_EV_PACKET_FREE_MULTI] = {.str = "odp_packet_free_multi()"}, + [_ODP_EV_PACKET_FREE_SP] = {.str = "odp_packet_free_sp()"}, + [_ODP_EV_PACKET_IS_VALID] = {.str = "odp_packet_is_valid()"} +}; + +ODP_STATIC_ASSERT(_ODP_ARRAY_SIZE(ev_info_tbl) == _ODP_EV_MAX, "ev_info_tbl missing entries"); + +static void print_event_data(odp_event_t event, odp_event_type_t type) +{ + const char *type_str; + const uint32_t bytes_per_row = 16; + uint32_t byte_len; + int num_rows, max_len, n; + int len = 0; + uint8_t *data; + + if (type == ODP_EVENT_PACKET) { + odp_packet_t pkt = odp_packet_from_event(event); + + data = odp_packet_data(pkt); + byte_len = odp_packet_seg_len(pkt); + type_str = "Packet"; + } else { + odp_buffer_t buf = odp_buffer_from_event(event); + + data = odp_buffer_addr(buf); + byte_len = odp_buffer_size(buf); + type_str = "Buffer"; + } + + if (byte_len > EVENT_DATA_PRINT_MAX_LEN) + byte_len = EVENT_DATA_PRINT_MAX_LEN; + + num_rows = (byte_len + bytes_per_row - 1) / bytes_per_row; + max_len = 256 + (3 * byte_len) + (3 * num_rows); + n = max_len - 1; + + char str[max_len]; + + len += _odp_snprint(&str[len], n - len, "%s %p data %p:\n", type_str, event, data); + while (byte_len) { + uint32_t row_len = byte_len > bytes_per_row ? bytes_per_row : byte_len; + + len += _odp_snprint(&str[len], n - len, " "); + + for (uint32_t i = 0; i < row_len; i++) + len += _odp_snprint(&str[len], n - len, " %02x", data[i]); + + len += _odp_snprint(&str[len], n - len, "\n"); + + byte_len -= row_len; + data += row_len; + } + + _ODP_PRINT("%s\n", str); +} + +static inline int validate_event_endmark(odp_event_t event, _odp_ev_id_t id, odp_event_type_t type) +{ + uint64_t err_count; + uint64_t *endmark_ptr = _odp_event_endmark_get_ptr(event); + + if (odp_likely(*endmark_ptr == _ODP_EV_ENDMARK_VAL)) + return 0; + + err_count = odp_atomic_fetch_inc_u64(&_odp_ev_glb->err_count[id]) + 1; + + _ODP_ERR("Event %p endmark mismatch in %s: endmark=0x%" PRIx64 " (expected 0x%" PRIx64 ") " + "err_count=%" PRIu64 "\n", event, ev_info_tbl[id].str, *endmark_ptr, + _ODP_EV_ENDMARK_VAL, err_count); + + print_event_data(event, type); + + if (_ODP_EVENT_VALIDATION == EVENT_VALIDATION_ABORT) + _ODP_ABORT("Abort due to event %p endmark mismatch\n", event); + + /* Fix endmark value */ + _odp_event_endmark_set(event); + + return -1; +} + +static inline int buffer_validate(odp_buffer_t buf, _odp_ev_id_t id) +{ + return validate_event_endmark(odp_buffer_to_event(buf), id, ODP_EVENT_BUFFER); +} + +static inline int packet_validate(odp_packet_t pkt, _odp_ev_id_t id) +{ + return validate_event_endmark(odp_packet_to_event(pkt), id, ODP_EVENT_PACKET); +} + +/* Enable usage from API inline files */ +#include + +int _odp_buffer_validate(odp_buffer_t buf, _odp_ev_id_t id) +{ + return buffer_validate(buf, id); +} + +int _odp_buffer_validate_multi(const odp_buffer_t buf[], int num, + _odp_ev_id_t id) +{ + for (int i = 0; i < num; i++) { + if (odp_unlikely(buffer_validate(buf[i], id))) + return -1; + } + return 0; +} + +int _odp_packet_validate(odp_packet_t pkt, _odp_ev_id_t id) +{ + return packet_validate(pkt, id); +} + +int _odp_packet_validate_multi(const odp_packet_t pkt[], int num, + _odp_ev_id_t id) +{ + for (int i = 0; i < num; i++) { + if (odp_unlikely(packet_validate(pkt[i], id))) + return -1; + } + return 0; +} + +#include + +#endif /* _ODP_EVENT_VALIDATION */ + +int _odp_event_validation_init_global(void) +{ + odp_shm_t shm; + + _ODP_PRINT("\nEvent validation mode: %s\n\n", + _ODP_EVENT_VALIDATION == EVENT_VALIDATION_NONE ? "none" : + _ODP_EVENT_VALIDATION == EVENT_VALIDATION_WARN ? "warn" : "abort"); + + if (_ODP_EVENT_VALIDATION == EVENT_VALIDATION_NONE) + return 0; + + shm = odp_shm_reserve("_odp_event_validation_global", + sizeof(event_validation_global_t), + ODP_CACHE_LINE_SIZE, ODP_SHM_EXPORT); + if (shm == ODP_SHM_INVALID) + return -1; + + _odp_ev_glb = odp_shm_addr(shm); + if (_odp_ev_glb == NULL) + return -1; + + memset(_odp_ev_glb, 0, sizeof(event_validation_global_t)); + _odp_ev_glb->shm = shm; + + for (int i = 0; i < _ODP_EV_MAX; i++) + odp_atomic_init_u64(&_odp_ev_glb->err_count[i], 0); + + return 0; +} + +int _odp_event_validation_term_global(void) +{ + int ret; + + if (_ODP_EVENT_VALIDATION == EVENT_VALIDATION_NONE) + return 0; + + if (_odp_ev_glb == NULL) + return 0; + + ret = odp_shm_free(_odp_ev_glb->shm); + if (ret) { + _ODP_ERR("SHM free failed: %d\n", ret); + return -1; + } + + return 0; +} diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 284b3e566..bd27641aa 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -35,6 +35,7 @@ enum init_stage { HASH_INIT, THREAD_INIT, POOL_INIT, + EVENT_VALIDATION_INIT, STASH_INIT, QUEUE_INIT, SCHED_INIT, @@ -242,6 +243,13 @@ static int term_global(enum init_stage stage) } /* Fall through */ + case EVENT_VALIDATION_INIT: + if (_odp_event_validation_term_global()) { + _ODP_ERR("ODP event validation term failed.\n"); + rc = -1; + } + /* Fall through */ + case POOL_INIT: if (_odp_pool_term_global()) { _ODP_ERR("ODP buffer pool term failed.\n"); @@ -412,6 +420,12 @@ int odp_init_global(odp_instance_t *instance, } stage = POOL_INIT; + if (_odp_event_validation_init_global()) { + _ODP_ERR("ODP event validation init failed.\n"); + goto init_failed; + } + stage = EVENT_VALIDATION_INIT; + if (_odp_stash_init_global()) { _ODP_ERR("ODP stash init failed.\n"); goto init_failed; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 1ce475625..639a74e0c 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -696,6 +697,8 @@ void odp_packet_free(odp_packet_t pkt) odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); int num_seg = pkt_hdr->seg_count; + _odp_packet_validate(pkt, _ODP_EV_PACKET_FREE); + _ODP_ASSERT(segment_ref(pkt_hdr) > 0); if (odp_likely(num_seg == 1)) @@ -704,12 +707,14 @@ void odp_packet_free(odp_packet_t pkt) free_all_segments(pkt_hdr, num_seg); } -void odp_packet_free_multi(const odp_packet_t pkt[], int num) +static inline void packet_free_multi_ev(const odp_packet_t pkt[], int num, _odp_ev_id_t id) { odp_packet_hdr_t *pkt_hdrs[num]; int i; int num_freed = 0; + _odp_packet_validate_multi(pkt, num, id); + for (i = 0; i < num; i++) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); int num_seg = pkt_hdr->seg_count; @@ -729,9 +734,14 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) packet_free_multi(pkt_hdrs, num - num_freed); } +void odp_packet_free_multi(const odp_packet_t pkt[], int num) +{ + packet_free_multi_ev(pkt, num, _ODP_EV_PACKET_FREE_MULTI); +} + void odp_packet_free_sp(const odp_packet_t pkt[], int num) { - odp_packet_free_multi(pkt, num); + packet_free_multi_ev(pkt, num, _ODP_EV_PACKET_FREE_SP); } int odp_packet_reset(odp_packet_t pkt, uint32_t len) @@ -1588,6 +1598,9 @@ int odp_packet_is_valid(odp_packet_t pkt) if (odp_event_type(ev) != ODP_EVENT_PACKET) return 0; + if (odp_unlikely(_odp_packet_validate(pkt, _ODP_EV_PACKET_IS_VALID))) + return 0; + switch (odp_event_subtype(ev)) { case ODP_EVENT_PACKET_BASIC: /* Fall through */ @@ -2344,6 +2357,7 @@ odp_packet_t odp_packet_reassemble(odp_pool_t pool_hdl, odp_packet_buf_t pkt_buf tailroom = pool->ext_param.pkt.buf_size - sizeof(odp_packet_hdr_t); tailroom -= pool->ext_param.pkt.app_header_size; tailroom -= odp_packet_buf_data_len(pkt_buf[num - 1]); + tailroom -= pool->trailer_size; pkt_hdr->seg_count = num; pkt_hdr->frame_len = data_len; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index f414b0626..59c007ee2 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -21,6 +22,8 @@ #include #include #include +#include +#include #include #include #include @@ -77,6 +80,7 @@ const _odp_pool_inline_offset_t _odp_pool_inline ODP_ALIGNED_CACHE = { .index = offsetof(pool_t, pool_idx), .seg_len = offsetof(pool_t, seg_len), .uarea_size = offsetof(pool_t, param_uarea_size), + .trailer_size = offsetof(pool_t, trailer_size), .ext_head_offset = offsetof(pool_t, ext_head_offset), .ext_pkt_buf_size = offsetof(pool_t, ext_param.pkt.buf_size) }; @@ -472,6 +476,7 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET) { event_hdr->base_data = data_ptr; event_hdr->buf_end = data_ptr + pool->seg_len + pool->tailroom; + _odp_event_endmark_set(_odp_event_from_hdr(event_hdr)); } if (type == ODP_POOL_BUFFER) { @@ -697,7 +702,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, uint32_t uarea_size, headroom, tailroom; odp_shm_t shm; uint32_t seg_len, align, num, hdr_size, block_size; - uint32_t max_len, cache_size; + uint32_t max_len, cache_size, trailer_size; uint32_t ring_size; odp_pool_type_t type = params->type; uint32_t shmflags = 0; @@ -743,6 +748,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, tailroom = 0; seg_len = 0; max_len = 0; + trailer_size = 0; uarea_size = 0; cache_size = 0; @@ -752,6 +758,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, seg_len = params->buf.size; uarea_size = params->buf.uarea_size; cache_size = params->buf.cache_size; + trailer_size = _ODP_EV_ENDMARK_SIZE; break; case ODP_POOL_PACKET: @@ -763,6 +770,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, num = params->pkt.num; seg_len = CONFIG_PACKET_MAX_SEG_LEN; max_len = _odp_pool_glb->config.pkt_max_len; + trailer_size = _ODP_EV_ENDMARK_SIZE; if (params->pkt.len && params->pkt.len < CONFIG_PACKET_MAX_SEG_LEN) @@ -840,7 +848,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, uint32_t adj_size; hdr_size = _ODP_ROUNDUP_CACHE_LINE(sizeof(odp_packet_hdr_t)); - block_size = hdr_size + align + headroom + seg_len + tailroom; + block_size = hdr_size + align + headroom + seg_len + tailroom + trailer_size; adj_size = block_size; if (pool->mem_src_ops && pool->mem_src_ops->adjust_size) { @@ -871,7 +879,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, else hdr_size = _ODP_ROUNDUP_CACHE_LINE(sizeof(odp_event_vector_hdr_t)); - block_size = _ODP_ROUNDUP_CACHE_LINE(hdr_size + align_pad + seg_len); + block_size = _ODP_ROUNDUP_CACHE_LINE(hdr_size + align_pad + seg_len + trailer_size); } /* Allocate extra memory for skipping packet buffers which cross huge @@ -894,6 +902,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, pool->align = align; pool->headroom = headroom; pool->seg_len = seg_len; + pool->trailer_size = trailer_size; pool->max_seg_len = headroom + seg_len + tailroom; pool->max_len = max_len; pool->tailroom = tailroom; @@ -1419,11 +1428,15 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) void odp_buffer_free(odp_buffer_t buf) { + _odp_buffer_validate(buf, _ODP_EV_BUFFER_FREE); + _odp_event_free_multi((_odp_event_hdr_t **)&buf, 1); } void odp_buffer_free_multi(const odp_buffer_t buf[], int num) { + _odp_buffer_validate_multi(buf, num, _ODP_EV_BUFFER_FREE_MULTI); + _odp_event_free_multi((_odp_event_hdr_t **)(uintptr_t)buf, num); } @@ -1530,6 +1543,7 @@ void odp_pool_print(odp_pool_t pool_hdl) _ODP_PRINT(" burst size %u\n", pool->burst_size); _ODP_PRINT(" mem src %s\n", pool->mem_src_ops ? pool->mem_src_ops->name : "(none)"); + _ODP_PRINT(" event valid. %d\n", _ODP_EVENT_VALIDATION); _ODP_PRINT("\n"); } @@ -1727,6 +1741,9 @@ int odp_buffer_is_valid(odp_buffer_t buf) if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER) return 0; + if (odp_unlikely(_odp_buffer_validate(buf, _ODP_EV_BUFFER_IS_VALID))) + return 0; + return 1; } @@ -1753,7 +1770,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, odp_pool_ext_capability_t *cap capa->pkt.max_num_buf = _odp_pool_glb->config.pkt_max_num; capa->pkt.max_buf_size = MAX_SIZE; capa->pkt.odp_header_size = sizeof(odp_packet_hdr_t); - capa->pkt.odp_trailer_size = 0; + capa->pkt.odp_trailer_size = _ODP_EV_ENDMARK_SIZE; capa->pkt.min_mem_align = ODP_CACHE_LINE_SIZE; capa->pkt.min_buf_align = ODP_CACHE_LINE_SIZE; capa->pkt.min_head_align = MIN_HEAD_ALIGN; @@ -1878,7 +1895,9 @@ odp_pool_t odp_pool_ext_create(const char *name, const odp_pool_ext_param_t *par pool->num = num_buf; pool->headroom = headroom; pool->tailroom = 0; - pool->seg_len = buf_size - head_offset - headroom - pool->tailroom; + pool->trailer_size = _ODP_EV_ENDMARK_SIZE; + pool->seg_len = buf_size - head_offset - headroom - pool->tailroom - + pool->trailer_size; pool->max_seg_len = headroom + pool->seg_len + pool->tailroom; pool->max_len = PKT_MAX_SEGS * pool->seg_len; pool->ext_head_offset = head_offset; -- cgit v1.2.3 From 85d0d642154a15ea46dd1a1fcbcde3c4bdeffa49 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 28 Dec 2022 08:49:16 +0200 Subject: linux-gen: event: add event validation support for queue enq operations Add event validation support to following queue enqueue functions: - odp_queue_enq() - odp_queue_enq_multi() Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../odp/api/plat/event_validation_external.h | 18 ++++++++++++++ .../include/odp/api/plat/queue_inlines.h | 10 ++++++++ platform/linux-generic/odp_event_validation.c | 28 +++++++++++++++++++++- 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/platform/linux-generic/include/odp/api/plat/event_validation_external.h b/platform/linux-generic/include/odp/api/plat/event_validation_external.h index 2d6c8613a..5a939402f 100644 --- a/platform/linux-generic/include/odp/api/plat/event_validation_external.h +++ b/platform/linux-generic/include/odp/api/plat/event_validation_external.h @@ -42,12 +42,18 @@ typedef enum { _ODP_EV_PACKET_FREE_MULTI, _ODP_EV_PACKET_FREE_SP, _ODP_EV_PACKET_IS_VALID, + _ODP_EV_QUEUE_ENQ, + _ODP_EV_QUEUE_ENQ_MULTI, _ODP_EV_MAX } _odp_ev_id_t; /* Implementation internal event validation functions */ #if _ODP_EVENT_VALIDATION +int _odp_event_validate(odp_event_t event, _odp_ev_id_t id); + +int _odp_event_validate_multi(const odp_event_t event[], int num, _odp_ev_id_t id); + int _odp_buffer_validate(odp_buffer_t buf, _odp_ev_id_t ev_id); int _odp_buffer_validate_multi(const odp_buffer_t buf[], int num, _odp_ev_id_t ev_id); @@ -58,6 +64,18 @@ int _odp_packet_validate_multi(const odp_packet_t pkt[], int num, _odp_ev_id_t e #else +static inline int _odp_event_validate(odp_event_t event ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + +static inline int _odp_event_validate_multi(const odp_event_t event[] ODP_UNUSED, + int num ODP_UNUSED, + _odp_ev_id_t ev_id ODP_UNUSED) +{ + return 0; +} + static inline int _odp_buffer_validate(odp_buffer_t buf ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED) { return 0; diff --git a/platform/linux-generic/include/odp/api/plat/queue_inlines.h b/platform/linux-generic/include/odp/api/plat/queue_inlines.h index 22673a887..609c0c9e4 100644 --- a/platform/linux-generic/include/odp/api/plat/queue_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/queue_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,6 +8,9 @@ #ifndef ODP_PLAT_QUEUE_INLINES_H_ #define ODP_PLAT_QUEUE_INLINES_H_ +#include + +#include #include /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ @@ -37,12 +41,18 @@ _ODP_INLINE void *odp_queue_context(odp_queue_t handle) _ODP_INLINE int odp_queue_enq(odp_queue_t queue, odp_event_t ev) { + if (odp_unlikely(_odp_event_validate(ev, _ODP_EV_QUEUE_ENQ))) + return -1; + return _odp_queue_api->queue_enq(queue, ev); } _ODP_INLINE int odp_queue_enq_multi(odp_queue_t queue, const odp_event_t events[], int num) { + if (odp_unlikely(_odp_event_validate_multi(events, num, _ODP_EV_QUEUE_ENQ_MULTI))) + return -1; + return _odp_queue_api->queue_enq_multi(queue, events, num); } diff --git a/platform/linux-generic/odp_event_validation.c b/platform/linux-generic/odp_event_validation.c index 8dd8d7b8e..c2d430f1a 100644 --- a/platform/linux-generic/odp_event_validation.c +++ b/platform/linux-generic/odp_event_validation.c @@ -57,7 +57,9 @@ static const _odp_ev_info_t ev_info_tbl[] = { [_ODP_EV_PACKET_FREE] = {.str = "odp_packet_free()"}, [_ODP_EV_PACKET_FREE_MULTI] = {.str = "odp_packet_free_multi()"}, [_ODP_EV_PACKET_FREE_SP] = {.str = "odp_packet_free_sp()"}, - [_ODP_EV_PACKET_IS_VALID] = {.str = "odp_packet_is_valid()"} + [_ODP_EV_PACKET_IS_VALID] = {.str = "odp_packet_is_valid()"}, + [_ODP_EV_QUEUE_ENQ] = {.str = "odp_queue_enq()"}, + [_ODP_EV_QUEUE_ENQ_MULTI] = {.str = "odp_queue_enq_multi()"} }; ODP_STATIC_ASSERT(_ODP_ARRAY_SIZE(ev_info_tbl) == _ODP_EV_MAX, "ev_info_tbl missing entries"); @@ -147,6 +149,15 @@ static inline int packet_validate(odp_packet_t pkt, _odp_ev_id_t id) return validate_event_endmark(odp_packet_to_event(pkt), id, ODP_EVENT_PACKET); } +static inline int event_validate(odp_event_t event, int id) +{ + if (odp_event_type(event) == ODP_EVENT_BUFFER) + return buffer_validate(odp_buffer_from_event(event), id); + if (odp_event_type(event) == ODP_EVENT_PACKET) + return packet_validate(odp_packet_from_event(event), id); + return 0; +} + /* Enable usage from API inline files */ #include @@ -180,6 +191,21 @@ int _odp_packet_validate_multi(const odp_packet_t pkt[], int num, return 0; } +int _odp_event_validate(odp_event_t event, _odp_ev_id_t id) +{ + return event_validate(event, id); +} + +int _odp_event_validate_multi(const odp_event_t event[], int num, + _odp_ev_id_t id) +{ + for (int i = 0; i < num; i++) { + if (odp_unlikely(event_validate(event[i], id))) + return -1; + } + return 0; +} + #include #endif /* _ODP_EVENT_VALIDATION */ -- cgit v1.2.3 From 2e230cfc92a17bdf72a359addaa31ebfc25d2cb1 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 3 Oct 2022 10:23:29 +0300 Subject: linux-gen: README: add documentation for event validation Document usage of additional event validation checks. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-generic/README | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/platform/linux-generic/README b/platform/linux-generic/README index 8f41d1d45..138e6040c 100644 --- a/platform/linux-generic/README +++ b/platform/linux-generic/README @@ -1,5 +1,5 @@ Copyright (c) 2014-2018, Linaro Limited -Copyright (c) 2019, Nokia +Copyright (c) 2019-2023, Nokia All rights reserved. SPDX-License-Identifier: BSD-3-Clause @@ -52,7 +52,29 @@ SPDX-License-Identifier: BSD-3-Clause Note that there may be issues with the quality or security of rdrand and rdseed. [2] -6. References +6. Event validation + ODP linux-generic implementation supports additional fast path event + validity checks which are disabled by default to minimize overhead. These + checks can be enabled with --enable-event-validation [abort/warn] or + --enabled-debug=full configuration options. + + Event validation adds additional endmark data to ODP buffers and packets, + which is used to detect data writes outside allowed areas. Endmarks are + checked by the implementation each time application calls one the following + API functions: + - odp_buffer_free() / odp_buffer_free_multi() + - odp_buffer_is_valid() + - odp_event_free() / odp_event_free_multi() / odp_event_free_sp() + - odp_event_is_valid() + - odp_packet_free() / odp_packet_free_multi() / odp_packet_free_sp() + - odp_packet_is_valid() + - odp_queue_enq() / odp_queue_enq_multi() + + Event validation can function in two modes: abort (default) and warn. In + abort mode the application is terminated immediately if an event validity + check fails. In warn mode only an error log message is printed. + +7. References [1] Intel Digital Random Number Generator (DRNG) Software Implementation Guide. John P Mechalas, 17 October 2018. https://www.intel.com/content/www/us/en/developer/articles/guide/intel-digital-random-number-generator-drng-software-implementation-guide.html -- cgit v1.2.3 From e42a3a737937782a2f389ce8e8578adedb066fb3 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 6 Jul 2022 17:31:59 +0300 Subject: github_ci: add tests for event validation Add tests for event validation warning mode. Abort mode is tested in '--enable-debug=full' tests. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .github/workflows/ci-pipeline-arm64.yml | 3 ++- .github/workflows/ci-pipeline.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index 33ac5c65c..31747b784 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -147,7 +147,8 @@ jobs: cc: [gcc, clang] conf: ['', '--enable-abi-compat', '--enable-deprecated --enable-helper-deprecated --enable-debug=full', '--enable-dpdk-zero-copy --disable-static-applications', - '--disable-host-optimization', '--disable-host-optimization --enable-abi-compat', + '--disable-host-optimization --enable-event-validation=warn', + '--disable-host-optimization --enable-abi-compat', '--without-openssl --without-pcap', '--with-crypto=armv8crypto', '--with-crypto=ipsecmb'] steps: - uses: AutoModality/action-clean@v1.1.0 diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 8b6a762a9..833ed92ce 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -334,7 +334,8 @@ jobs: cc: [gcc, clang] conf: ['', '--enable-abi-compat', '--enable-deprecated --enable-helper-deprecated --enable-debug=full', '--enable-dpdk-zero-copy --disable-static-applications', - '--disable-host-optimization', '--disable-host-optimization --enable-abi-compat', + '--disable-host-optimization --enable-event-validation=warn', + '--disable-host-optimization --enable-abi-compat', '--without-openssl --without-pcap'] steps: - uses: actions/checkout@v3 -- cgit v1.2.3 From 77d2986ac48a62b7c0493bbd666c33915969677b Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Tue, 27 Dec 2022 12:36:00 +0000 Subject: linux-gen: loop: add multi-queue support Add multi-queue support to loop packet I/O. This enables more realistic packet I/O application testing. RX and TX specific locking is removed as all data shared between transmitting and receiving threads is already multi-thread safe. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- .../linux-generic/include/odp_packet_io_internal.h | 8 +- platform/linux-generic/pktio/loop.c | 130 +++++++++++++-------- 2 files changed, 84 insertions(+), 54 deletions(-) diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 954602959..187a3a76f 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -68,12 +68,8 @@ struct pktio_if_ops; #define PKTIO_PRIVATE_SIZE 33792 #elif defined(_ODP_PKTIO_XDP) #define PKTIO_PRIVATE_SIZE 29696 -#elif defined(_ODP_PKTIO_DPDK) && ODP_CACHE_LINE_SIZE == 128 -#define PKTIO_PRIVATE_SIZE 4160 -#elif defined(_ODP_PKTIO_DPDK) -#define PKTIO_PRIVATE_SIZE 3968 #else -#define PKTIO_PRIVATE_SIZE 384 +#define PKTIO_PRIVATE_SIZE 9216 #endif typedef struct ODP_ALIGNED_CACHE { diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index b30535f22..f5190b96f 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2013-2022, Nokia Solutions and Networks + * Copyright (c) 2013-2023, Nokia Solutions and Networks * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -46,13 +45,28 @@ #define LOOP_MAX_QUEUE_SIZE 1024 +typedef struct ODP_ALIGNED_CACHE { + /* queue handle as the "wire" */ + odp_queue_t queue; + /* config input queue size */ + uint32_t in_size; + /* config output queue size */ + uint32_t out_size; +} loop_queue_t; + typedef struct { - odp_queue_t loopq; /**< loopback queue for "loop" device */ - uint32_t pktin_queue_size; /**< input queue size */ - uint32_t pktout_queue_size; /**< output queue size */ - uint16_t mtu; /**< link MTU */ - uint8_t idx; /**< index of "loop" device */ - uint8_t queue_create; /**< create or re-create queue during start */ + /* loopback entries for "loop" device */ + loop_queue_t loopqs[ODP_PKTIN_MAX_QUEUES]; + /* config queue count */ + uint32_t num_conf_qs; + /* actual number queues */ + uint32_t num_qs; + /* link MTU */ + uint16_t mtu; + /* index of "loop" device */ + uint8_t idx; + /* create or re-create queue during start */ + uint8_t queue_create; } pkt_loop_t; ODP_STATIC_ASSERT(PKTIO_PRIVATE_SIZE >= sizeof(pkt_loop_t), @@ -88,11 +102,9 @@ static int loopback_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, } memset(pkt_loop, 0, sizeof(pkt_loop_t)); - pkt_loop->idx = idx; pkt_loop->mtu = LOOP_MTU_MAX; - pkt_loop->loopq = ODP_QUEUE_INVALID; + pkt_loop->idx = idx; pkt_loop->queue_create = 1; - loopback_stats_reset(pktio_entry); loopback_init_capability(pktio_entry); @@ -117,6 +129,18 @@ static int loopback_queue_destroy(odp_queue_t queue) return 0; } +static int loopback_queues_destroy(loop_queue_t *queues, uint32_t num_queues) +{ + int ret = 0; + + for (uint32_t i = 0; i < num_queues; i++) { + if (loopback_queue_destroy(queues[i].queue)) + ret = -1; + } + + return ret; +} + static int loopback_start(pktio_entry_t *pktio_entry) { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); @@ -127,23 +151,28 @@ static int loopback_start(pktio_entry_t *pktio_entry) if (!pkt_loop->queue_create) return 0; - /* Destroy old queue */ - if (pkt_loop->loopq != ODP_QUEUE_INVALID && loopback_queue_destroy(pkt_loop->loopq)) + /* Destroy old queues */ + if (loopback_queues_destroy(pkt_loop->loopqs, pkt_loop->num_qs)) return -1; - odp_queue_param_init(&queue_param); - queue_param.size = pkt_loop->pktin_queue_size > pkt_loop->pktout_queue_size ? - pkt_loop->pktin_queue_size : pkt_loop->pktout_queue_size; + pkt_loop->num_qs = 0; - snprintf(queue_name, sizeof(queue_name), "_odp_pktio_loopq-%" PRIu64 "", - odp_pktio_to_u64(pktio_entry->handle)); + for (uint32_t i = 0; i < pkt_loop->num_conf_qs; i++) { + odp_queue_param_init(&queue_param); + queue_param.size = _ODP_MAX(pkt_loop->loopqs[i].in_size, + pkt_loop->loopqs[i].out_size); + snprintf(queue_name, sizeof(queue_name), "_odp_pktio_loopq-%" PRIu64 "-%u", + odp_pktio_to_u64(pktio_entry->handle), i); + pkt_loop->loopqs[i].queue = odp_queue_create(queue_name, &queue_param); - pkt_loop->loopq = odp_queue_create(queue_name, &queue_param); - if (pkt_loop->loopq == ODP_QUEUE_INVALID) { - _ODP_ERR("Creating loopback pktio queue failed\n"); - return -1; + if (pkt_loop->loopqs[i].queue == ODP_QUEUE_INVALID) { + _ODP_ERR("Creating loopback pktio queue %s failed\n", queue_name); + (void)loopback_queues_destroy(pkt_loop->loopqs, i); + return -1; + } } - pkt_loop->queue_create = 0; + + pkt_loop->num_qs = pkt_loop->num_conf_qs; return 0; } @@ -153,9 +182,16 @@ static int loopback_pktin_queue_config(pktio_entry_t *pktio_entry, { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); + pkt_loop->num_conf_qs = param->num_queues; + pkt_loop->queue_create = 1; + if (pktio_entry->param.in_mode == ODP_PKTIN_MODE_DIRECT) { - pkt_loop->pktin_queue_size = param->queue_size[0]; - pkt_loop->queue_create = 1; + for (uint32_t i = 0; i < ODP_PKTIN_MAX_QUEUES; i++) { + if (i < pkt_loop->num_conf_qs) + pkt_loop->loopqs[i].in_size = param->queue_size[i]; + else + pkt_loop->loopqs[i].in_size = 0; + } } return 0; @@ -166,9 +202,15 @@ static int loopback_pktout_queue_config(pktio_entry_t *pktio_entry, { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); - pkt_loop->pktout_queue_size = param->queue_size[0]; pkt_loop->queue_create = 1; + for (uint32_t i = 0; i < ODP_PKTIN_MAX_QUEUES; i++) { + if (i < param->num_queues) + pkt_loop->loopqs[i].out_size = param->queue_size[i]; + else + pkt_loop->loopqs[i].out_size = 0; + } + return 0; } @@ -176,14 +218,10 @@ static int loopback_close(pktio_entry_t *pktio_entry) { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); - if (pkt_loop->loopq != ODP_QUEUE_INVALID) - return loopback_queue_destroy(pkt_loop->loopq); - - return 0; + return loopback_queues_destroy(pkt_loop->loopqs, pkt_loop->num_qs); } -static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, - odp_packet_t pkts[], int num) +static int loopback_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t pkts[], int num) { int nbr, i; _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; @@ -201,9 +239,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (odp_unlikely(num > QUEUE_MULTI_MAX)) num = QUEUE_MULTI_MAX; - odp_ticketlock_lock(&pktio_entry->rxl); - - queue = pkt_priv(pktio_entry)->loopq; + queue = pkt_priv(pktio_entry)->loopqs[index].queue; nbr = odp_queue_deq_multi(queue, (odp_event_t *)hdr_tbl, num); if (opt.bit.ts_all || opt.bit.ts_ptp) { @@ -288,8 +324,6 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pktio_entry->stats.in_octets += octets; pktio_entry->stats.in_packets += packets; - odp_ticketlock_unlock(&pktio_entry->rxl); - return num_rx; } @@ -381,8 +415,8 @@ static inline void loopback_fix_checksums(odp_packet_t pkt, _odp_packet_sctp_chksum_insert(pkt); } -static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, - const odp_packet_t pkt_tbl[], int num) +static int loopback_send(pktio_entry_t *pktio_entry, int index, const odp_packet_t pkt_tbl[], + int num) { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; @@ -395,8 +429,12 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, uint32_t bytes = 0; uint32_t out_octets_tbl[num]; odp_pktout_config_opt_t *pktout_cfg = &pktio_entry->config.pktout; - odp_pktout_config_opt_t *pktout_capa = - &pktio_entry->capa.config.pktout; + odp_pktout_config_opt_t *pktout_capa = &pktio_entry->capa.config.pktout; + + if (pkt_loop->num_qs == 0) + return 0; + + memset(hdr_tbl, 0, sizeof(hdr_tbl)); if (odp_unlikely(num > QUEUE_MULTI_MAX)) num = QUEUE_MULTI_MAX; @@ -430,9 +468,7 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, loopback_fix_checksums(pkt_tbl[i], pktout_cfg, pktout_capa); } - odp_ticketlock_lock(&pktio_entry->txl); - - queue = pkt_priv(pktio_entry)->loopq; + queue = pkt_loop->loopqs[index % pkt_loop->num_qs].queue; ret = odp_queue_enq_multi(queue, (odp_event_t *)hdr_tbl, nb_tx); if (ret > 0) { @@ -446,8 +482,6 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, ret = -1; } - odp_ticketlock_unlock(&pktio_entry->txl); - return ret; } @@ -508,8 +542,8 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) memset(capa, 0, sizeof(odp_pktio_capability_t)); - capa->max_input_queues = 1; - capa->max_output_queues = 1; + capa->max_input_queues = ODP_PKTIN_MAX_QUEUES; + capa->max_output_queues = ODP_PKTOUT_MAX_QUEUES; capa->set_op.op.promisc_mode = 0; capa->set_op.op.maxlen = 1; -- cgit v1.2.3 From fa2ec099d0e168f484098d5c66bec8611528e9ee Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Tue, 27 Dec 2022 13:17:14 +0000 Subject: linux-gen: loop: refactor statistics Add new atomic, queue specific statistics and remove usage of packet I/O level common statistics. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- platform/linux-generic/pktio/loop.c | 109 +++++++++++++++++++++++++++--------- 1 file changed, 82 insertions(+), 27 deletions(-) diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index f5190b96f..817dfb622 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -38,6 +38,9 @@ #include #include +#define MAX_QUEUES (ODP_PKTIN_MAX_QUEUES > ODP_PKTOUT_MAX_QUEUES ? \ + ODP_PKTIN_MAX_QUEUES : ODP_PKTOUT_MAX_QUEUES) + #define MAX_LOOP 16 #define LOOP_MTU_MIN 68 @@ -45,9 +48,20 @@ #define LOOP_MAX_QUEUE_SIZE 1024 +typedef struct { + odp_atomic_u64_t in_octets; + odp_atomic_u64_t in_packets; + odp_atomic_u64_t in_discards; + odp_atomic_u64_t in_errors; + odp_atomic_u64_t out_octets; + odp_atomic_u64_t out_packets; +} stats_t; + typedef struct ODP_ALIGNED_CACHE { /* queue handle as the "wire" */ odp_queue_t queue; + /* queue specific statistics */ + stats_t stats; /* config input queue size */ uint32_t in_size; /* config output queue size */ @@ -56,7 +70,7 @@ typedef struct ODP_ALIGNED_CACHE { typedef struct { /* loopback entries for "loop" device */ - loop_queue_t loopqs[ODP_PKTIN_MAX_QUEUES]; + loop_queue_t loopqs[MAX_QUEUES]; /* config queue count */ uint32_t num_conf_qs; /* actual number queues */ @@ -80,7 +94,6 @@ static inline pkt_loop_t *pkt_priv(pktio_entry_t *pktio_entry) /* MAC address for the "loop" interface */ static const uint8_t pktio_loop_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x01}; -static int loopback_stats_reset(pktio_entry_t *pktio_entry); static int loopback_init_capability(pktio_entry_t *pktio_entry); static int loopback_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, @@ -105,9 +118,17 @@ static int loopback_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry, pkt_loop->mtu = LOOP_MTU_MAX; pkt_loop->idx = idx; pkt_loop->queue_create = 1; - loopback_stats_reset(pktio_entry); loopback_init_capability(pktio_entry); + for (uint32_t i = 0; i < MAX_QUEUES; i++) { + odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.in_octets, 0); + odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.in_packets, 0); + odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.in_discards, 0); + odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.in_errors, 0); + odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.out_octets, 0); + odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.out_packets, 0); + } + return 0; } @@ -186,7 +207,7 @@ static int loopback_pktin_queue_config(pktio_entry_t *pktio_entry, pkt_loop->queue_create = 1; if (pktio_entry->param.in_mode == ODP_PKTIN_MODE_DIRECT) { - for (uint32_t i = 0; i < ODP_PKTIN_MAX_QUEUES; i++) { + for (uint32_t i = 0; i < MAX_QUEUES; i++) { if (i < pkt_loop->num_conf_qs) pkt_loop->loopqs[i].in_size = param->queue_size[i]; else @@ -204,7 +225,7 @@ static int loopback_pktout_queue_config(pktio_entry_t *pktio_entry, pkt_loop->queue_create = 1; - for (uint32_t i = 0; i < ODP_PKTIN_MAX_QUEUES; i++) { + for (uint32_t i = 0; i < MAX_QUEUES; i++) { if (i < param->num_queues) pkt_loop->loopqs[i].out_size = param->queue_size[i]; else @@ -224,8 +245,10 @@ static int loopback_close(pktio_entry_t *pktio_entry) static int loopback_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t pkts[], int num) { int nbr, i; + loop_queue_t *entry = &pkt_priv(pktio_entry)->loopqs[index]; + odp_queue_t queue = entry->queue; + stats_t *stats = &entry->stats; _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; - odp_queue_t queue; odp_packet_hdr_t *pkt_hdr; odp_packet_t pkt; odp_time_t ts_val; @@ -239,7 +262,6 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t pkt if (odp_unlikely(num > QUEUE_MULTI_MAX)) num = QUEUE_MULTI_MAX; - queue = pkt_priv(pktio_entry)->loopqs[index].queue; nbr = odp_queue_deq_multi(queue, (odp_event_t *)hdr_tbl, num); if (opt.bit.ts_all || opt.bit.ts_ptp) { @@ -275,7 +297,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t pkt ret = _odp_packet_parse_common(pkt_hdr, pkt_addr, pkt_len, seg_len, layer, opt); if (ret) - odp_atomic_inc_u64(&pktio_entry->stats_extra.in_errors); + odp_atomic_inc_u64(&stats->in_errors); if (ret < 0) { odp_packet_free(pkt); @@ -288,7 +310,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t pkt ret = _odp_cls_classify_packet(pktio_entry, pkt_addr, &new_pool, pkt_hdr); if (ret < 0) - odp_atomic_inc_u64(&pktio_entry->stats_extra.in_discards); + odp_atomic_inc_u64(&stats->in_discards); if (ret) { odp_packet_free(pkt); @@ -298,7 +320,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t pkt if (odp_unlikely(_odp_pktio_packet_to_pool( &pkt, &pkt_hdr, new_pool))) { odp_packet_free(pkt); - odp_atomic_inc_u64(&pktio_entry->stats_extra.in_discards); + odp_atomic_inc_u64(&stats->in_discards); continue; } } @@ -321,8 +343,8 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t pkt pkts[num_rx++] = pkt; } - pktio_entry->stats.in_octets += octets; - pktio_entry->stats.in_packets += packets; + odp_atomic_add_u64(&stats->in_octets, octets); + odp_atomic_add_u64(&stats->in_packets, packets); return num_rx; } @@ -421,6 +443,7 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index, const odp_packet pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; odp_queue_t queue; + stats_t *stats; int i; int ret; int nb_tx = 0; @@ -469,14 +492,15 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index, const odp_packet } queue = pkt_loop->loopqs[index % pkt_loop->num_qs].queue; + stats = &pkt_loop->loopqs[index].stats; ret = odp_queue_enq_multi(queue, (odp_event_t *)hdr_tbl, nb_tx); if (ret > 0) { if (odp_unlikely(tx_ts_idx) && ret >= tx_ts_idx) _odp_pktio_tx_ts_set(pktio_entry); - pktio_entry->stats.out_packets += ret; - pktio_entry->stats.out_octets += out_octets_tbl[ret - 1]; + odp_atomic_add_u64(&stats->out_packets, ret); + odp_atomic_add_u64(&stats->out_octets, out_octets_tbl[ret - 1]); } else { _ODP_DBG("queue enqueue failed %i\n", ret); ret = -1; @@ -600,6 +624,7 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry) capa->stats.pktin_queue.counter.octets = 1; capa->stats.pktin_queue.counter.packets = 1; capa->stats.pktin_queue.counter.errors = 1; + capa->stats.pktin_queue.counter.discards = 1; capa->stats.pktout_queue.counter.octets = 1; capa->stats.pktout_queue.counter.packets = 1; return 0; @@ -616,37 +641,67 @@ static int loopback_promisc_mode_get(pktio_entry_t *pktio_entry ODP_UNUSED) return 1; } -static int loopback_stats(pktio_entry_t *pktio_entry, - odp_pktio_stats_t *stats) +static int loopback_stats(pktio_entry_t *pktio_entry, odp_pktio_stats_t *stats) { - memcpy(stats, &pktio_entry->stats, sizeof(odp_pktio_stats_t)); + pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); + + memset(stats, 0, sizeof(odp_pktio_stats_t)); + + for (uint32_t i = 0; i < MAX_QUEUES; i++) { + stats_t *qs = &pkt_loop->loopqs[i].stats; + + stats->in_octets += odp_atomic_load_u64(&qs->in_octets); + stats->in_packets += odp_atomic_load_u64(&qs->in_packets); + stats->in_discards += odp_atomic_load_u64(&qs->in_discards); + stats->in_errors += odp_atomic_load_u64(&qs->in_errors); + stats->out_octets += odp_atomic_load_u64(&qs->out_octets); + stats->out_packets += odp_atomic_load_u64(&qs->out_packets); + } + return 0; } static int loopback_stats_reset(pktio_entry_t *pktio_entry) { - memset(&pktio_entry->stats, 0, sizeof(odp_pktio_stats_t)); + pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); + + for (uint32_t i = 0; i < MAX_QUEUES; i++) { + stats_t *qs = &pkt_loop->loopqs[i].stats; + + odp_atomic_store_u64(&qs->in_octets, 0); + odp_atomic_store_u64(&qs->in_packets, 0); + odp_atomic_store_u64(&qs->in_discards, 0); + odp_atomic_store_u64(&qs->in_errors, 0); + odp_atomic_store_u64(&qs->out_octets, 0); + odp_atomic_store_u64(&qs->out_packets, 0); + } + return 0; } -static int loopback_pktin_stats(pktio_entry_t *pktio_entry, - uint32_t index ODP_UNUSED, +static int loopback_pktin_stats(pktio_entry_t *pktio_entry, uint32_t index, odp_pktin_queue_stats_t *pktin_stats) { + stats_t *qs = &pkt_priv(pktio_entry)->loopqs[index].stats; + memset(pktin_stats, 0, sizeof(odp_pktin_queue_stats_t)); - pktin_stats->octets = pktio_entry->stats.in_octets; - pktin_stats->packets = pktio_entry->stats.in_packets; - pktin_stats->errors = pktio_entry->stats.in_errors; + pktin_stats->octets = odp_atomic_load_u64(&qs->in_octets); + pktin_stats->packets = odp_atomic_load_u64(&qs->in_packets); + pktin_stats->discards = odp_atomic_load_u64(&qs->in_discards); + pktin_stats->errors = odp_atomic_load_u64(&qs->in_errors); + return 0; } -static int loopback_pktout_stats(pktio_entry_t *pktio_entry, - uint32_t index ODP_UNUSED, +static int loopback_pktout_stats(pktio_entry_t *pktio_entry, uint32_t index, odp_pktout_queue_stats_t *pktout_stats) { + stats_t *qs = &pkt_priv(pktio_entry)->loopqs[index].stats; + memset(pktout_stats, 0, sizeof(odp_pktout_queue_stats_t)); - pktout_stats->octets = pktio_entry->stats.out_octets; - pktout_stats->packets = pktio_entry->stats.out_packets; + pktout_stats->octets = odp_atomic_load_u64(&qs->out_octets); + pktout_stats->packets = odp_atomic_load_u64(&qs->out_packets); + return 0; } -- cgit v1.2.3 From 02496026906cb05b5955e7bf1ab5bb82849bcd2f Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Wed, 28 Dec 2022 13:58:37 +0000 Subject: linux-gen: loop: add support for flow hashing Add support for flow hashing. When sending packets, distribute them among configured loopback queues based on active flow hashing configuration. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- platform/linux-generic/pktio/loop.c | 104 +++++++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 26 deletions(-) diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 817dfb622..a69daf56b 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -71,6 +72,8 @@ typedef struct ODP_ALIGNED_CACHE { typedef struct { /* loopback entries for "loop" device */ loop_queue_t loopqs[MAX_QUEUES]; + /* hash config */ + odp_pktin_hash_proto_t hash; /* config queue count */ uint32_t num_conf_qs; /* actual number queues */ @@ -205,6 +208,7 @@ static int loopback_pktin_queue_config(pktio_entry_t *pktio_entry, pkt_loop->num_conf_qs = param->num_queues; pkt_loop->queue_create = 1; + pkt_loop->hash.all_bits = param->hash_enable ? param->hash_proto.all_bits : 0; if (pktio_entry->param.in_mode == ODP_PKTIN_MODE_DIRECT) { for (uint32_t i = 0; i < MAX_QUEUES; i++) { @@ -437,11 +441,68 @@ static inline void loopback_fix_checksums(odp_packet_t pkt, _odp_packet_sctp_chksum_insert(pkt); } +static inline uint8_t *add_data(uint8_t *data, void *src, uint32_t len) +{ + return (uint8_t *)memcpy(data, src, len) + len; +} + +static inline odp_queue_t get_dest_queue(const pkt_loop_t *pkt_loop, odp_packet_t pkt, int index) +{ + const odp_pktin_hash_proto_t *hash = &pkt_loop->hash; + _odp_udphdr_t udp; + _odp_tcphdr_t tcp; + _odp_ipv4hdr_t ipv4; + _odp_ipv6hdr_t ipv6; + uint32_t off; + /* Space for UDP/TCP source and destination ports and IPv4/IPv6 source and destination + * addresses. */ + uint8_t data[2 * sizeof(uint16_t) + 2 * 4 * sizeof(uint32_t)]; + uint8_t *head = data; + + if (hash->all_bits == 0) + return pkt_loop->loopqs[index % pkt_loop->num_qs].queue; + + memset(data, 0, sizeof(data)); + off = odp_packet_l4_offset(pkt); + + if (off != ODP_PACKET_OFFSET_INVALID) { + if ((hash->proto.ipv4_udp || hash->proto.ipv6_udp) && odp_packet_has_udp(pkt)) { + if (odp_packet_copy_to_mem(pkt, off, _ODP_UDPHDR_LEN, &udp) == 0) { + head = add_data(head, &udp.src_port, sizeof(udp.src_port)); + head = add_data(head, &udp.dst_port, sizeof(udp.dst_port)); + } + } else if ((hash->proto.ipv4_tcp || hash->proto.ipv6_tcp) && + odp_packet_has_tcp(pkt)) { + if (odp_packet_copy_to_mem(pkt, off, _ODP_TCPHDR_LEN, &tcp) == 0) { + head = add_data(head, &tcp.src_port, sizeof(tcp.src_port)); + head = add_data(head, &tcp.dst_port, sizeof(tcp.dst_port)); + } + } + } + + off = odp_packet_l3_offset(pkt); + + if (off != ODP_PACKET_OFFSET_INVALID) { + if (hash->proto.ipv4 && odp_packet_has_ipv4(pkt)) { + if (odp_packet_copy_to_mem(pkt, off, _ODP_IPV4HDR_LEN, &ipv4) == 0) { + head = add_data(head, &ipv4.src_addr, sizeof(ipv4.src_addr)); + head = add_data(head, &ipv4.dst_addr, sizeof(ipv4.dst_addr)); + } + } else if (hash->proto.ipv6 && odp_packet_has_ipv6(pkt)) { + if (odp_packet_copy_to_mem(pkt, off, _ODP_IPV6HDR_LEN, &ipv6) == 0) { + head = add_data(head, &ipv6.src_addr, sizeof(ipv6.src_addr)); + head = add_data(head, &ipv6.dst_addr, sizeof(ipv6.dst_addr)); + } + } + } + + return pkt_loop->loopqs[odp_hash_crc32c(data, head - data, 0) % pkt_loop->num_qs].queue; +} + static int loopback_send(pktio_entry_t *pktio_entry, int index, const odp_packet_t pkt_tbl[], int num) { pkt_loop_t *pkt_loop = pkt_priv(pktio_entry); - _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; odp_queue_t queue; stats_t *stats; int i; @@ -449,15 +510,13 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index, const odp_packet int nb_tx = 0; int tx_ts_idx = 0; uint8_t tx_ts_enabled = _odp_pktio_tx_ts_enabled(pktio_entry); - uint32_t bytes = 0; - uint32_t out_octets_tbl[num]; odp_pktout_config_opt_t *pktout_cfg = &pktio_entry->config.pktout; odp_pktout_config_opt_t *pktout_capa = &pktio_entry->capa.config.pktout; if (pkt_loop->num_qs == 0) return 0; - memset(hdr_tbl, 0, sizeof(hdr_tbl)); + stats = &pkt_loop->loopqs[index].stats; if (odp_unlikely(num > QUEUE_MULTI_MAX)) num = QUEUE_MULTI_MAX; @@ -472,41 +531,34 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index, const odp_packet } break; } - hdr_tbl[i] = packet_to_event_hdr(pkt_tbl[i]); - bytes += pkt_len; - /* Store cumulative byte counts to update 'stats.out_octets' - * correctly in case enq_multi() fails to enqueue all packets. - */ - out_octets_tbl[i] = bytes; - nb_tx++; if (tx_ts_enabled && tx_ts_idx == 0) { if (odp_unlikely(packet_hdr(pkt_tbl[i])->p.flags.ts_set)) tx_ts_idx = i + 1; } - } - for (i = 0; i < nb_tx; ++i) { packet_subtype_set(pkt_tbl[i], ODP_EVENT_PACKET_BASIC); loopback_fix_checksums(pkt_tbl[i], pktout_cfg, pktout_capa); - } + queue = get_dest_queue(pkt_loop, pkt_tbl[i], index); + ret = odp_queue_enq(queue, odp_packet_to_event(pkt_tbl[i])); + + if (ret < 0) { + _ODP_DBG("queue enqueue failed %i to queue: %" PRIu64 "\n", ret, + odp_queue_to_u64(queue)); + break; + } - queue = pkt_loop->loopqs[index % pkt_loop->num_qs].queue; - stats = &pkt_loop->loopqs[index].stats; - ret = odp_queue_enq_multi(queue, (odp_event_t *)hdr_tbl, nb_tx); + nb_tx++; + odp_atomic_inc_u64(&stats->out_packets); + odp_atomic_add_u64(&stats->out_octets, pkt_len); + } - if (ret > 0) { - if (odp_unlikely(tx_ts_idx) && ret >= tx_ts_idx) + if (nb_tx > 0) { + if (odp_unlikely(tx_ts_idx) && nb_tx >= tx_ts_idx) _odp_pktio_tx_ts_set(pktio_entry); - - odp_atomic_add_u64(&stats->out_packets, ret); - odp_atomic_add_u64(&stats->out_octets, out_octets_tbl[ret - 1]); - } else { - _ODP_DBG("queue enqueue failed %i\n", ret); - ret = -1; } - return ret; + return nb_tx; } static uint32_t loopback_mtu_get(pktio_entry_t *pktio_entry) -- cgit v1.2.3 From 8aee6094ef983d1f537d738b8a87186fb61e7cdd Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 13 Jan 2023 16:23:30 +0200 Subject: linux-gen: stash: reserve all memory from a single shm block Use one common SHM block for all stash memory. This enables creating a large number of stashes without hitting the SHM block count limit, for the cost of having to pre-reserve the memory during global init. The amount of required memory can be adjusted with new configuration file options: - stash:max_num: maximum number of stashes - stash:max_num_obj: maximum number of objects in a stash Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- config/odp-linux-generic.conf | 13 +- .../linux-generic/include/odp_config_internal.h | 10 +- platform/linux-generic/m4/odp_libconfig.m4 | 2 +- platform/linux-generic/odp_stash.c | 148 +++++++++++++-------- platform/linux-generic/test/inline-timer.conf | 2 +- platform/linux-generic/test/packet_align.conf | 2 +- platform/linux-generic/test/process-mode.conf | 2 +- platform/linux-generic/test/sched-basic.conf | 2 +- 8 files changed, 114 insertions(+), 67 deletions(-) diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index 814035d61..3cb862ee4 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.23" +config_file_version = "0.1.24" # System options system: { @@ -245,6 +245,17 @@ sched_basic: { order_stash_size = 512 } +stash: { + # Maximum number of stashes + max_num = 512 + + # Maximum number of objects in a stash + # + # The value may be rounded up by the implementation. For optimal memory + # usage set value to a power of two - 1. + max_num_obj = 4095 +} + timer: { # Use inline timer implementation # diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index d3d09abf4..e4f8d6d6d 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -66,7 +66,7 @@ extern "C" { /* * Maximum number of stashes */ -#define CONFIG_MAX_STASHES 128 +#define CONFIG_MAX_STASHES 2048 /* * Maximum number of packet IO resources @@ -134,10 +134,10 @@ extern "C" { /* * Number of shared memory blocks reserved for implementation internal use. * - * Each stash requires one SHM block, each pool requires three blocks (buffers, - * ring, user area), and 20 blocks are reserved for per ODP module global data. + * Each pool requires three blocks (buffers, ring, user area), and 20 blocks + * are reserved for per ODP module global data. */ -#define CONFIG_INTERNAL_SHM_BLOCKS (CONFIG_MAX_STASHES + (ODP_CONFIG_POOLS * 3) + 20) +#define CONFIG_INTERNAL_SHM_BLOCKS ((ODP_CONFIG_POOLS * 3) + 20) /* * Maximum number of shared memory blocks. diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4 index 658138c31..d96a95055 100644 --- a/platform/linux-generic/m4/odp_libconfig.m4 +++ b/platform/linux-generic/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [23]) +m4_define([_odp_config_version_minor], [24]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index c7d4136ab..12e52afec 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2020-2022, Nokia +/* Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -26,14 +27,18 @@ ODP_STATIC_ASSERT(CONFIG_INTERNAL_STASHES < CONFIG_MAX_STASHES, "TOO_MANY_INTERNAL_STASHES"); -#define MAX_RING_SIZE (1024 * 1024) #define MIN_RING_SIZE 64 +enum { + STASH_FREE = 0, + STASH_RESERVED, + STASH_ACTIVE +}; + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpedantic" typedef struct stash_t { char name[ODP_STASH_NAME_LEN]; - odp_shm_t shm; int index; uint32_t ring_mask; uint32_t obj_size; @@ -57,8 +62,12 @@ typedef struct stash_t { typedef struct stash_global_t { odp_ticketlock_t lock; odp_shm_t shm; - uint8_t stash_reserved[CONFIG_MAX_STASHES]; + uint32_t max_num; + uint32_t max_num_obj; + uint32_t num_internal; + uint8_t stash_state[CONFIG_MAX_STASHES]; stash_t *stash[CONFIG_MAX_STASHES]; + uint8_t data[] ODP_ALIGNED_CACHE; } stash_global_t; @@ -67,13 +76,58 @@ static stash_global_t *stash_global; int _odp_stash_init_global(void) { odp_shm_t shm; - - if (odp_global_ro.disable.stash) { + uint32_t max_num, max_num_obj; + const char *str; + uint64_t ring_max_size, stash_max_size, stash_data_size, offset; + const uint32_t internal_stashes = odp_global_ro.disable.dma ? 0 : CONFIG_INTERNAL_STASHES; + uint8_t *stash_data; + int val = 0; + + if (odp_global_ro.disable.stash && odp_global_ro.disable.dma) { _ODP_PRINT("Stash is DISABLED\n"); return 0; } - shm = odp_shm_reserve("_odp_stash_global", sizeof(stash_global_t), + _ODP_PRINT("Stash config:\n"); + + str = "stash.max_num"; + if (!_odp_libconfig_lookup_int(str, &val)) { + _ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + _ODP_PRINT(" %s: %i\n", str, val); + max_num = val; + + str = "stash.max_num_obj"; + if (!_odp_libconfig_lookup_int(str, &val)) { + _ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + _ODP_PRINT(" %s: %i\n", str, val); + max_num_obj = val; + + _ODP_PRINT("\n"); + + /* Reserve resources for implementation internal stashes */ + if (max_num > CONFIG_MAX_STASHES - internal_stashes) { + _ODP_ERR("Maximum supported number of stashes: %d\n", + CONFIG_MAX_STASHES - internal_stashes); + return -1; + } + max_num += internal_stashes; + + /* Must have room for minimum sized ring */ + if (max_num_obj < MIN_RING_SIZE) + max_num_obj = MIN_RING_SIZE - 1; + + /* Ring size must be larger than the number of items stored */ + ring_max_size = _ODP_ROUNDUP_POWER2_U32(max_num_obj + 1); + + stash_max_size = _ODP_ROUNDUP_CACHE_LINE(sizeof(stash_t) + + (ring_max_size * sizeof(uint64_t))); + stash_data_size = max_num * stash_max_size; + + shm = odp_shm_reserve("_odp_stash_global", sizeof(stash_global_t) + stash_data_size, ODP_CACHE_LINE_SIZE, 0); stash_global = odp_shm_addr(shm); @@ -85,8 +139,20 @@ int _odp_stash_init_global(void) memset(stash_global, 0, sizeof(stash_global_t)); stash_global->shm = shm; + stash_global->max_num = max_num; + stash_global->max_num_obj = max_num_obj; + stash_global->num_internal = internal_stashes; odp_ticketlock_init(&stash_global->lock); + /* Initialize stash pointers */ + stash_data = stash_global->data; + offset = 0; + + for (uint32_t i = 0; i < max_num; i++) { + stash_global->stash[i] = (stash_t *)(uintptr_t)(stash_data + offset); + offset += stash_max_size; + } + return 0; } @@ -108,17 +174,21 @@ int _odp_stash_term_global(void) int odp_stash_capability(odp_stash_capability_t *capa, odp_stash_type_t type) { + uint32_t max_stashes; + if (odp_global_ro.disable.stash) { _ODP_ERR("Stash is disabled\n"); return -1; } (void)type; + max_stashes = stash_global->max_num - stash_global->num_internal; + memset(capa, 0, sizeof(odp_stash_capability_t)); - capa->max_stashes_any_type = CONFIG_MAX_STASHES - CONFIG_INTERNAL_STASHES; - capa->max_stashes = CONFIG_MAX_STASHES - CONFIG_INTERNAL_STASHES; - capa->max_num_obj = MAX_RING_SIZE; + capa->max_stashes_any_type = max_stashes; + capa->max_stashes = max_stashes; + capa->max_num_obj = stash_global->max_num_obj; capa->max_obj_size = sizeof(uint64_t); capa->max_get_batch = MIN_RING_SIZE; capa->max_put_batch = MIN_RING_SIZE; @@ -137,15 +207,14 @@ void odp_stash_param_init(odp_stash_param_t *param) static int reserve_index(void) { - int i; int index = -1; odp_ticketlock_lock(&stash_global->lock); - for (i = 0; i < CONFIG_MAX_STASHES; i++) { - if (stash_global->stash_reserved[i] == 0) { + for (uint32_t i = 0; i < stash_global->max_num; i++) { + if (stash_global->stash_state[i] == STASH_FREE) { index = i; - stash_global->stash_reserved[i] = 1; + stash_global->stash_state[i] = STASH_RESERVED; break; } } @@ -159,20 +228,16 @@ static void free_index(int i) { odp_ticketlock_lock(&stash_global->lock); - stash_global->stash[i] = NULL; - stash_global->stash_reserved[i] = 0; + stash_global->stash_state[i] = STASH_FREE; odp_ticketlock_unlock(&stash_global->lock); } odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) { - odp_shm_t shm; stash_t *stash; - uint64_t i, ring_size, shm_size; + uint64_t i, ring_size; int ring_u64, index; - char shm_name[ODP_STASH_NAME_LEN + 8]; - uint32_t shm_flags = 0; if (odp_global_ro.disable.stash) { _ODP_ERR("Stash is disabled\n"); @@ -184,7 +249,7 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) return ODP_STASH_INVALID; } - if (param->num_obj > MAX_RING_SIZE) { + if (param->num_obj > stash_global->max_num_obj) { _ODP_ERR("Too many objects.\n"); return ODP_STASH_INVALID; } @@ -213,26 +278,7 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) else ring_size = _ODP_ROUNDUP_POWER2_U32(ring_size + 1); - memset(shm_name, 0, sizeof(shm_name)); - snprintf(shm_name, sizeof(shm_name) - 1, "_stash_%s", name); - - if (ring_u64) - shm_size = sizeof(stash_t) + (ring_size * sizeof(uint64_t)); - else - shm_size = sizeof(stash_t) + (ring_size * sizeof(uint32_t)); - - if (odp_global_ro.shm_single_va) - shm_flags |= ODP_SHM_SINGLE_VA; - - shm = odp_shm_reserve(shm_name, shm_size, ODP_CACHE_LINE_SIZE, shm_flags); - - if (shm == ODP_SHM_INVALID) { - _ODP_ERR("SHM reserve failed.\n"); - free_index(index); - return ODP_STASH_INVALID; - } - - stash = odp_shm_addr(shm); + stash = stash_global->stash[index]; memset(stash, 0, sizeof(stash_t)); if (ring_u64) { @@ -251,13 +297,12 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) strcpy(stash->name, name); stash->index = index; - stash->shm = shm; stash->obj_size = param->obj_size; stash->ring_mask = ring_size - 1; /* This makes stash visible to lookups */ odp_ticketlock_lock(&stash_global->lock); - stash_global->stash[index] = stash; + stash_global->stash_state[index] = STASH_ACTIVE; odp_ticketlock_unlock(&stash_global->lock); return (odp_stash_t)stash; @@ -266,22 +311,13 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) int odp_stash_destroy(odp_stash_t st) { stash_t *stash; - int index; - odp_shm_t shm; if (st == ODP_STASH_INVALID) return -1; stash = (stash_t *)(uintptr_t)st; - index = stash->index; - shm = stash->shm; - - free_index(index); - if (odp_shm_free(shm)) { - _ODP_ERR("SHM free failed.\n"); - return -1; - } + free_index(stash->index); return 0; } @@ -293,7 +329,6 @@ uint64_t odp_stash_to_u64(odp_stash_t st) odp_stash_t odp_stash_lookup(const char *name) { - int i; stash_t *stash; if (name == NULL) @@ -301,10 +336,11 @@ odp_stash_t odp_stash_lookup(const char *name) odp_ticketlock_lock(&stash_global->lock); - for (i = 0; i < CONFIG_MAX_STASHES; i++) { + for (uint32_t i = 0; i < stash_global->max_num; i++) { stash = stash_global->stash[i]; - if (stash && strcmp(stash->name, name) == 0) { + if (stash_global->stash_state[i] == STASH_ACTIVE && + strcmp(stash->name, name) == 0) { odp_ticketlock_unlock(&stash_global->lock); return (odp_stash_t)stash; } diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index 5afcad2cb..56c1d40d1 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.23" +config_file_version = "0.1.24" timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf index 9aa6109a6..1aa7efc69 100644 --- a/platform/linux-generic/test/packet_align.conf +++ b/platform/linux-generic/test/packet_align.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.23" +config_file_version = "0.1.24" pool: { pkt: { diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index cf32fb86d..fb59a4f2a 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.23" +config_file_version = "0.1.24" # Shared memory options shm: { diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf index 5710c43c4..f75141806 100644 --- a/platform/linux-generic/test/sched-basic.conf +++ b/platform/linux-generic/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.23" +config_file_version = "0.1.24" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { -- cgit v1.2.3 From 113b1c421623f01c938d8e1218088f46e3535bca Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 18 Jan 2023 13:54:57 +0200 Subject: linux-gen: stash: add helper functions for stash handle conversions Replace stash handle casts with internal helper functions stash_entry() and stash_handle(). Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-generic/odp_stash.c | 50 ++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index 12e52afec..fecbabac3 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -73,6 +73,16 @@ typedef struct stash_global_t { static stash_global_t *stash_global; +static inline stash_t *stash_entry(odp_stash_t st) +{ + return (stash_t *)(uintptr_t)st; +} + +static inline odp_stash_t stash_handle(stash_t *stash) +{ + return (odp_stash_t)(uintptr_t)stash; +} + int _odp_stash_init_global(void) { odp_shm_t shm; @@ -305,19 +315,15 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) stash_global->stash_state[index] = STASH_ACTIVE; odp_ticketlock_unlock(&stash_global->lock); - return (odp_stash_t)stash; + return stash_handle(stash); } int odp_stash_destroy(odp_stash_t st) { - stash_t *stash; - if (st == ODP_STASH_INVALID) return -1; - stash = (stash_t *)(uintptr_t)st; - - free_index(stash->index); + free_index(stash_entry(st)->index); return 0; } @@ -342,7 +348,7 @@ odp_stash_t odp_stash_lookup(const char *name) if (stash_global->stash_state[i] == STASH_ACTIVE && strcmp(stash->name, name) == 0) { odp_ticketlock_unlock(&stash_global->lock); - return (odp_stash_t)stash; + return stash_handle(stash); } } @@ -353,12 +359,10 @@ odp_stash_t odp_stash_lookup(const char *name) static inline int32_t stash_put(odp_stash_t st, const void *obj, int32_t num) { - stash_t *stash; + stash_t *stash = stash_entry(st); uint32_t obj_size; int32_t i; - stash = (stash_t *)(uintptr_t)st; - if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -421,7 +425,7 @@ int32_t odp_stash_put_batch(odp_stash_t st, const void *obj, int32_t num) static inline int32_t stash_put_u32(odp_stash_t st, const uint32_t val[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -448,7 +452,7 @@ int32_t odp_stash_put_u32_batch(odp_stash_t st, const uint32_t val[], static inline int32_t stash_put_u64(odp_stash_t st, const uint64_t val[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -475,7 +479,7 @@ int32_t odp_stash_put_u64_batch(odp_stash_t st, const uint64_t val[], static inline int32_t stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -508,12 +512,10 @@ int32_t odp_stash_put_ptr_batch(odp_stash_t st, const uintptr_t ptr[], static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool_t batch) { - stash_t *stash; + stash_t *stash = stash_entry(st); uint32_t obj_size; uint32_t i, num_deq; - stash = (stash_t *)(uintptr_t)st; - if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -584,7 +586,7 @@ int32_t odp_stash_get_batch(odp_stash_t st, void *obj, int32_t num) int32_t odp_stash_get_u32(odp_stash_t st, uint32_t val[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -597,7 +599,7 @@ int32_t odp_stash_get_u32(odp_stash_t st, uint32_t val[], int32_t num) int32_t odp_stash_get_u32_batch(odp_stash_t st, uint32_t val[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -609,7 +611,7 @@ int32_t odp_stash_get_u32_batch(odp_stash_t st, uint32_t val[], int32_t num) int32_t odp_stash_get_u64(odp_stash_t st, uint64_t val[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -622,7 +624,7 @@ int32_t odp_stash_get_u64(odp_stash_t st, uint64_t val[], int32_t num) int32_t odp_stash_get_u64_batch(odp_stash_t st, uint64_t val[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -634,7 +636,7 @@ int32_t odp_stash_get_u64_batch(odp_stash_t st, uint64_t val[], int32_t num) int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -654,7 +656,7 @@ int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num) int32_t odp_stash_get_ptr_batch(odp_stash_t st, uintptr_t ptr[], int32_t num) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; @@ -696,7 +698,7 @@ static uint32_t stash_obj_count(stash_t *stash) void odp_stash_print(odp_stash_t st) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (st == ODP_STASH_INVALID) { _ODP_ERR("Bad stash handle\n"); @@ -716,7 +718,7 @@ void odp_stash_print(odp_stash_t st) int odp_stash_stats(odp_stash_t st, odp_stash_stats_t *stats) { - stash_t *stash = (stash_t *)(uintptr_t)st; + stash_t *stash = stash_entry(st); if (st == ODP_STASH_INVALID) { _ODP_ERR("Bad stash handle\n"); -- cgit v1.2.3 From 579d8105776ea23f097fdb894a50759b6055b8b4 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 23 Jan 2023 13:01:26 +0200 Subject: api: buffer: split header files Split buffer API into separate header files for functions and types. This fixes circular dependency issues with inline headers. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- include/Makefile.am | 9 +++++ include/odp/api/abi-default/buffer.h | 15 +------- include/odp/api/abi-default/buffer_types.h | 34 ++++++++++++++++ include/odp/api/buffer_types.h | 28 ++++++++++++++ include/odp/api/spec/buffer.h | 15 ++------ include/odp/api/spec/buffer_types.h | 45 ++++++++++++++++++++++ .../arch/arm32-linux/odp/api/abi/buffer_types.h | 7 ++++ .../arch/arm64-linux/odp/api/abi/buffer_types.h | 7 ++++ .../arch/default-linux/odp/api/abi/buffer_types.h | 7 ++++ .../arch/power64-linux/odp/api/abi/buffer_types.h | 7 ++++ .../arch/x86_32-linux/odp/api/abi/buffer_types.h | 7 ++++ .../arch/x86_64-linux/odp/api/abi/buffer_types.h | 7 ++++ platform/linux-generic/Makefile.am | 1 + .../linux-generic/include-abi/odp/api/abi/buffer.h | 20 ++-------- .../include-abi/odp/api/abi/buffer_types.h | 40 +++++++++++++++++++ .../include/odp/api/plat/buffer_inlines.h | 5 +-- .../odp/api/plat/event_validation_external.h | 2 +- 17 files changed, 209 insertions(+), 47 deletions(-) create mode 100644 include/odp/api/abi-default/buffer_types.h create mode 100644 include/odp/api/buffer_types.h create mode 100644 include/odp/api/spec/buffer_types.h create mode 100644 include/odp/arch/arm32-linux/odp/api/abi/buffer_types.h create mode 100644 include/odp/arch/arm64-linux/odp/api/abi/buffer_types.h create mode 100644 include/odp/arch/default-linux/odp/api/abi/buffer_types.h create mode 100644 include/odp/arch/power64-linux/odp/api/abi/buffer_types.h create mode 100644 include/odp/arch/x86_32-linux/odp/api/abi/buffer_types.h create mode 100644 include/odp/arch/x86_64-linux/odp/api/abi/buffer_types.h create mode 100644 platform/linux-generic/include-abi/odp/api/abi/buffer_types.h diff --git a/include/Makefile.am b/include/Makefile.am index 7e9b8fedf..cc2f17ae2 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -13,6 +13,7 @@ odpapiinclude_HEADERS = \ odp/api/atomic.h \ odp/api/barrier.h \ odp/api/buffer.h \ + odp/api/buffer_types.h \ odp/api/byteorder.h \ odp/api/chksum.h \ odp/api/classification.h \ @@ -78,6 +79,7 @@ odpapispecinclude_HEADERS = \ odp/api/spec/atomic.h \ odp/api/spec/barrier.h \ odp/api/spec/buffer.h \ + odp/api/spec/buffer_types.h \ odp/api/spec/byteorder.h \ odp/api/spec/chksum.h \ odp/api/spec/classification.h \ @@ -146,6 +148,7 @@ odpapiabidefaultinclude_HEADERS = \ odp/api/abi-default/atomic.h \ odp/api/abi-default/barrier.h \ odp/api/abi-default/buffer.h \ + odp/api/abi-default/buffer_types.h \ odp/api/abi-default/byteorder.h \ odp/api/abi-default/classification.h \ odp/api/abi-default/comp.h \ @@ -205,6 +208,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm32-linux/odp/api/abi/atomic.h \ odp/arch/arm32-linux/odp/api/abi/barrier.h \ odp/arch/arm32-linux/odp/api/abi/buffer.h \ + odp/arch/arm32-linux/odp/api/abi/buffer_types.h \ odp/arch/arm32-linux/odp/api/abi/byteorder.h \ odp/arch/arm32-linux/odp/api/abi/classification.h \ odp/arch/arm32-linux/odp/api/abi/comp.h \ @@ -260,6 +264,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm64-linux/odp/api/abi/atomic.h \ odp/arch/arm64-linux/odp/api/abi/barrier.h \ odp/arch/arm64-linux/odp/api/abi/buffer.h \ + odp/arch/arm64-linux/odp/api/abi/buffer_types.h \ odp/arch/arm64-linux/odp/api/abi/byteorder.h \ odp/arch/arm64-linux/odp/api/abi/classification.h \ odp/arch/arm64-linux/odp/api/abi/comp.h \ @@ -315,6 +320,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/default-linux/odp/api/abi/atomic.h \ odp/arch/default-linux/odp/api/abi/barrier.h \ odp/arch/default-linux/odp/api/abi/buffer.h \ + odp/arch/default-linux/odp/api/abi/buffer_types.h \ odp/arch/default-linux/odp/api/abi/byteorder.h \ odp/arch/default-linux/odp/api/abi/classification.h \ odp/arch/default-linux/odp/api/abi/comp.h \ @@ -370,6 +376,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/power64-linux/odp/api/abi/atomic.h \ odp/arch/power64-linux/odp/api/abi/barrier.h \ odp/arch/power64-linux/odp/api/abi/buffer.h \ + odp/arch/power64-linux/odp/api/abi/buffer_types.h \ odp/arch/power64-linux/odp/api/abi/byteorder.h \ odp/arch/power64-linux/odp/api/abi/classification.h \ odp/arch/power64-linux/odp/api/abi/comp.h \ @@ -425,6 +432,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_32-linux/odp/api/abi/atomic.h \ odp/arch/x86_32-linux/odp/api/abi/barrier.h \ odp/arch/x86_32-linux/odp/api/abi/buffer.h \ + odp/arch/x86_32-linux/odp/api/abi/buffer_types.h \ odp/arch/x86_32-linux/odp/api/abi/byteorder.h \ odp/arch/x86_32-linux/odp/api/abi/classification.h \ odp/arch/x86_32-linux/odp/api/abi/comp.h \ @@ -480,6 +488,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_64-linux/odp/api/abi/atomic.h \ odp/arch/x86_64-linux/odp/api/abi/barrier.h \ odp/arch/x86_64-linux/odp/api/abi/buffer.h \ + odp/arch/x86_64-linux/odp/api/abi/buffer_types.h \ odp/arch/x86_64-linux/odp/api/abi/byteorder.h \ odp/arch/x86_64-linux/odp/api/abi/classification.h \ odp/arch/x86_64-linux/odp/api/abi/comp.h \ diff --git a/include/odp/api/abi-default/buffer.h b/include/odp/api/abi-default/buffer.h index 2bc27bba7..59b81186b 100644 --- a/include/odp/api/abi-default/buffer.h +++ b/include/odp/api/abi-default/buffer.h @@ -11,20 +11,7 @@ extern "C" { #endif -/** @internal Dummy type for strong typing */ -typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_buffer_t; - -/** @ingroup odp_buffer - * @{ - */ - -typedef _odp_abi_buffer_t *odp_buffer_t; - -#define ODP_BUFFER_INVALID ((odp_buffer_t)0) - -/** - * @} - */ +/* Empty header required due to the inline functions */ #ifdef __cplusplus } diff --git a/include/odp/api/abi-default/buffer_types.h b/include/odp/api/abi-default/buffer_types.h new file mode 100644 index 000000000..f01466151 --- /dev/null +++ b/include/odp/api/abi-default/buffer_types.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ABI_BUFFER_TYPES_H_ +#define ODP_ABI_BUFFER_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/** @internal Dummy type for strong typing */ +typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_buffer_t; + +/** @ingroup odp_buffer + * @{ + */ + +typedef _odp_abi_buffer_t *odp_buffer_t; + +#define ODP_BUFFER_INVALID ((odp_buffer_t)0) + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/buffer_types.h b/include/odp/api/buffer_types.h new file mode 100644 index 000000000..767aae560 --- /dev/null +++ b/include/odp/api/buffer_types.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP buffer types + */ + +#ifndef ODP_API_BUFFER_TYPES_H_ +#define ODP_API_BUFFER_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/spec/buffer.h b/include/odp/api/spec/buffer.h index cfb85df17..dd3749afe 100644 --- a/include/odp/api/spec/buffer.h +++ b/include/odp/api/spec/buffer.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -8,7 +8,7 @@ /** * @file * - * ODP buffer descriptor + * ODP buffer */ #ifndef ODP_API_SPEC_BUFFER_H_ @@ -19,6 +19,7 @@ extern "C" { #endif +#include #include #include #include @@ -28,16 +29,6 @@ extern "C" { * @{ */ -/** - * @typedef odp_buffer_t - * ODP buffer - */ - -/** - * @def ODP_BUFFER_INVALID - * Invalid buffer - */ - /** * Get buffer handle from event * diff --git a/include/odp/api/spec/buffer_types.h b/include/odp/api/spec/buffer_types.h new file mode 100644 index 000000000..7a7bc770f --- /dev/null +++ b/include/odp/api/spec/buffer_types.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP buffer types + */ + +#ifndef ODP_API_SPEC_BUFFER_TYPES_H_ +#define ODP_API_SPEC_BUFFER_TYPES_H_ +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup odp_buffer + * @{ + */ + +/** + * @typedef odp_buffer_t + * ODP buffer + */ + +/** + * @def ODP_BUFFER_INVALID + * Invalid buffer + */ + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include +#endif diff --git a/include/odp/arch/arm32-linux/odp/api/abi/buffer_types.h b/include/odp/arch/arm32-linux/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..8dd78321f --- /dev/null +++ b/include/odp/arch/arm32-linux/odp/api/abi/buffer_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/include/odp/arch/arm64-linux/odp/api/abi/buffer_types.h b/include/odp/arch/arm64-linux/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..8dd78321f --- /dev/null +++ b/include/odp/arch/arm64-linux/odp/api/abi/buffer_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/include/odp/arch/default-linux/odp/api/abi/buffer_types.h b/include/odp/arch/default-linux/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..8dd78321f --- /dev/null +++ b/include/odp/arch/default-linux/odp/api/abi/buffer_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/include/odp/arch/power64-linux/odp/api/abi/buffer_types.h b/include/odp/arch/power64-linux/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..8dd78321f --- /dev/null +++ b/include/odp/arch/power64-linux/odp/api/abi/buffer_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/buffer_types.h b/include/odp/arch/x86_32-linux/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..8dd78321f --- /dev/null +++ b/include/odp/arch/x86_32-linux/odp/api/abi/buffer_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/buffer_types.h b/include/odp/arch/x86_64-linux/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..8dd78321f --- /dev/null +++ b/include/odp/arch/x86_64-linux/odp/api/abi/buffer_types.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 43fcfa950..6e6cfa5f6 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -72,6 +72,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/atomic.h \ include-abi/odp/api/abi/barrier.h \ include-abi/odp/api/abi/buffer.h \ + include-abi/odp/api/abi/buffer_types.h \ include-abi/odp/api/abi/byteorder.h \ include-abi/odp/api/abi/classification.h \ include-abi/odp/api/abi/comp.h \ diff --git a/platform/linux-generic/include-abi/odp/api/abi/buffer.h b/platform/linux-generic/include-abi/odp/api/abi/buffer.h index 8239e15da..a6309fe39 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/buffer.h +++ b/platform/linux-generic/include-abi/odp/api/abi/buffer.h @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,7 +8,7 @@ /** * @file * - * ODP buffer descriptor + * ODP buffer */ #ifndef ODP_API_ABI_BUFFER_H_ @@ -17,24 +18,9 @@ extern "C" { #endif -#include -#include - -/** @ingroup odp_buffer - * @{ - */ - -typedef ODP_HANDLE_T(odp_buffer_t); - -#define ODP_BUFFER_INVALID _odp_cast_scalar(odp_buffer_t, 0) - -/* Inlined functions for non-ABI compat mode */ +/* Inlined API functions */ #include -/** - * @} - */ - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h b/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h new file mode 100644 index 000000000..1d54bab07 --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h @@ -0,0 +1,40 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP buffer types + */ + +#ifndef ODP_API_ABI_BUFFER_TYPES_H_ +#define ODP_API_ABI_BUFFER_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +/** @ingroup odp_buffer + * @{ + */ + +typedef ODP_HANDLE_T(odp_buffer_t); + +#define ODP_BUFFER_INVALID _odp_cast_scalar(odp_buffer_t, 0) + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 34d4b5675..75ef36cf3 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019-2022, Nokia +/* Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,11 +7,10 @@ #ifndef ODP_PLAT_BUFFER_INLINES_H_ #define ODP_PLAT_BUFFER_INLINES_H_ +#include #include #include -#include - #include #include #include diff --git a/platform/linux-generic/include/odp/api/plat/event_validation_external.h b/platform/linux-generic/include/odp/api/plat/event_validation_external.h index 5a939402f..7f5c0364f 100644 --- a/platform/linux-generic/include/odp/api/plat/event_validation_external.h +++ b/platform/linux-generic/include/odp/api/plat/event_validation_external.h @@ -18,7 +18,7 @@ #include -#include +#include #include #include #include -- cgit v1.2.3 From 8a4ed8578638e7a161aaa420a9770d7bd59f2086 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 27 Jan 2023 12:43:04 +0200 Subject: test: common: add reusable functions for checking packet metadata Add a common test library for setting and saving packet metadata and for comparing saved metadata. The functions can be used by different test modules that need to check that packet metadata has not changed during some operation under test. Signed-off-by: Janne Peltonen Reviewed-by: Matias Elo --- test/Makefile.inc | 1 + test/common/Makefile.am | 8 ++- test/common/packet_common.c | 133 ++++++++++++++++++++++++++++++++++++++++++++ test/common/packet_common.h | 67 ++++++++++++++++++++++ 4 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 test/common/packet_common.c create mode 100644 test/common/packet_common.h diff --git a/test/Makefile.inc b/test/Makefile.inc index 5764ac9b4..cc6a33df5 100644 --- a/test/Makefile.inc +++ b/test/Makefile.inc @@ -6,6 +6,7 @@ LIBODP = $(LIB)/libodphelper.la $(LIB)/lib$(ODP_LIB_NAME).la LIBCUNIT_COMMON = $(COMMON_DIR)/libcunit_common.la LIBCPUMASK_COMMON = $(COMMON_DIR)/libcpumask_common.la +LIBPACKET_COMMON = $(COMMON_DIR)/libpacket_common.la LIBTHRMASK_COMMON = $(COMMON_DIR)/libthrmask_common.la #in the following line, the libs using the symbols should come before diff --git a/test/common/Makefile.am b/test/common/Makefile.am index 694b442c2..72658df73 100644 --- a/test/common/Makefile.am +++ b/test/common/Makefile.am @@ -2,13 +2,19 @@ include $(top_srcdir)/test/Makefile.inc if cunit_support -noinst_LTLIBRARIES = libcunit_common.la libcpumask_common.la libthrmask_common.la +noinst_LTLIBRARIES = \ + libcunit_common.la \ + libcpumask_common.la \ + libpacket_common.la \ + libthrmask_common.la libcunit_common_la_SOURCES = odp_cunit_common.c odp_cunit_common.h libcunit_common_la_LIBADD = $(CUNIT_LIBS) libcpumask_common_la_SOURCES = mask_common.c mask_common.h +libpacket_common_la_SOURCES = packet_common.c packet_common.h + libthrmask_common_la_SOURCES = mask_common.c mask_common.h libthrmask_common_la_CFLAGS = $(AM_CFLAGS) -DTEST_THRMASK diff --git a/test/common/packet_common.c b/test/common/packet_common.c new file mode 100644 index 000000000..e0bca3147 --- /dev/null +++ b/test/common/packet_common.c @@ -0,0 +1,133 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include + +void test_packet_set_md(odp_packet_t pkt) +{ + const int binary_flag = 1; + const uint32_t offset = 1; + uint8_t *uarea = odp_packet_user_area(pkt); + uint32_t uarea_size = odp_packet_user_area_size(pkt); + + for (uint32_t n = 0; n < uarea_size; n++) + uarea[n] = n; + + /* + * Some of these flags cannot be set simultaneously, but we do not + * care here if some flags get cleared. + */ + odp_packet_has_l2_set(pkt, binary_flag); + odp_packet_has_l3_set(pkt, binary_flag); + odp_packet_has_l4_set(pkt, binary_flag); + odp_packet_has_eth_set(pkt, binary_flag); + odp_packet_has_eth_bcast_set(pkt, binary_flag); + odp_packet_has_eth_mcast_set(pkt, !binary_flag); + odp_packet_has_jumbo_set(pkt, binary_flag); + odp_packet_has_vlan_set(pkt, binary_flag); + odp_packet_has_vlan_qinq_set(pkt, binary_flag); + odp_packet_has_arp_set(pkt, binary_flag); + odp_packet_has_ipv4_set(pkt, binary_flag); + odp_packet_has_ipv6_set(pkt, !binary_flag); + odp_packet_has_ip_bcast_set(pkt, binary_flag); + odp_packet_has_ip_mcast_set(pkt, binary_flag); + odp_packet_has_ipfrag_set(pkt, binary_flag); + odp_packet_has_ipopt_set(pkt, binary_flag); + odp_packet_has_ipsec_set(pkt, binary_flag); + odp_packet_has_udp_set(pkt, binary_flag); + odp_packet_has_tcp_set(pkt, !binary_flag); + odp_packet_has_sctp_set(pkt, binary_flag); + odp_packet_has_icmp_set(pkt, binary_flag); + + odp_packet_user_ptr_set(pkt, &pkt); + odp_packet_user_flag_set(pkt, binary_flag); + (void)odp_packet_l2_offset_set(pkt, offset); + (void)odp_packet_l3_offset_set(pkt, offset); + (void)odp_packet_l4_offset_set(pkt, offset); + odp_packet_flow_hash_set(pkt, 0x12345678); + odp_packet_ts_set(pkt, odp_time_local_from_ns(ODP_TIME_SEC_IN_NS)); + odp_packet_color_set(pkt, ODP_PACKET_YELLOW); + odp_packet_drop_eligible_set(pkt, binary_flag); + odp_packet_shaper_len_adjust_set(pkt, -42); + (void)odp_packet_payload_offset_set(pkt, offset); +} + +void test_packet_get_md(odp_packet_t pkt, test_packet_md_t *md) +{ + uint8_t *uarea = odp_packet_user_area(pkt); + uint32_t uarea_size = odp_packet_user_area_size(pkt); + + memset(md, 0, sizeof(*md)); + + if (uarea) + md->user_area_chksum = odp_chksum_ones_comp16(uarea, uarea_size); + + md->has_error = !!odp_packet_has_error(pkt); + md->has_l2_error = !!odp_packet_has_l2_error(pkt); + md->has_l3_error = !!odp_packet_has_l3_error(pkt); + md->has_l4_error = !!odp_packet_has_l4_error(pkt); + md->has_l2 = !!odp_packet_has_l2(pkt); + md->has_l3 = !!odp_packet_has_l3(pkt); + md->has_l4 = !!odp_packet_has_l4(pkt); + md->has_eth = !!odp_packet_has_eth(pkt); + md->has_eth_bcast = !!odp_packet_has_eth_bcast(pkt); + md->has_eth_mcast = !!odp_packet_has_eth_mcast(pkt); + md->has_jumbo = !!odp_packet_has_jumbo(pkt); + md->has_vlan = !!odp_packet_has_vlan(pkt); + md->has_vlan_qinq = !!odp_packet_has_vlan_qinq(pkt); + md->has_arp = !!odp_packet_has_arp(pkt); + md->has_ipv4 = !!odp_packet_has_ipv4(pkt); + md->has_ipv6 = !!odp_packet_has_ipv6(pkt); + md->has_ip_bcast = !!odp_packet_has_ip_bcast(pkt); + md->has_ip_mcast = !!odp_packet_has_ip_mcast(pkt); + md->has_ipfrag = !!odp_packet_has_ipfrag(pkt); + md->has_ipopt = !!odp_packet_has_ipopt(pkt); + md->has_ipsec = !!odp_packet_has_ipsec(pkt); + md->has_udp = !!odp_packet_has_udp(pkt); + md->has_tcp = !!odp_packet_has_tcp(pkt); + md->has_sctp = !!odp_packet_has_sctp(pkt); + md->has_icmp = !!odp_packet_has_icmp(pkt); + md->has_flow_hash = !!odp_packet_has_flow_hash(pkt); + md->has_ts = !!odp_packet_has_ts(pkt); + + md->len = odp_packet_len(pkt); + md->packet_pool = odp_packet_pool(pkt); + md->packet_input = odp_packet_input(pkt); + md->user_ptr = odp_packet_user_ptr(pkt); + md->user_flag = odp_packet_user_flag(pkt); + md->l2_offset = odp_packet_l2_offset(pkt); + md->l3_offset = odp_packet_l3_offset(pkt); + md->l4_offset = odp_packet_l4_offset(pkt); + md->l2_type = odp_packet_l2_type(pkt); + md->l3_type = odp_packet_l3_type(pkt); + md->l4_type = odp_packet_l4_type(pkt); + md->l3_chksum_status = odp_packet_l3_chksum_status(pkt); + md->l4_chksum_status = odp_packet_l4_chksum_status(pkt); + md->flow_hash = md->has_flow_hash ? odp_packet_flow_hash(pkt) : 0; + md->ts = md->has_ts ? odp_packet_ts(pkt) : odp_time_global_from_ns(0); + md->color = odp_packet_color(pkt); + md->drop_eligible = !!odp_packet_drop_eligible(pkt); + md->shaper_len_adjust = odp_packet_shaper_len_adjust(pkt); + md->cls_mark = odp_packet_cls_mark(pkt); + md->has_lso_request = !!odp_packet_has_lso_request(pkt); + md->payload_offset = odp_packet_payload_offset(pkt); + md->aging_tmo = odp_packet_aging_tmo(pkt); + md->has_tx_compl_request = !!odp_packet_has_tx_compl_request(pkt); + md->proto_stats = odp_packet_proto_stats(pkt); +} + +int test_packet_is_md_equal(const test_packet_md_t *md_1, + const test_packet_md_t *md_2) +{ + /* + * With certain assumptions this should typically work. If it does + * not, we would get false negatives which we should spot as test + * failures. + */ + + return (!memcmp(md_1, md_2, sizeof(*md_1))); +} diff --git a/test/common/packet_common.h b/test/common/packet_common.h new file mode 100644 index 000000000..c7cd5e27f --- /dev/null +++ b/test/common/packet_common.h @@ -0,0 +1,67 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +typedef struct test_packet_md_t { + int has_error; + int has_l2_error; + int has_l3_error; + int has_l4_error; + int has_l2; + int has_l3; + int has_l4; + int has_eth; + int has_eth_bcast; + int has_eth_mcast; + int has_jumbo; + int has_vlan; + int has_vlan_qinq; + int has_arp; + int has_ipv4; + int has_ipv6; + int has_ip_bcast; + int has_ip_mcast; + int has_ipfrag; + int has_ipopt; + int has_ipsec; + int has_udp; + int has_tcp; + int has_sctp; + int has_icmp; + int has_flow_hash; + int has_ts; + uint32_t len; + odp_pool_t packet_pool; + odp_pktio_t packet_input; + void *user_ptr; + uint16_t user_area_chksum; + int user_flag; + uint32_t l2_offset; + uint32_t l3_offset; + uint32_t l4_offset; + odp_proto_l2_type_t l2_type; + odp_proto_l3_type_t l3_type; + odp_proto_l4_type_t l4_type; + odp_packet_chksum_status_t l3_chksum_status; + odp_packet_chksum_status_t l4_chksum_status; + uint32_t flow_hash; + odp_time_t ts; + odp_packet_color_t color; + odp_bool_t drop_eligible; + int8_t shaper_len_adjust; + uint64_t cls_mark; + int has_lso_request; + uint32_t payload_offset; + uint64_t aging_tmo; + int has_tx_compl_request; + odp_proto_stats_t proto_stats; +} test_packet_md_t; + +void test_packet_set_md(odp_packet_t pkt); +void test_packet_get_md(odp_packet_t pkt, test_packet_md_t *md); +int test_packet_is_md_equal(const test_packet_md_t *md_1, + const test_packet_md_t *md_2); -- cgit v1.2.3 From db873a539267da82d9ede8ba358a4f219fa43cbe Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 27 Jan 2023 14:02:31 +0200 Subject: validation: crypto: check that packet metadata does not change Add a check that packet metadata not related to crypto is not altered by crypto operations. Signed-off-by: Janne Peltonen Reviewed-by: Matias Elo --- test/validation/api/crypto/Makefile.am | 1 + test/validation/api/crypto/odp_crypto_test_inp.c | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/test/validation/api/crypto/Makefile.am b/test/validation/api/crypto/Makefile.am index cc4d49d60..e438ec8ca 100644 --- a/test/validation/api/crypto/Makefile.am +++ b/test/validation/api/crypto/Makefile.am @@ -5,3 +5,4 @@ crypto_main_SOURCES = \ odp_crypto_test_inp.c \ test_vectors.h \ test_vectors_len.h +PRELDADD += $(LIBPACKET_COMMON) diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index 10f1b5ee2..20a346c55 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "test_vectors.h" #define PKT_POOL_NUM 64 @@ -497,6 +498,7 @@ static void alg_test_execute(const alg_test_param_t *param) odp_packet_data_range_t auth_range; uint8_t *cipher_iv = param->override_iv ? ref->cipher_iv : NULL; uint8_t *auth_iv = param->override_iv ? ref->auth_iv : NULL; + test_packet_md_t md_in, md_out; cipher_range.offset = param->header_len; cipher_range.length = reflength; @@ -564,12 +566,25 @@ static void alg_test_execute(const alg_test_param_t *param) } } + test_packet_set_md(pkt); + test_packet_get_md(pkt, &md_in); + if (crypto_op(pkt, &ok, param->session, cipher_iv, auth_iv, &cipher_range, &auth_range, ref->aad, digest_offset)) break; + /* + * API is not explicit about whether a failed crypto op + * sets the has_error packet flag or leaves it unchanged. + * Let's allow both behaviours. + */ + test_packet_get_md(pkt, &md_out); + if (iteration == WRONG_DIGEST_TEST) + md_out.has_error = 0; + CU_ASSERT(test_packet_is_md_equal(&md_in, &md_out)); + if (iteration == WRONG_DIGEST_TEST) { CU_ASSERT(!ok); odp_packet_free(pkt); -- cgit v1.2.3 From 9079409fcd9754ce0086c5967fb5a7c32183c583 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 16 Jan 2023 17:02:12 +0200 Subject: linux-gen: ring: add 64-bit variant of mpmc ring Add new 64-bit variant of implementation internal multi-producer multi- consumer ring. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-generic/Makefile.am | 2 + .../include/odp_queue_basic_internal.h | 5 +- .../linux-generic/include/odp_ring_mpmc_internal.h | 115 ++++++++++++++------- .../include/odp_ring_mpmc_u32_internal.h | 25 +++++ .../include/odp_ring_mpmc_u64_internal.h | 25 +++++ platform/linux-generic/odp_ipsec_sad.c | 38 +++---- platform/linux-generic/odp_queue_basic.c | 22 ++-- 7 files changed, 165 insertions(+), 67 deletions(-) create mode 100644 platform/linux-generic/include/odp_ring_mpmc_u32_internal.h create mode 100644 platform/linux-generic/include/odp_ring_mpmc_u64_internal.h diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 6e6cfa5f6..d49a2138b 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -169,6 +169,8 @@ noinst_HEADERS = \ include/odp_ring_common.h \ include/odp_ring_internal.h \ include/odp_ring_mpmc_internal.h \ + include/odp_ring_mpmc_u32_internal.h \ + include/odp_ring_mpmc_u64_internal.h \ include/odp_ring_ptr_internal.h \ include/odp_ring_spsc_internal.h \ include/odp_ring_st_internal.h \ diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index 830f50a9d..3cdcf8600 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -23,7 +24,7 @@ extern "C" { #include #include #include -#include +#include #include #include #include @@ -47,7 +48,7 @@ typedef struct ODP_ALIGNED_CACHE queue_entry_s { odp_queue_type_t type; /* MPMC ring (2 cache lines). */ - ring_mpmc_t ring_mpmc; + ring_mpmc_u32_t ring_mpmc; odp_ticketlock_t lock; union { diff --git a/platform/linux-generic/include/odp_ring_mpmc_internal.h b/platform/linux-generic/include/odp_ring_mpmc_internal.h index 6ed4dd4d1..37b7780b5 100644 --- a/platform/linux-generic/include/odp_ring_mpmc_internal.h +++ b/platform/linux-generic/include/odp_ring_mpmc_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -19,6 +20,8 @@ extern "C" { #include #include +#include + /* Ring of uint32_t data * * Ring stores head and tail counters. Ring indexes are formed from these @@ -34,14 +37,22 @@ extern "C" { * r_tail r_head w_tail w_head * */ -typedef struct { + +struct ring_mpmc_common { odp_atomic_u32_t r_head ODP_ALIGNED_CACHE; odp_atomic_u32_t r_tail; odp_atomic_u32_t w_head ODP_ALIGNED_CACHE; odp_atomic_u32_t w_tail; +}; + +typedef struct ODP_ALIGNED_CACHE { + struct ring_mpmc_common r; +} ring_mpmc_u32_t; -} ring_mpmc_t; +typedef struct ODP_ALIGNED_CACHE { + struct ring_mpmc_common r; +} ring_mpmc_u64_t; static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, uint32_t new_val) @@ -52,21 +63,57 @@ static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, __ATOMIC_RELAXED); } +#endif /* End of include guards */ + +#undef _ring_mpmc_gen_t +#undef _ring_mpmc_data_t +#undef _RING_MPMC_INIT +#undef _RING_MPMC_DEQ_MULTI +#undef _RING_MPMC_ENQ_MULTI +#undef _RING_MPMC_IS_EMPTY +#undef _RING_MPMC_LEN + +/* This header should NOT be included directly. There are no include guards for + * the following types and function definitions! */ +#ifndef _ODP_RING_TYPE +#error Include type specific (u32/u64) ring header instead of this common file. +#endif + +#if _ODP_RING_TYPE == _ODP_RING_TYPE_U32 + #define _ring_mpmc_gen_t ring_mpmc_u32_t + #define _ring_mpmc_data_t uint32_t + + #define _RING_MPMC_INIT ring_mpmc_u32_init + #define _RING_MPMC_DEQ_MULTI ring_mpmc_u32_deq_multi + #define _RING_MPMC_ENQ_MULTI ring_mpmc_u32_enq_multi + #define _RING_MPMC_IS_EMPTY ring_mpmc_u32_is_empty + #define _RING_MPMC_LEN ring_mpmc_u32_len +#elif _ODP_RING_TYPE == _ODP_RING_TYPE_U64 + #define _ring_mpmc_gen_t ring_mpmc_u64_t + #define _ring_mpmc_data_t uint64_t + + #define _RING_MPMC_INIT ring_mpmc_u64_init + #define _RING_MPMC_DEQ_MULTI ring_mpmc_u64_deq_multi + #define _RING_MPMC_ENQ_MULTI ring_mpmc_u64_enq_multi + #define _RING_MPMC_IS_EMPTY ring_mpmc_u64_is_empty + #define _RING_MPMC_LEN ring_mpmc_u64_len +#endif + /* Initialize ring */ -static inline void ring_mpmc_init(ring_mpmc_t *ring) +static inline void _RING_MPMC_INIT(_ring_mpmc_gen_t *ring) { - odp_atomic_init_u32(&ring->w_head, 0); - odp_atomic_init_u32(&ring->w_tail, 0); - odp_atomic_init_u32(&ring->r_head, 0); - odp_atomic_init_u32(&ring->r_tail, 0); + odp_atomic_init_u32(&ring->r.w_head, 0); + odp_atomic_init_u32(&ring->r.w_tail, 0); + odp_atomic_init_u32(&ring->r.r_head, 0); + odp_atomic_init_u32(&ring->r.r_tail, 0); } /* Dequeue data from the ring head. Num is smaller than ring size. */ -static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, - uint32_t *ring_data, - uint32_t ring_mask, - uint32_t data[], - uint32_t num) +static inline uint32_t _RING_MPMC_DEQ_MULTI(_ring_mpmc_gen_t *ring, + _ring_mpmc_data_t *ring_data, + uint32_t ring_mask, + _ring_mpmc_data_t data[], + uint32_t num) { uint32_t old_head, new_head, w_tail, num_data, i; @@ -75,9 +122,9 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, * When CAS operation succeeds, this thread owns data between old * and new r_head. */ do { - old_head = odp_atomic_load_acq_u32(&ring->r_head); + old_head = odp_atomic_load_acq_u32(&ring->r.r_head); odp_prefetch(&ring_data[(old_head + 1) & ring_mask]); - w_tail = odp_atomic_load_acq_u32(&ring->w_tail); + w_tail = odp_atomic_load_acq_u32(&ring->r.w_tail); num_data = w_tail - old_head; /* Ring is empty */ @@ -90,7 +137,7 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, new_head = old_head + num; - } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r_head, &old_head, + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.r_head, &old_head, new_head) == 0)); /* Read data. This will not move above load acquire of r_head. */ @@ -98,21 +145,21 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring, data[i] = ring_data[(old_head + 1 + i) & ring_mask]; /* Wait until other readers have updated the tail */ - while (odp_unlikely(odp_atomic_load_u32(&ring->r_tail) != old_head)) + while (odp_unlikely(odp_atomic_load_u32(&ring->r.r_tail) != old_head)) odp_cpu_pause(); /* Release the new reader tail, writers acquire it. */ - odp_atomic_store_rel_u32(&ring->r_tail, new_head); + odp_atomic_store_rel_u32(&ring->r.r_tail, new_head); return num; } /* Enqueue multiple data into the ring tail. Num is smaller than ring size. */ -static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, - uint32_t *ring_data, - uint32_t ring_mask, - const uint32_t data[], - uint32_t num) +static inline uint32_t _RING_MPMC_ENQ_MULTI(_ring_mpmc_gen_t *ring, + _ring_mpmc_data_t *ring_data, + uint32_t ring_mask, + const _ring_mpmc_data_t data[], + uint32_t num) { uint32_t old_head, new_head, r_tail, num_free, i; uint32_t size = ring_mask + 1; @@ -122,8 +169,8 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, * When CAS operation succeeds, this thread owns data between old * and new w_head. */ do { - r_tail = odp_atomic_load_acq_u32(&ring->r_tail); - old_head = odp_atomic_load_acq_u32(&ring->w_head); + r_tail = odp_atomic_load_acq_u32(&ring->r.r_tail); + old_head = odp_atomic_load_acq_u32(&ring->r.w_head); num_free = size - (old_head - r_tail); @@ -137,7 +184,7 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, new_head = old_head + num; - } while (odp_unlikely(ring_mpmc_cas_u32(&ring->w_head, &old_head, + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.w_head, &old_head, new_head) == 0)); /* Write data. This will not move above load acquire of w_head. */ @@ -145,29 +192,29 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring, ring_data[(old_head + 1 + i) & ring_mask] = data[i]; /* Wait until other writers have updated the tail */ - while (odp_unlikely(odp_atomic_load_u32(&ring->w_tail) != old_head)) + while (odp_unlikely(odp_atomic_load_u32(&ring->r.w_tail) != old_head)) odp_cpu_pause(); /* Release the new writer tail, readers acquire it. */ - odp_atomic_store_rel_u32(&ring->w_tail, new_head); + odp_atomic_store_rel_u32(&ring->r.w_tail, new_head); return num; } /* Check if ring is empty */ -static inline int ring_mpmc_is_empty(ring_mpmc_t *ring) +static inline int _RING_MPMC_IS_EMPTY(_ring_mpmc_gen_t *ring) { - uint32_t head = odp_atomic_load_u32(&ring->r_head); - uint32_t tail = odp_atomic_load_u32(&ring->w_tail); + uint32_t head = odp_atomic_load_u32(&ring->r.r_head); + uint32_t tail = odp_atomic_load_u32(&ring->r.w_tail); return head == tail; } /* Return current ring length */ -static inline uint32_t ring_mpmc_length(ring_mpmc_t *ring) +static inline uint32_t _RING_MPMC_LEN(_ring_mpmc_gen_t *ring) { - uint32_t head = odp_atomic_load_u32(&ring->r_head); - uint32_t tail = odp_atomic_load_u32(&ring->w_tail); + uint32_t head = odp_atomic_load_u32(&ring->r.r_head); + uint32_t tail = odp_atomic_load_u32(&ring->r.w_tail); return tail - head; } @@ -175,5 +222,3 @@ static inline uint32_t ring_mpmc_length(ring_mpmc_t *ring) #ifdef __cplusplus } #endif - -#endif diff --git a/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h b/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h new file mode 100644 index 000000000..4699b5b47 --- /dev/null +++ b/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_RING_MPMC_U32_INTERNAL_H_ +#define ODP_RING_MPMC_U32_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#undef _ODP_RING_TYPE +#define _ODP_RING_TYPE _ODP_RING_TYPE_U32 + +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h b/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h new file mode 100644 index 000000000..e7bf31a94 --- /dev/null +++ b/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_RING_MPMC_U64_INTERNAL_H_ +#define ODP_RING_MPMC_U64_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#undef _ODP_RING_TYPE +#define _ODP_RING_TYPE _ODP_RING_TYPE_U64 + +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 9bb89ad8b..4cf265694 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -1,5 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited - * Copyright (c) 2018-2022, Nokia + * Copyright (c) 2018-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include @@ -103,7 +103,7 @@ typedef struct ODP_ALIGNED_CACHE ipsec_thread_local_s { typedef struct ipsec_sa_table_t { ipsec_sa_t ipsec_sa[CONFIG_IPSEC_MAX_NUM_SA]; struct ODP_ALIGNED_CACHE { - ring_mpmc_t ipv4_id_ring; + ring_mpmc_u32_t ipv4_id_ring; uint32_t ipv4_id_data[IPV4_ID_RING_SIZE] ODP_ALIGNED_CACHE; } hot; struct { @@ -193,7 +193,7 @@ int _odp_ipsec_sad_init_global(void) ipsec_sa_tbl->shm = shm; ipsec_sa_tbl->max_num_sa = max_num_sa; - ring_mpmc_init(&ipsec_sa_tbl->hot.ipv4_id_ring); + ring_mpmc_u32_init(&ipsec_sa_tbl->hot.ipv4_id_ring); for (i = 0; i < thread_count_max; i++) { /* * Make the current ID block fully used, forcing allocation @@ -211,11 +211,11 @@ int _odp_ipsec_sad_init_global(void) for (i = 0; i < IPV4_ID_RING_SIZE - 1; i++) { uint32_t data = i * IPV4_ID_BLOCK_SIZE; - ring_mpmc_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, - ipsec_sa_tbl->hot.ipv4_id_data, - IPV4_ID_RING_MASK, - &data, - 1); + ring_mpmc_u32_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, + ipsec_sa_tbl->hot.ipv4_id_data, + IPV4_ID_RING_MASK, + &data, + 1); } for (i = 0; i < ipsec_sa_tbl->max_num_sa; i++) { @@ -1119,17 +1119,17 @@ uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa) tl->first_ipv4_id + IPV4_ID_BLOCK_SIZE)) { /* Return used ID block to the ring */ data = tl->first_ipv4_id; - ring_mpmc_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, - ipsec_sa_tbl->hot.ipv4_id_data, - IPV4_ID_RING_MASK, - &data, - 1); + ring_mpmc_u32_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, + ipsec_sa_tbl->hot.ipv4_id_data, + IPV4_ID_RING_MASK, + &data, + 1); /* Get new ID block */ - ring_mpmc_deq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, - ipsec_sa_tbl->hot.ipv4_id_data, - IPV4_ID_RING_MASK, - &data, - 1); + ring_mpmc_u32_deq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring, + ipsec_sa_tbl->hot.ipv4_id_data, + IPV4_ID_RING_MASK, + &data, + 1); tl->first_ipv4_id = data; tl->next_ipv4_id = data; } diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index eebfb92ab..83694f84f 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -408,7 +408,7 @@ static int queue_destroy(odp_queue_t handle) else if (queue->type == ODP_QUEUE_TYPE_SCHED) empty = ring_st_is_empty(&queue->ring_st); else - empty = ring_mpmc_is_empty(&queue->ring_mpmc); + empty = ring_mpmc_u32_is_empty(&queue->ring_mpmc); if (!empty) { UNLOCK(queue); @@ -497,7 +497,7 @@ static inline int _plain_queue_enq_multi(odp_queue_t handle, { queue_entry_t *queue; int ret, num_enq; - ring_mpmc_t *ring_mpmc; + ring_mpmc_u32_t *ring_mpmc; uint32_t event_idx[num]; queue = qentry_from_handle(handle); @@ -508,8 +508,8 @@ static inline int _plain_queue_enq_multi(odp_queue_t handle, event_index_from_hdr(event_idx, event_hdr, num); - num_enq = ring_mpmc_enq_multi(ring_mpmc, queue->ring_data, - queue->ring_mask, event_idx, num); + num_enq = ring_mpmc_u32_enq_multi(ring_mpmc, queue->ring_data, + queue->ring_mask, event_idx, num); return num_enq; } @@ -519,14 +519,14 @@ static inline int _plain_queue_deq_multi(odp_queue_t handle, { int num_deq; queue_entry_t *queue; - ring_mpmc_t *ring_mpmc; + ring_mpmc_u32_t *ring_mpmc; uint32_t event_idx[num]; queue = qentry_from_handle(handle); ring_mpmc = &queue->ring_mpmc; - num_deq = ring_mpmc_deq_multi(ring_mpmc, queue->ring_data, - queue->ring_mask, event_idx, num); + num_deq = ring_mpmc_u32_deq_multi(ring_mpmc, queue->ring_data, + queue->ring_mask, event_idx, num); if (num_deq == 0) return 0; @@ -754,7 +754,7 @@ static void queue_print(odp_queue_t handle) } else { _ODP_PRINT(" implementation ring_mpmc\n"); _ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n", - ring_mpmc_length(&queue->ring_mpmc), queue->ring_mask + 1); + ring_mpmc_u32_len(&queue->ring_mpmc), queue->ring_mask + 1); } _ODP_PRINT("\n"); @@ -820,7 +820,7 @@ static void queue_print_all(void) if (_odp_sched_id == _ODP_SCHED_ID_BASIC) spr = _odp_sched_basic_get_spread(index); } else { - len = ring_mpmc_length(&queue->ring_mpmc); + len = ring_mpmc_u32_len(&queue->ring_mpmc); max_len = queue->ring_mask + 1; } @@ -1073,7 +1073,7 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->ring_data = &_odp_queue_glb->ring_data[offset]; queue->ring_mask = queue_size - 1; - ring_mpmc_init(&queue->ring_mpmc); + ring_mpmc_u32_init(&queue->ring_mpmc); } else { queue->enqueue = sched_queue_enq; -- cgit v1.2.3 From bba00b595276b556564fa05ce1dff51f29417103 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 19 Jan 2023 09:30:25 +0200 Subject: linux-gen: ring: add batch enqueue and dequeue operations to mpmc ring Add support for batch enqueue/dequeue operations to MPMC ring (read/write 0 or N objects). Unnecessary comments about 'num' parameter being smaller than ring size have been removed. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../linux-generic/include/odp_ring_mpmc_internal.h | 95 +++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/platform/linux-generic/include/odp_ring_mpmc_internal.h b/platform/linux-generic/include/odp_ring_mpmc_internal.h index 37b7780b5..e35179267 100644 --- a/platform/linux-generic/include/odp_ring_mpmc_internal.h +++ b/platform/linux-generic/include/odp_ring_mpmc_internal.h @@ -70,6 +70,8 @@ static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, #undef _RING_MPMC_INIT #undef _RING_MPMC_DEQ_MULTI #undef _RING_MPMC_ENQ_MULTI +#undef _RING_MPMC_DEQ_BATCH +#undef _RING_MPMC_ENQ_BATCH #undef _RING_MPMC_IS_EMPTY #undef _RING_MPMC_LEN @@ -86,6 +88,8 @@ static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, #define _RING_MPMC_INIT ring_mpmc_u32_init #define _RING_MPMC_DEQ_MULTI ring_mpmc_u32_deq_multi #define _RING_MPMC_ENQ_MULTI ring_mpmc_u32_enq_multi + #define _RING_MPMC_DEQ_BATCH ring_mpmc_u32_deq_batch + #define _RING_MPMC_ENQ_BATCH ring_mpmc_u32_enq_batch #define _RING_MPMC_IS_EMPTY ring_mpmc_u32_is_empty #define _RING_MPMC_LEN ring_mpmc_u32_len #elif _ODP_RING_TYPE == _ODP_RING_TYPE_U64 @@ -95,6 +99,8 @@ static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom, #define _RING_MPMC_INIT ring_mpmc_u64_init #define _RING_MPMC_DEQ_MULTI ring_mpmc_u64_deq_multi #define _RING_MPMC_ENQ_MULTI ring_mpmc_u64_enq_multi + #define _RING_MPMC_DEQ_BATCH ring_mpmc_u64_deq_batch + #define _RING_MPMC_ENQ_BATCH ring_mpmc_u64_enq_batch #define _RING_MPMC_IS_EMPTY ring_mpmc_u64_is_empty #define _RING_MPMC_LEN ring_mpmc_u64_len #endif @@ -108,7 +114,7 @@ static inline void _RING_MPMC_INIT(_ring_mpmc_gen_t *ring) odp_atomic_init_u32(&ring->r.r_tail, 0); } -/* Dequeue data from the ring head. Num is smaller than ring size. */ +/* Dequeue data from the ring head */ static inline uint32_t _RING_MPMC_DEQ_MULTI(_ring_mpmc_gen_t *ring, _ring_mpmc_data_t *ring_data, uint32_t ring_mask, @@ -154,7 +160,49 @@ static inline uint32_t _RING_MPMC_DEQ_MULTI(_ring_mpmc_gen_t *ring, return num; } -/* Enqueue multiple data into the ring tail. Num is smaller than ring size. */ +/* Dequeue num or 0 data from the ring head */ +static inline uint32_t _RING_MPMC_DEQ_BATCH(_ring_mpmc_gen_t *ring, + _ring_mpmc_data_t *ring_data, + uint32_t ring_mask, + _ring_mpmc_data_t data[], + uint32_t num) +{ + uint32_t old_head, new_head, w_tail, num_data, i; + + /* Load acquires ensure that w_tail load happens after r_head load, + * and thus r_head value is always behind or equal to w_tail value. + * When CAS operation succeeds, this thread owns data between old + * and new r_head. */ + do { + old_head = odp_atomic_load_acq_u32(&ring->r.r_head); + odp_prefetch(&ring_data[(old_head + 1) & ring_mask]); + w_tail = odp_atomic_load_acq_u32(&ring->r.w_tail); + num_data = w_tail - old_head; + + /* Not enough data available */ + if (num_data < num) + return 0; + + new_head = old_head + num; + + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.r_head, &old_head, + new_head) == 0)); + + /* Read data. This will not move above load acquire of r_head. */ + for (i = 0; i < num; i++) + data[i] = ring_data[(old_head + 1 + i) & ring_mask]; + + /* Wait until other readers have updated the tail */ + while (odp_unlikely(odp_atomic_load_u32(&ring->r.r_tail) != old_head)) + odp_cpu_pause(); + + /* Release the new reader tail, writers acquire it. */ + odp_atomic_store_rel_u32(&ring->r.r_tail, new_head); + + return num; +} + +/* Enqueue multiple data into the ring tail */ static inline uint32_t _RING_MPMC_ENQ_MULTI(_ring_mpmc_gen_t *ring, _ring_mpmc_data_t *ring_data, uint32_t ring_mask, @@ -201,6 +249,49 @@ static inline uint32_t _RING_MPMC_ENQ_MULTI(_ring_mpmc_gen_t *ring, return num; } +/* Enqueue num or 0 data into the ring tail */ +static inline uint32_t _RING_MPMC_ENQ_BATCH(_ring_mpmc_gen_t *ring, + _ring_mpmc_data_t *ring_data, + uint32_t ring_mask, + const _ring_mpmc_data_t data[], + uint32_t num) +{ + uint32_t old_head, new_head, r_tail, num_free, i; + uint32_t size = ring_mask + 1; + + /* Load acquires ensure that w_head load happens after r_tail load, + * and thus r_tail value is always behind or equal to w_head value. + * When CAS operation succeeds, this thread owns data between old + * and new w_head. */ + do { + r_tail = odp_atomic_load_acq_u32(&ring->r.r_tail); + old_head = odp_atomic_load_acq_u32(&ring->r.w_head); + + num_free = size - (old_head - r_tail); + + /* Not enough free space available */ + if (num_free < num) + return 0; + + new_head = old_head + num; + + } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.w_head, &old_head, + new_head) == 0)); + + /* Write data. This will not move above load acquire of w_head. */ + for (i = 0; i < num; i++) + ring_data[(old_head + 1 + i) & ring_mask] = data[i]; + + /* Wait until other writers have updated the tail */ + while (odp_unlikely(odp_atomic_load_u32(&ring->r.w_tail) != old_head)) + odp_cpu_pause(); + + /* Release the new writer tail, readers acquire it. */ + odp_atomic_store_rel_u32(&ring->r.w_tail, new_head); + + return num; +} + /* Check if ring is empty */ static inline int _RING_MPMC_IS_EMPTY(_ring_mpmc_gen_t *ring) { -- cgit v1.2.3 From 24697a217b8c832609a39274b3a09448ad1562d0 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 19 Jan 2023 09:37:27 +0200 Subject: linux-gen: stash: add framework for using different ring types Add ring function pointer framework to the stash implementation, which enables using different ring types as stash implementation. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-generic/odp_stash.c | 354 +++++++++++++++++++++++-------------- 1 file changed, 220 insertions(+), 134 deletions(-) diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index fecbabac3..a954cf866 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include #include #include #include @@ -35,14 +36,53 @@ enum { STASH_ACTIVE }; +typedef struct stash_t stash_t; + +typedef void (*ring_u32_init_fn_t)(stash_t *stash); +typedef int32_t (*ring_u32_enq_multi_fn_t)(stash_t *stash, const uint32_t val[], int32_t num); +typedef int32_t (*ring_u32_enq_batch_fn_t)(stash_t *stash, const uint32_t val[], int32_t num); +typedef int32_t (*ring_u32_deq_multi_fn_t)(stash_t *stash, uint32_t val[], int32_t num); +typedef int32_t (*ring_u32_deq_batch_fn_t)(stash_t *stash, uint32_t val[], int32_t num); +typedef int32_t (*ring_u32_len_fn_t)(stash_t *stash); + +typedef void (*ring_u64_init_fn_t)(stash_t *stash); +typedef int32_t (*ring_u64_enq_multi_fn_t)(stash_t *stash, const uint64_t val[], int32_t num); +typedef int32_t (*ring_u64_enq_batch_fn_t)(stash_t *stash, const uint64_t val[], int32_t num); +typedef int32_t (*ring_u64_deq_multi_fn_t)(stash_t *stash, uint64_t val[], int32_t num); +typedef int32_t (*ring_u64_deq_batch_fn_t)(stash_t *stash, uint64_t val[], int32_t num); +typedef int32_t (*ring_u64_len_fn_t)(stash_t *stash); + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpedantic" -typedef struct stash_t { - char name[ODP_STASH_NAME_LEN]; - int index; +typedef struct ODP_ALIGNED_CACHE stash_t { + /* Ring functions */ + union { + struct { + ring_u32_enq_multi_fn_t enq_multi; + ring_u32_enq_batch_fn_t enq_batch; + ring_u32_deq_multi_fn_t deq_multi; + ring_u32_deq_batch_fn_t deq_batch; + ring_u32_init_fn_t init; + ring_u32_len_fn_t len; + } u32; + + struct { + ring_u64_enq_multi_fn_t enq_multi; + ring_u64_enq_batch_fn_t enq_batch; + ring_u64_deq_multi_fn_t deq_multi; + ring_u64_deq_batch_fn_t deq_batch; + ring_u64_init_fn_t init; + ring_u64_len_fn_t len; + } u64; + } ring_fn; + uint32_t ring_mask; + uint32_t ring_size; uint32_t obj_size; + char name[ODP_STASH_NAME_LEN]; + int index; + /* Ring header followed by variable sized data (object handles) */ union { struct ODP_ALIGNED_CACHE { @@ -243,10 +283,72 @@ static void free_index(int i) odp_ticketlock_unlock(&stash_global->lock); } +static inline void strict_ring_u32_init(stash_t *stash) +{ + ring_u32_init(&stash->ring_u32.hdr); + + for (uint32_t i = 0; i < stash->ring_size; i++) + stash->ring_u32.data[i] = 0; +} + +static inline void strict_ring_u64_init(stash_t *stash) +{ + ring_u64_init(&stash->ring_u64.hdr); + + for (uint32_t i = 0; i < stash->ring_size; i++) + stash->ring_u64.data[i] = 0; +} + +static inline int32_t strict_ring_u32_enq_multi(stash_t *stash, const uint32_t val[], int32_t num) +{ + /* Success always */ + ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, (uint32_t *)(uintptr_t)val, num); + + return num; +} + +static inline int32_t strict_ring_u64_enq_multi(stash_t *stash, const uint64_t val[], int32_t num) +{ + /* Success always */ + ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, (uint64_t *)(uintptr_t)val, num); + + return num; +} + +static inline int32_t strict_ring_u32_deq_multi(stash_t *stash, uint32_t val[], int32_t num) +{ + return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, val, num); +} + +static inline int32_t strict_ring_u64_deq_multi(stash_t *stash, uint64_t val[], int32_t num) +{ + return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, val, num); +} + +static inline int32_t strict_ring_u32_deq_batch(stash_t *stash, uint32_t val[], int32_t num) +{ + return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask, val, num); +} + +static inline int32_t strict_ring_u64_deq_batch(stash_t *stash, uint64_t val[], int32_t num) +{ + return ring_u64_deq_batch(&stash->ring_u64.hdr, stash->ring_mask, val, num); +} + +static inline int32_t strict_ring_u32_len(stash_t *stash) +{ + return ring_u32_len(&stash->ring_u32.hdr); +} + +static inline int32_t strict_ring_u64_len(stash_t *stash) +{ + return ring_u64_len(&stash->ring_u64.hdr); +} + odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) { stash_t *stash; - uint64_t i, ring_size; + uint64_t ring_size; int ring_u64, index; if (odp_global_ro.disable.stash) { @@ -291,16 +393,21 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) stash = stash_global->stash[index]; memset(stash, 0, sizeof(stash_t)); + /* Set ring function pointers */ if (ring_u64) { - ring_u64_init(&stash->ring_u64.hdr); - - for (i = 0; i < ring_size; i++) - stash->ring_u64.data[i] = 0; + stash->ring_fn.u64.init = strict_ring_u64_init; + stash->ring_fn.u64.enq_multi = strict_ring_u64_enq_multi; + stash->ring_fn.u64.enq_batch = strict_ring_u64_enq_multi; + stash->ring_fn.u64.deq_multi = strict_ring_u64_deq_multi; + stash->ring_fn.u64.deq_batch = strict_ring_u64_deq_batch; + stash->ring_fn.u64.len = strict_ring_u64_len; } else { - ring_u32_init(&stash->ring_u32.hdr); - - for (i = 0; i < ring_size; i++) - stash->ring_u32.data[i] = 0; + stash->ring_fn.u32.init = strict_ring_u32_init; + stash->ring_fn.u32.enq_multi = strict_ring_u32_enq_multi; + stash->ring_fn.u32.enq_batch = strict_ring_u32_enq_multi; + stash->ring_fn.u32.deq_multi = strict_ring_u32_deq_multi; + stash->ring_fn.u32.deq_batch = strict_ring_u32_deq_batch; + stash->ring_fn.u32.len = strict_ring_u32_len; } if (name) @@ -309,6 +416,12 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) stash->index = index; stash->obj_size = param->obj_size; stash->ring_mask = ring_size - 1; + stash->ring_size = ring_size; + + if (ring_u64) + stash->ring_fn.u64.init(stash); + else + stash->ring_fn.u32.init(stash); /* This makes stash visible to lookups */ odp_ticketlock_lock(&stash_global->lock); @@ -357,8 +470,10 @@ odp_stash_t odp_stash_lookup(const char *name) return ODP_STASH_INVALID; } -static inline int32_t stash_put(odp_stash_t st, const void *obj, int32_t num) +static inline int32_t stash_put(odp_stash_t st, const void *obj, int32_t num, odp_bool_t is_batch) { + int32_t (*ring_u32_enq)(stash_t *stash, const uint32_t val[], int32_t num); + int32_t (*ring_u64_enq)(stash_t *stash, const uint64_t val[], int32_t num); stash_t *stash = stash_entry(st); uint32_t obj_size; int32_t i; @@ -366,46 +481,40 @@ static inline int32_t stash_put(odp_stash_t st, const void *obj, int32_t num) if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - obj_size = stash->obj_size; - - if (obj_size == sizeof(uint64_t)) { - ring_u64_t *ring_u64 = &stash->ring_u64.hdr; - - ring_u64_enq_multi(ring_u64, stash->ring_mask, - (uint64_t *)(uintptr_t)obj, num); - return num; + if (is_batch) { + ring_u32_enq = stash->ring_fn.u32.enq_batch; + ring_u64_enq = stash->ring_fn.u64.enq_batch; + } else { + ring_u32_enq = stash->ring_fn.u32.enq_multi; + ring_u64_enq = stash->ring_fn.u64.enq_multi; } - if (obj_size == sizeof(uint32_t)) { - ring_u32_t *ring_u32 = &stash->ring_u32.hdr; + obj_size = stash->obj_size; - ring_u32_enq_multi(ring_u32, stash->ring_mask, - (uint32_t *)(uintptr_t)obj, num); - return num; - } + if (obj_size == sizeof(uint64_t)) + return ring_u64_enq(stash, (uint64_t *)(uintptr_t)obj, num); + + if (obj_size == sizeof(uint32_t)) + return ring_u32_enq(stash, (uint32_t *)(uintptr_t)obj, num); if (obj_size == sizeof(uint16_t)) { const uint16_t *u16_ptr = obj; - ring_u32_t *ring_u32 = &stash->ring_u32.hdr; uint32_t u32[num]; for (i = 0; i < num; i++) u32[i] = u16_ptr[i]; - ring_u32_enq_multi(ring_u32, stash->ring_mask, u32, num); - return num; + return ring_u32_enq(stash, u32, num); } if (obj_size == sizeof(uint8_t)) { const uint8_t *u8_ptr = obj; - ring_u32_t *ring_u32 = &stash->ring_u32.hdr; uint32_t u32[num]; for (i = 0; i < num; i++) u32[i] = u8_ptr[i]; - ring_u32_enq_multi(ring_u32, stash->ring_mask, u32, num); - return num; + return ring_u32_enq(stash, u32, num); } return -1; @@ -413,17 +522,15 @@ static inline int32_t stash_put(odp_stash_t st, const void *obj, int32_t num) int32_t odp_stash_put(odp_stash_t st, const void *obj, int32_t num) { - return stash_put(st, obj, num); + return stash_put(st, obj, num, false); } int32_t odp_stash_put_batch(odp_stash_t st, const void *obj, int32_t num) { - /* Returns always 'num', or -1 on failure. */ - return stash_put(st, obj, num); + return stash_put(st, obj, num, true); } -static inline int32_t stash_put_u32(odp_stash_t st, const uint32_t val[], - int32_t num) +int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t val[], int32_t num) { stash_t *stash = stash_entry(st); @@ -432,25 +539,22 @@ static inline int32_t stash_put_u32(odp_stash_t st, const uint32_t val[], _ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); - ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, - (uint32_t *)(uintptr_t)val, num); - return num; + return stash->ring_fn.u32.enq_multi(stash, val, num); } -int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t val[], int32_t num) +int32_t odp_stash_put_u32_batch(odp_stash_t st, const uint32_t val[], int32_t num) { - return stash_put_u32(st, val, num); -} + stash_t *stash = stash_entry(st); -int32_t odp_stash_put_u32_batch(odp_stash_t st, const uint32_t val[], - int32_t num) -{ - /* Returns always 'num', or -1 on failure. */ - return stash_put_u32(st, val, num); + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + _ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); + + return stash->ring_fn.u32.enq_batch(stash, val, num); } -static inline int32_t stash_put_u64(odp_stash_t st, const uint64_t val[], - int32_t num) +int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t val[], int32_t num) { stash_t *stash = stash_entry(st); @@ -459,25 +563,23 @@ static inline int32_t stash_put_u64(odp_stash_t st, const uint64_t val[], _ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); - ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, - (uint64_t *)(uintptr_t)val, num); - return num; -} - -int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t val[], int32_t num) -{ - return stash_put_u64(st, val, num); + return stash->ring_fn.u64.enq_multi(stash, (uint64_t *)(uintptr_t)val, num); } int32_t odp_stash_put_u64_batch(odp_stash_t st, const uint64_t val[], int32_t num) { - /* Returns always 'num', or -1 on failure. */ - return stash_put_u64(st, val, num); + stash_t *stash = stash_entry(st); + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + _ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); + + return stash->ring_fn.u64.enq_batch(stash, (uint64_t *)(uintptr_t)val, num); } -static inline int32_t stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], - int32_t num) +int32_t odp_stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], int32_t num) { stash_t *stash = stash_entry(st); @@ -487,31 +589,37 @@ static inline int32_t stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], _ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); if (sizeof(uintptr_t) == sizeof(uint32_t)) - ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, - (uint32_t *)(uintptr_t)ptr, num); - else if (sizeof(uintptr_t) == sizeof(uint64_t)) - ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, - (uint64_t *)(uintptr_t)ptr, num); - else - return -1; + return stash->ring_fn.u32.enq_multi(stash, (uint32_t *)(uintptr_t)ptr, num); - return num; -} + if (sizeof(uintptr_t) == sizeof(uint64_t)) + return stash->ring_fn.u64.enq_multi(stash, (uint64_t *)(uintptr_t)ptr, num); -int32_t odp_stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], int32_t num) -{ - return stash_put_ptr(st, ptr, num); + return -1; } int32_t odp_stash_put_ptr_batch(odp_stash_t st, const uintptr_t ptr[], int32_t num) { - /* Returns always 'num', or -1 on failure. */ - return stash_put_ptr(st, ptr, num); + stash_t *stash = stash_entry(st); + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + _ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); + + if (sizeof(uintptr_t) == sizeof(uint32_t)) + return stash->ring_fn.u32.enq_batch(stash, (uint32_t *)(uintptr_t)ptr, num); + + if (sizeof(uintptr_t) == sizeof(uint64_t)) + return stash->ring_fn.u64.enq_batch(stash, (uint64_t *)(uintptr_t)ptr, num); + + return -1; } -static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool_t batch) +static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool_t is_batch) { + int32_t (*ring_u32_deq)(stash_t *stash, uint32_t val[], int32_t num); + int32_t (*ring_u64_deq)(stash_t *stash, uint64_t val[], int32_t num); stash_t *stash = stash_entry(st); uint32_t obj_size; uint32_t i, num_deq; @@ -519,35 +627,27 @@ static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool if (odp_unlikely(st == ODP_STASH_INVALID)) return -1; - obj_size = stash->obj_size; - - if (obj_size == sizeof(uint64_t)) { - ring_u64_t *ring_u64 = &stash->ring_u64.hdr; - - if (batch) - return ring_u64_deq_batch(ring_u64, stash->ring_mask, obj, num); - else - return ring_u64_deq_multi(ring_u64, stash->ring_mask, obj, num); + if (is_batch) { + ring_u32_deq = stash->ring_fn.u32.deq_batch; + ring_u64_deq = stash->ring_fn.u64.deq_batch; + } else { + ring_u32_deq = stash->ring_fn.u32.deq_multi; + ring_u64_deq = stash->ring_fn.u64.deq_multi; } - if (obj_size == sizeof(uint32_t)) { - ring_u32_t *ring_u32 = &stash->ring_u32.hdr; + obj_size = stash->obj_size; - if (batch) - return ring_u32_deq_batch(ring_u32, stash->ring_mask, obj, num); - else - return ring_u32_deq_multi(ring_u32, stash->ring_mask, obj, num); - } + if (obj_size == sizeof(uint64_t)) + return ring_u64_deq(stash, obj, num); + + if (obj_size == sizeof(uint32_t)) + return ring_u32_deq(stash, obj, num); if (obj_size == sizeof(uint16_t)) { uint16_t *u16_ptr = obj; - ring_u32_t *ring_u32 = &stash->ring_u32.hdr; uint32_t u32[num]; - if (batch) - num_deq = ring_u32_deq_batch(ring_u32, stash->ring_mask, u32, num); - else - num_deq = ring_u32_deq_multi(ring_u32, stash->ring_mask, u32, num); + num_deq = ring_u32_deq(stash, u32, num); for (i = 0; i < num_deq; i++) u16_ptr[i] = u32[i]; @@ -557,13 +657,9 @@ static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool if (obj_size == sizeof(uint8_t)) { uint8_t *u8_ptr = obj; - ring_u32_t *ring_u32 = &stash->ring_u32.hdr; uint32_t u32[num]; - if (batch) - num_deq = ring_u32_deq_batch(ring_u32, stash->ring_mask, u32, num); - else - num_deq = ring_u32_deq_multi(ring_u32, stash->ring_mask, u32, num); + num_deq = ring_u32_deq(stash, u32, num); for (i = 0; i < num_deq; i++) u8_ptr[i] = u32[i]; @@ -576,12 +672,12 @@ static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) { - return stash_get(st, obj, num, 0); + return stash_get(st, obj, num, false); } int32_t odp_stash_get_batch(odp_stash_t st, void *obj, int32_t num) { - return stash_get(st, obj, num, 1); + return stash_get(st, obj, num, true); } int32_t odp_stash_get_u32(odp_stash_t st, uint32_t val[], int32_t num) @@ -593,8 +689,7 @@ int32_t odp_stash_get_u32(odp_stash_t st, uint32_t val[], int32_t num) _ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); - return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, val, - num); + return stash->ring_fn.u32.deq_multi(stash, val, num); } int32_t odp_stash_get_u32_batch(odp_stash_t st, uint32_t val[], int32_t num) @@ -606,7 +701,7 @@ int32_t odp_stash_get_u32_batch(odp_stash_t st, uint32_t val[], int32_t num) _ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); - return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask, val, num); + return stash->ring_fn.u32.deq_batch(stash, val, num); } int32_t odp_stash_get_u64(odp_stash_t st, uint64_t val[], int32_t num) @@ -618,8 +713,7 @@ int32_t odp_stash_get_u64(odp_stash_t st, uint64_t val[], int32_t num) _ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); - return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, val, - num); + return stash->ring_fn.u64.deq_multi(stash, val, num); } int32_t odp_stash_get_u64_batch(odp_stash_t st, uint64_t val[], int32_t num) @@ -631,7 +725,7 @@ int32_t odp_stash_get_u64_batch(odp_stash_t st, uint64_t val[], int32_t num) _ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); - return ring_u64_deq_batch(&stash->ring_u64.hdr, stash->ring_mask, val, num); + return stash->ring_fn.u64.deq_batch(stash, val, num); } int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num) @@ -644,13 +738,11 @@ int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num) _ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); if (sizeof(uintptr_t) == sizeof(uint32_t)) - return ring_u32_deq_multi(&stash->ring_u32.hdr, - stash->ring_mask, - (uint32_t *)(uintptr_t)ptr, num); - else if (sizeof(uintptr_t) == sizeof(uint64_t)) - return ring_u64_deq_multi(&stash->ring_u64.hdr, - stash->ring_mask, - (uint64_t *)(uintptr_t)ptr, num); + return stash->ring_fn.u32.deq_multi(stash, (uint32_t *)(uintptr_t)ptr, num); + + if (sizeof(uintptr_t) == sizeof(uint64_t)) + return stash->ring_fn.u64.deq_multi(stash, (uint64_t *)(uintptr_t)ptr, num); + return -1; } @@ -664,11 +756,11 @@ int32_t odp_stash_get_ptr_batch(odp_stash_t st, uintptr_t ptr[], int32_t num) _ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); if (sizeof(uintptr_t) == sizeof(uint32_t)) - return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask, - (uint32_t *)(uintptr_t)ptr, num); - else if (sizeof(uintptr_t) == sizeof(uint64_t)) - return ring_u64_deq_batch(&stash->ring_u64.hdr, stash->ring_mask, - (uint64_t *)(uintptr_t)ptr, num); + return stash->ring_fn.u32.deq_batch(stash, (uint32_t *)(uintptr_t)ptr, num); + + if (sizeof(uintptr_t) == sizeof(uint64_t)) + return stash->ring_fn.u64.deq_batch(stash, (uint64_t *)(uintptr_t)ptr, num); + return -1; } @@ -682,18 +774,12 @@ int odp_stash_flush_cache(odp_stash_t st) static uint32_t stash_obj_count(stash_t *stash) { - ring_u32_t *ring_u32; uint32_t obj_size = stash->obj_size; - if (obj_size == sizeof(uint64_t)) { - ring_u64_t *ring_u64 = &stash->ring_u64.hdr; - - return ring_u64_len(ring_u64); - } - - ring_u32 = &stash->ring_u32.hdr; + if (obj_size == sizeof(uint64_t)) + return stash->ring_fn.u64.len(stash); - return ring_u32_len(ring_u32); + return stash->ring_fn.u32.len(stash); } void odp_stash_print(odp_stash_t st) @@ -712,7 +798,7 @@ void odp_stash_print(odp_stash_t st) _ODP_PRINT(" index %i\n", stash->index); _ODP_PRINT(" obj size %u\n", stash->obj_size); _ODP_PRINT(" obj count %u\n", stash_obj_count(stash)); - _ODP_PRINT(" ring size %u\n", stash->ring_mask + 1); + _ODP_PRINT(" ring size %u\n", stash->ring_size); _ODP_PRINT("\n"); } -- cgit v1.2.3 From 151f650e8705e7bfa484c936155001fc17ceb447 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 19 Jan 2023 09:42:54 +0200 Subject: linux-gen: stash: add mpmc ring based implementation Use MPMC rings to implement overflow safe stash mode (application can attempt to store more handles into a stash than it specified in the creation parameters). This mode can be enabled by setting stash.strict_size option to zero in the config file. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- config/odp-linux-generic.conf | 8 +- platform/linux-generic/m4/odp_libconfig.m4 | 2 +- platform/linux-generic/odp_stash.c | 141 +++++++++++++++++++++++--- platform/linux-generic/test/inline-timer.conf | 2 +- platform/linux-generic/test/packet_align.conf | 2 +- platform/linux-generic/test/process-mode.conf | 2 +- platform/linux-generic/test/sched-basic.conf | 2 +- 7 files changed, 140 insertions(+), 19 deletions(-) diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index 3cb862ee4..df5328d1f 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.24" +config_file_version = "0.1.25" # System options system: { @@ -254,6 +254,12 @@ stash: { # The value may be rounded up by the implementation. For optimal memory # usage set value to a power of two - 1. max_num_obj = 4095 + + # Strict size + # + # When set to 0, application can attempt to store more handles into a + # stash than it specified in the creation parameters. + strict_size = 1 } timer: { diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4 index d96a95055..0d268c935 100644 --- a/platform/linux-generic/m4/odp_libconfig.m4 +++ b/platform/linux-generic/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [24]) +m4_define([_odp_config_version_minor], [25]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index a954cf866..5ff499843 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include #include @@ -94,6 +96,16 @@ typedef struct ODP_ALIGNED_CACHE stash_t { ring_u64_t hdr; uint64_t data[]; } ring_u64; + + struct ODP_ALIGNED_CACHE { + ring_mpmc_u32_t hdr; + uint32_t data[]; + } ring_mpmc_u32; + + struct ODP_ALIGNED_CACHE { + ring_mpmc_u64_t hdr; + uint64_t data[]; + } ring_mpmc_u64; }; } stash_t; @@ -105,6 +117,7 @@ typedef struct stash_global_t { uint32_t max_num; uint32_t max_num_obj; uint32_t num_internal; + uint8_t strict_size; uint8_t stash_state[CONFIG_MAX_STASHES]; stash_t *stash[CONFIG_MAX_STASHES]; uint8_t data[] ODP_ALIGNED_CACHE; @@ -131,6 +144,7 @@ int _odp_stash_init_global(void) uint64_t ring_max_size, stash_max_size, stash_data_size, offset; const uint32_t internal_stashes = odp_global_ro.disable.dma ? 0 : CONFIG_INTERNAL_STASHES; uint8_t *stash_data; + uint8_t strict_size; int val = 0; if (odp_global_ro.disable.stash && odp_global_ro.disable.dma) { @@ -156,6 +170,14 @@ int _odp_stash_init_global(void) _ODP_PRINT(" %s: %i\n", str, val); max_num_obj = val; + str = "stash.strict_size"; + if (!_odp_libconfig_lookup_int(str, &val)) { + _ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + _ODP_PRINT(" %s: %i\n", str, val); + strict_size = !!val; + _ODP_PRINT("\n"); /* Reserve resources for implementation internal stashes */ @@ -191,6 +213,7 @@ int _odp_stash_init_global(void) stash_global->shm = shm; stash_global->max_num = max_num; stash_global->max_num_obj = max_num_obj; + stash_global->strict_size = strict_size; stash_global->num_internal = internal_stashes; odp_ticketlock_init(&stash_global->lock); @@ -345,6 +368,80 @@ static inline int32_t strict_ring_u64_len(stash_t *stash) return ring_u64_len(&stash->ring_u64.hdr); } +static inline void mpmc_ring_u32_init(stash_t *stash) +{ + ring_mpmc_u32_init(&stash->ring_mpmc_u32.hdr); + + for (uint32_t i = 0; i < stash->ring_size; i++) + stash->ring_mpmc_u32.data[i] = 0; +} + +static inline void mpmc_ring_u64_init(stash_t *stash) +{ + ring_mpmc_u64_init(&stash->ring_mpmc_u64.hdr); + + for (uint32_t i = 0; i < stash->ring_size; i++) + stash->ring_mpmc_u64.data[i] = 0; +} + +static inline int32_t mpmc_ring_u32_enq_multi(stash_t *stash, const uint32_t val[], int32_t num) +{ + return ring_mpmc_u32_enq_multi(&stash->ring_mpmc_u32.hdr, stash->ring_mpmc_u32.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u64_enq_multi(stash_t *stash, const uint64_t val[], int32_t num) +{ + return ring_mpmc_u64_enq_multi(&stash->ring_mpmc_u64.hdr, stash->ring_mpmc_u64.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u32_enq_batch(stash_t *stash, const uint32_t val[], int32_t num) +{ + return ring_mpmc_u32_enq_batch(&stash->ring_mpmc_u32.hdr, stash->ring_mpmc_u32.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u64_enq_batch(stash_t *stash, const uint64_t val[], int32_t num) +{ + return ring_mpmc_u64_enq_batch(&stash->ring_mpmc_u64.hdr, stash->ring_mpmc_u64.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u32_deq_multi(stash_t *stash, uint32_t val[], int32_t num) +{ + return ring_mpmc_u32_deq_multi(&stash->ring_mpmc_u32.hdr, stash->ring_mpmc_u32.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u64_deq_multi(stash_t *stash, uint64_t val[], int32_t num) +{ + return ring_mpmc_u64_deq_multi(&stash->ring_mpmc_u64.hdr, stash->ring_mpmc_u64.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u32_deq_batch(stash_t *stash, uint32_t val[], int32_t num) +{ + return ring_mpmc_u32_deq_batch(&stash->ring_mpmc_u32.hdr, stash->ring_mpmc_u32.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u64_deq_batch(stash_t *stash, uint64_t val[], int32_t num) +{ + return ring_mpmc_u64_deq_batch(&stash->ring_mpmc_u64.hdr, stash->ring_mpmc_u64.data, + stash->ring_mask, val, num); +} + +static inline int32_t mpmc_ring_u32_len(stash_t *stash) +{ + return ring_mpmc_u32_len(&stash->ring_mpmc_u32.hdr); +} + +static inline int32_t mpmc_ring_u64_len(stash_t *stash) +{ + return ring_mpmc_u64_len(&stash->ring_mpmc_u64.hdr); +} + odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) { stash_t *stash; @@ -394,20 +491,38 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) memset(stash, 0, sizeof(stash_t)); /* Set ring function pointers */ - if (ring_u64) { - stash->ring_fn.u64.init = strict_ring_u64_init; - stash->ring_fn.u64.enq_multi = strict_ring_u64_enq_multi; - stash->ring_fn.u64.enq_batch = strict_ring_u64_enq_multi; - stash->ring_fn.u64.deq_multi = strict_ring_u64_deq_multi; - stash->ring_fn.u64.deq_batch = strict_ring_u64_deq_batch; - stash->ring_fn.u64.len = strict_ring_u64_len; + if (stash_global->strict_size) { + if (ring_u64) { + stash->ring_fn.u64.init = strict_ring_u64_init; + stash->ring_fn.u64.enq_multi = strict_ring_u64_enq_multi; + stash->ring_fn.u64.enq_batch = strict_ring_u64_enq_multi; + stash->ring_fn.u64.deq_multi = strict_ring_u64_deq_multi; + stash->ring_fn.u64.deq_batch = strict_ring_u64_deq_batch; + stash->ring_fn.u64.len = strict_ring_u64_len; + } else { + stash->ring_fn.u32.init = strict_ring_u32_init; + stash->ring_fn.u32.enq_multi = strict_ring_u32_enq_multi; + stash->ring_fn.u32.enq_batch = strict_ring_u32_enq_multi; + stash->ring_fn.u32.deq_multi = strict_ring_u32_deq_multi; + stash->ring_fn.u32.deq_batch = strict_ring_u32_deq_batch; + stash->ring_fn.u32.len = strict_ring_u32_len; + } } else { - stash->ring_fn.u32.init = strict_ring_u32_init; - stash->ring_fn.u32.enq_multi = strict_ring_u32_enq_multi; - stash->ring_fn.u32.enq_batch = strict_ring_u32_enq_multi; - stash->ring_fn.u32.deq_multi = strict_ring_u32_deq_multi; - stash->ring_fn.u32.deq_batch = strict_ring_u32_deq_batch; - stash->ring_fn.u32.len = strict_ring_u32_len; + if (ring_u64) { + stash->ring_fn.u64.init = mpmc_ring_u64_init; + stash->ring_fn.u64.enq_multi = mpmc_ring_u64_enq_multi; + stash->ring_fn.u64.enq_batch = mpmc_ring_u64_enq_batch; + stash->ring_fn.u64.deq_multi = mpmc_ring_u64_deq_multi; + stash->ring_fn.u64.deq_batch = mpmc_ring_u64_deq_batch; + stash->ring_fn.u64.len = mpmc_ring_u64_len; + } else { + stash->ring_fn.u32.init = mpmc_ring_u32_init; + stash->ring_fn.u32.enq_multi = mpmc_ring_u32_enq_multi; + stash->ring_fn.u32.enq_batch = mpmc_ring_u32_enq_batch; + stash->ring_fn.u32.deq_multi = mpmc_ring_u32_deq_multi; + stash->ring_fn.u32.deq_batch = mpmc_ring_u32_deq_batch; + stash->ring_fn.u32.len = mpmc_ring_u32_len; + } } if (name) diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index 56c1d40d1..44db4e337 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.24" +config_file_version = "0.1.25" timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf index 1aa7efc69..26491bd53 100644 --- a/platform/linux-generic/test/packet_align.conf +++ b/platform/linux-generic/test/packet_align.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.24" +config_file_version = "0.1.25" pool: { pkt: { diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index fb59a4f2a..2277aabdf 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.24" +config_file_version = "0.1.25" # Shared memory options shm: { diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf index f75141806..c9f7c79fd 100644 --- a/platform/linux-generic/test/sched-basic.conf +++ b/platform/linux-generic/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.24" +config_file_version = "0.1.25" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { -- cgit v1.2.3 From 2136847979cb513b67a245752943ebc373fd9445 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 19 Jan 2023 11:19:31 +0200 Subject: github_ci: add test case for custom stash config Add new test case for custom stash configuration (overflow safe mode). Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .github/workflows/ci-pipeline-arm64.yml | 12 ++++++++++++ .github/workflows/ci-pipeline.yml | 10 ++++++++++ platform/linux-generic/test/stash-custom.conf | 8 ++++++++ 3 files changed, 30 insertions(+) create mode 100644 platform/linux-generic/test/stash-custom.conf diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index 31747b784..9aac70664 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -204,6 +204,18 @@ jobs: if: ${{ failure() }} run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + Run_stash_config: + if: ${{ github.repository == 'OpenDataPlane/odp' }} + runs-on: [self-hosted, ARM64] + steps: + - uses: AutoModality/action-clean@v1.1.0 + - uses: actions/checkout@v3 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" + -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-generic/test/stash-custom.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + Run_scheduler_sp: if: ${{ github.repository == 'OpenDataPlane/odp' }} runs-on: [self-hosted, ARM64] diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 833ed92ce..fef67dcb8 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -370,6 +370,16 @@ jobs: if: ${{ failure() }} run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + Run_stash_config: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" + -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-generic/test/stash-custom.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh + - name: Failure log + if: ${{ failure() }} + run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done + Run_scheduler_sp: runs-on: ubuntu-20.04 steps: diff --git a/platform/linux-generic/test/stash-custom.conf b/platform/linux-generic/test/stash-custom.conf new file mode 100644 index 000000000..95af7a259 --- /dev/null +++ b/platform/linux-generic/test/stash-custom.conf @@ -0,0 +1,8 @@ +# Mandatory fields +odp_implementation = "linux-generic" +config_file_version = "0.1.25" + +# Test overflow safe stash variant +stash: { + strict_size = 0 +} -- cgit v1.2.3 From 4050304bd96f80f4fb94cc9443131dfe5be9f2a4 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 23 Dec 2022 16:47:15 +0200 Subject: validation: crypto: remove testing of deprecated per-session IVs Do not test deprecated per-session IVs. This prepares for full removal of per-session IVs from ODP API. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Anoob Joseph --- test/validation/api/crypto/odp_crypto_test_inp.c | 71 +++--------------------- 1 file changed, 7 insertions(+), 64 deletions(-) diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index 20a346c55..802a419e2 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -46,13 +46,6 @@ static void test_defaults(uint8_t fill) CU_ASSERT_EQUAL(param.auth_alg, ODP_AUTH_ALG_NULL); CU_ASSERT_EQUAL(param.auth_iv_len, 0); CU_ASSERT_EQUAL(param.auth_aad_len, 0); - -#if ODP_DEPRECATED_API - CU_ASSERT_EQUAL(param.cipher_iv.data, NULL); - CU_ASSERT_EQUAL(param.cipher_iv.length, 0); - CU_ASSERT_EQUAL(param.auth_iv.data, NULL); - CU_ASSERT_EQUAL(param.auth_iv.length, 0); -#endif } static void test_default_values(void) @@ -478,7 +471,6 @@ typedef struct alg_test_param_t { odp_auth_alg_t auth_alg; crypto_test_reference_t *ref; uint32_t digest_offset; - odp_bool_t override_iv; odp_bool_t is_bit_mode_cipher; odp_bool_t is_bit_mode_auth; odp_bool_t adjust_segmentation; @@ -496,8 +488,8 @@ static void alg_test_execute(const alg_test_param_t *param) odp_packet_data_range_t zero_range = {.offset = 0, .length = 0}; odp_packet_data_range_t cipher_range; odp_packet_data_range_t auth_range; - uint8_t *cipher_iv = param->override_iv ? ref->cipher_iv : NULL; - uint8_t *auth_iv = param->override_iv ? ref->auth_iv : NULL; + uint8_t *cipher_iv = ref->cipher_iv; + uint8_t *auth_iv = ref->auth_iv; test_packet_md_t md_in, md_out; cipher_range.offset = param->header_len; @@ -624,12 +616,6 @@ static void alg_test_execute(const alg_test_param_t *param) } } -typedef enum { - PACKET_IV, - OLD_PACKET_IV, - OLD_SESSION_IV, -} iv_test_mode_t; - typedef enum { HASH_NO_OVERLAP, HASH_OVERLAP, @@ -639,7 +625,6 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, odp_cipher_alg_t cipher_alg, odp_auth_alg_t auth_alg, crypto_test_reference_t *ref, - iv_test_mode_t iv_mode, hash_test_mode_t hash_mode) { odp_crypto_session_t session = ODP_CRYPTO_SESSION_INVALID; @@ -656,23 +641,6 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, .data = auth_key_data, .length = ref->auth_key_length }; -#if ODP_DEPRECATED_API - uint8_t cipher_iv_data[ref->cipher_iv_length]; - uint8_t auth_iv_data[ref->auth_iv_length]; - odp_crypto_iv_t cipher_iv = { - .length = ref->cipher_iv_length - }; - odp_crypto_iv_t auth_iv = { - .length = ref->auth_iv_length - }; - - if (iv_mode == OLD_SESSION_IV) { - memcpy(cipher_iv_data, ref->cipher_iv, ref->cipher_iv_length); - memcpy(auth_iv_data, ref->auth_iv, ref->auth_iv_length); - cipher_iv.data = cipher_iv_data; - auth_iv.data = auth_iv_data; - } -#endif memcpy(cipher_key_data, ref->cipher_key, ref->cipher_key_length); memcpy(auth_key_data, ref->auth_key, ref->auth_key_length); @@ -690,15 +658,8 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, ses_params.compl_queue = suite_context.queue; ses_params.output_pool = suite_context.pool; ses_params.cipher_key = cipher_key; - if (iv_mode == PACKET_IV) { - ses_params.cipher_iv_len = ref->cipher_iv_length; - ses_params.auth_iv_len = ref->auth_iv_length; - } else { -#if ODP_DEPRECATED_API - ses_params.cipher_iv = cipher_iv; - ses_params.auth_iv = auth_iv; -#endif - } + ses_params.cipher_iv_len = ref->cipher_iv_length; + ses_params.auth_iv_len = ref->auth_iv_length; ses_params.auth_key = auth_key; ses_params.auth_digest_len = ref->digest_length; ses_params.auth_aad_len = ref->aad_length; @@ -736,10 +697,6 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, */ memset(cipher_key_data, 0, sizeof(cipher_key_data)); memset(auth_key_data, 0, sizeof(auth_key_data)); -#if ODP_DEPRECATED_API - memset(cipher_iv_data, 0, sizeof(cipher_iv_data)); - memset(auth_iv_data, 0, sizeof(auth_iv_data)); -#endif memset(&ses_params, 0, sizeof(ses_params)); return session; @@ -750,7 +707,6 @@ static void alg_test(odp_crypto_op_t op, odp_auth_alg_t auth_alg, crypto_test_reference_t *ref, uint32_t digest_offset, - iv_test_mode_t iv_mode, odp_bool_t is_bit_mode_cipher, odp_bool_t is_bit_mode_auth) { @@ -764,7 +720,7 @@ static void alg_test(odp_crypto_op_t op, uint32_t max_shift; alg_test_param_t test_param; - session = session_create(op, cipher_alg, auth_alg, ref, iv_mode, hash_mode); + session = session_create(op, cipher_alg, auth_alg, ref, hash_mode); if (session == ODP_CRYPTO_SESSION_INVALID) return; @@ -774,7 +730,6 @@ static void alg_test(odp_crypto_op_t op, test_param.cipher_alg = cipher_alg; test_param.auth_alg = auth_alg; test_param.ref = ref; - test_param.override_iv = (iv_mode != OLD_SESSION_IV); test_param.is_bit_mode_cipher = is_bit_mode_cipher; test_param.is_bit_mode_auth = is_bit_mode_auth; test_param.digest_offset = digest_offset; @@ -923,18 +878,8 @@ static void check_alg(odp_crypto_op_t op, continue; } - /* test with per-packet IV */ alg_test(op, cipher_alg, auth_alg, &ref[idx], digest_offs, - PACKET_IV, is_bit_mode_cipher, is_bit_mode_auth); -#if ODP_DEPRECATED_API - /* test with per-packet IV using the old API*/ - alg_test(op, cipher_alg, auth_alg, &ref[idx], digest_offs, - OLD_PACKET_IV, is_bit_mode_cipher, is_bit_mode_auth); - - /* test with per-session IV */ - alg_test(op, cipher_alg, auth_alg, &ref[idx], digest_offs, - OLD_SESSION_IV, is_bit_mode_cipher, is_bit_mode_auth); -#endif + is_bit_mode_cipher, is_bit_mode_auth); cipher_tested[cipher_idx] = true; auth_tested[auth_idx] = true; @@ -1239,7 +1184,7 @@ static int create_hash_test_reference(odp_auth_alg_t auth, CU_ASSERT(rc == 0); session = session_create(ODP_CRYPTO_OP_ENCODE, ODP_CIPHER_ALG_NULL, - auth, ref, PACKET_IV, HASH_NO_OVERLAP); + auth, ref, HASH_NO_OVERLAP); if (session == ODP_CRYPTO_SESSION_INVALID) return -1; @@ -1295,7 +1240,6 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, auth, &ref, digest_offset, - PACKET_IV, false, capa->bit_mode); @@ -1318,7 +1262,6 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, auth, &ref, digest_offset, - PACKET_IV, false, capa->bit_mode); } -- cgit v1.2.3 From 6873292bf6fead357eebbd7368f13d91f7ddc769 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 23 Dec 2022 16:47:17 +0200 Subject: api: crypto: remove deprecated per-session IVs Remove deprecated per-session IV configuration. Require that IV is passed via crypto operation parameters also when deprecated API is enabled. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Anoob Joseph --- include/odp/api/spec/crypto_types.h | 82 +++---------------------------------- 1 file changed, 6 insertions(+), 76 deletions(-) diff --git a/include/odp/api/spec/crypto_types.h b/include/odp/api/spec/crypto_types.h index 7f2c49a7d..01947c532 100644 --- a/include/odp/api/spec/crypto_types.h +++ b/include/odp/api/spec/crypto_types.h @@ -240,8 +240,8 @@ typedef enum { * require the data to be contiguous in memory, are ignored with * AES-GMAC. * - * GMAC needs an initialization vector, which can be passed via - * session (auth_iv) or packet (auth_iv_ptr) level parameters. + * GMAC needs an initialization vector, which must be passed via + * operation parameters (auth_iv_ptr). */ ODP_AUTH_ALG_AES_GMAC, @@ -511,28 +511,6 @@ typedef struct odp_crypto_key { } odp_crypto_key_t; -/** - * Crypto API IV structure - * - * @deprecated Use per-packet IV in crypto operation parameters - */ -typedef struct odp_crypto_iv { - /** IV data - * - * Ignored when length is zero. Null value indicates that an - * IV will be provided for each packet through the crypto - * operation parameters. In that case the per-operation - * IV parameter must always point to a valid IV. - * - * Default value is NULL. - */ - uint8_t *data; - - /** IV length in bytes. Default value is zero. */ - uint32_t length; - -} ODP_DEPRECATE(odp_crypto_iv_t); - /** * Crypto API session creation parameters */ @@ -602,32 +580,8 @@ typedef struct odp_crypto_session_param_t { */ odp_crypto_key_t cipher_key; - /** Cipher Initialization Vector (IV) - * - * Unless using the deprecated API, this specifies the length of - * the IV only. The actual IV must then be provided in per-packet - * parameters of crypto operations. - */ - union { -#if ODP_DEPRECATED_API - /** @deprecated Cipher IV */ - odp_crypto_iv_t ODP_DEPRECATE(cipher_iv); -#endif - /** Cipher IV length */ - struct { -#if ODP_DEPRECATED_API - /** @cond - * Unused padding field - */ - uint8_t *dummy_padding_0; - /** @endcond */ -#endif - /** Length of cipher initialization vector. - * Default value is zero. - */ - uint32_t cipher_iv_len; - }; - }; + /** Cipher IV length. The default value is zero. */ + uint32_t cipher_iv_len; /** Authentication algorithm * @@ -652,32 +606,8 @@ typedef struct odp_crypto_session_param_t { */ odp_crypto_key_t auth_key; - /** Authentication Initialization Vector (IV) - * - * Unless using the deprecated API, this specifies the length of - * the IV only. The actual IV must then be provided in per-packet - * parameters of crypto operations. - */ - union { -#if ODP_DEPRECATED_API - /** @deprecated Authentication IV */ - odp_crypto_iv_t ODP_DEPRECATE(auth_iv); -#endif - /** Authentication IV length */ - struct { -#if ODP_DEPRECATED_API - /** @cond - * Unused padding field - */ - uint8_t *dummy_padding_1; - /** @endcond */ -#endif - /** Length of authentication initialization vector. - * Default value is zero. - */ - uint32_t auth_iv_len; - }; - }; + /** Authentication IV length. The default value is zero. */ + uint32_t auth_iv_len; /** Authentication digest length in bytes * -- cgit v1.2.3 From 5292e45beafbeb63a9c848f9f54f740d8921d551 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 23 Dec 2022 16:47:18 +0200 Subject: linux-gen: crypto: openssl: remove deprecated per-session IVs Stop supporting deprecated per-session IVs. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Anoob Joseph --- platform/linux-generic/odp_crypto_openssl.c | 213 ++-------------------------- 1 file changed, 15 insertions(+), 198 deletions(-) diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 88b9e5800..f629ddf67 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -219,12 +219,7 @@ struct odp_crypto_generic_session_t { odp_bool_t do_cipher_first; struct { -#if ODP_DEPRECATED_API - /* Copy of session IV data */ - uint8_t iv_data[EVP_MAX_IV_LENGTH]; -#endif uint8_t key_data[EVP_MAX_KEY_LENGTH]; - const EVP_CIPHER *evp_cipher; crypto_func_t func; crypto_init_func_t init; @@ -232,9 +227,6 @@ struct odp_crypto_generic_session_t { struct { uint8_t key[EVP_MAX_KEY_LENGTH]; -#if ODP_DEPRECATED_API - uint8_t iv_data[EVP_MAX_IV_LENGTH]; -#endif union { const EVP_MD *evp_md; const EVP_CIPHER *evp_cipher; @@ -710,23 +702,11 @@ int packet_cmac_eia2(odp_packet_t pkt, uint8_t *hash) { CMAC_CTX *ctx = local.cmac_ctx[session->idx]; - void *iv_ptr; + void *iv_ptr = param->auth_iv_ptr; uint32_t offset = param->auth_range.offset; uint32_t len = param->auth_range.length; size_t outlen; -#if ODP_DEPRECATED_API - if (param->auth_iv_ptr) - iv_ptr = param->auth_iv_ptr; - else if (session->p.auth_iv.data) - iv_ptr = session->auth.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->auth_iv_ptr; - _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); -#endif - _ODP_ASSERT(offset + len <= odp_packet_len(pkt)); /* Reinitialize CMAC calculation without resetting the key */ @@ -1073,22 +1053,9 @@ odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; - void *iv_ptr; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); ret = internal_encrypt(ctx, pkt, param); @@ -1112,22 +1079,9 @@ odp_crypto_alg_err_t cipher_decrypt(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; - void *iv_ptr; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); ret = internal_decrypt(ctx, pkt, param); @@ -1172,7 +1126,6 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; - void *iv_ptr; int dummy_len = 0; int cipher_len; uint32_t in_len = (param->cipher_range.length + 7) / 8; @@ -1183,18 +1136,7 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt, /* Range offset is in bits in bit mode but must be divisible by 8. */ offset = param->cipher_range.offset / 8; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); odp_packet_copy_to_mem(pkt, offset, in_len, data); @@ -1216,7 +1158,6 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; - void *iv_ptr; int dummy_len = 0; int cipher_len; uint32_t in_len = (param->cipher_range.length + 7) / 8; @@ -1227,18 +1168,7 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt, /* Range offset is in bits in bit mode but must be divisible by 8. */ offset = param->cipher_range.offset / 8; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); odp_packet_copy_to_mem(pkt, offset, in_len, data); @@ -1303,24 +1233,11 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt, EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; const uint8_t *aad_head = param->aad_ptr; uint32_t aad_len = session->p.auth_aad_len; - void *iv_ptr; int dummy_len = 0; uint8_t block[EVP_MAX_MD_SIZE]; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); /* Authenticate header data (if any) without encrypting them */ if (aad_len > 0) @@ -1359,23 +1276,10 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt, const uint8_t *aad_head = param->aad_ptr; uint32_t aad_len = session->p.auth_aad_len; int dummy_len = 0; - void *iv_ptr; uint8_t block[EVP_MAX_MD_SIZE]; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); odp_packet_copy_to_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); @@ -1440,23 +1344,10 @@ odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx]; - void *iv_ptr; uint8_t block[EVP_MAX_MD_SIZE]; int ret; -#if ODP_DEPRECATED_API - if (param->auth_iv_ptr) - iv_ptr = param->auth_iv_ptr; - else if (session->p.auth_iv.data) - iv_ptr = session->auth.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->auth_iv_ptr; - _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->auth_iv_ptr); ret = internal_aad(ctx, pkt, param, true); @@ -1487,23 +1378,10 @@ odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx]; - void *iv_ptr; uint8_t block[EVP_MAX_MD_SIZE]; int ret; -#if ODP_DEPRECATED_API - if (param->auth_iv_ptr) - iv_ptr = param->auth_iv_ptr; - else if (session->p.auth_iv.data) - iv_ptr = session->auth.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->auth_iv_ptr; - _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->auth_iv_ptr); odp_packet_copy_to_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); @@ -1568,7 +1446,6 @@ odp_crypto_alg_err_t aes_ccm_encrypt(odp_packet_t pkt, EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; const uint8_t *aad_head = param->aad_ptr; uint32_t aad_len = session->p.auth_aad_len; - void *iv_ptr; int dummy_len = 0; int cipher_len; uint32_t in_len = param->cipher_range.length; @@ -1576,21 +1453,9 @@ odp_crypto_alg_err_t aes_ccm_encrypt(odp_packet_t pkt, uint8_t block[EVP_MAX_MD_SIZE]; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, session->p.auth_digest_len, NULL); - EVP_EncryptInit_ex(ctx, NULL, NULL, session->cipher.key_data, iv_ptr); + EVP_EncryptInit_ex(ctx, NULL, NULL, session->cipher.key_data, param->cipher_iv_ptr); /* Set len */ EVP_EncryptUpdate(ctx, NULL, &dummy_len, NULL, in_len); @@ -1640,7 +1505,6 @@ odp_crypto_alg_err_t aes_ccm_decrypt(odp_packet_t pkt, EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; const uint8_t *aad_head = param->aad_ptr; uint32_t aad_len = session->p.auth_aad_len; - void *iv_ptr; int dummy_len = 0; int cipher_len; uint32_t in_len = param->cipher_range.length; @@ -1648,23 +1512,11 @@ odp_crypto_alg_err_t aes_ccm_decrypt(odp_packet_t pkt, uint8_t block[EVP_MAX_MD_SIZE]; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - odp_packet_copy_to_mem(pkt, param->hash_result_offset, session->p.auth_digest_len, block); EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, session->p.auth_digest_len, block); - EVP_DecryptInit_ex(ctx, NULL, NULL, session->cipher.key_data, iv_ptr); + EVP_DecryptInit_ex(ctx, NULL, NULL, session->cipher.key_data, param->cipher_iv_ptr); /* Set len */ EVP_DecryptUpdate(ctx, NULL, &dummy_len, NULL, in_len); @@ -1725,26 +1577,13 @@ odp_crypto_alg_err_t xts_encrypt(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; - void *iv_ptr; int dummy_len = 0; int cipher_len; uint32_t in_len = param->cipher_range.length; uint8_t data[in_len]; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); odp_packet_copy_to_mem(pkt, param->cipher_range.offset, in_len, data); @@ -1767,26 +1606,13 @@ odp_crypto_alg_err_t xts_decrypt(odp_packet_t pkt, odp_crypto_generic_session_t *session) { EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx]; - void *iv_ptr; int dummy_len = 0; int cipher_len; uint32_t in_len = param->cipher_range.length; uint8_t data[in_len]; int ret; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif - - EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr); odp_packet_copy_to_mem(pkt, param->cipher_range.offset, in_len, data); @@ -2234,17 +2060,6 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, goto err; } -#if ODP_DEPRECATED_API - /* Copy IV data */ - if (session->p.cipher_iv.data) - memcpy(session->cipher.iv_data, session->p.cipher_iv.data, - session->p.cipher_iv.length); - - if (session->p.auth_iv.data) - memcpy(session->auth.iv_data, session->p.auth_iv.data, - session->p.auth_iv.length); -#endif - /* Derive order */ if (ODP_CRYPTO_OP_ENCODE == param->op) session->do_cipher_first = param->auth_cipher_text; @@ -2857,6 +2672,8 @@ int crypto_int(odp_packet_t pkt_in, goto out; } } + _ODP_ASSERT(session->p.cipher_iv_len == 0 || param->cipher_iv_ptr != NULL); + _ODP_ASSERT(session->p.auth_iv_len == 0 || param->auth_iv_ptr != NULL); crypto_init(session); -- cgit v1.2.3 From 000b0d661b65075424579fb7324576cacd1719df Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 23 Dec 2022 16:47:20 +0200 Subject: linux-gen: crypto: ipsecmb: remove deprecated per-session IVs Stop supporting deprecated per-session IVs. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Anoob Joseph --- platform/linux-generic/odp_crypto_ipsecmb.c | 73 +++-------------------------- 1 file changed, 6 insertions(+), 67 deletions(-) diff --git a/platform/linux-generic/odp_crypto_ipsecmb.c b/platform/linux-generic/odp_crypto_ipsecmb.c index 89e964a83..3532be908 100644 --- a/platform/linux-generic/odp_crypto_ipsecmb.c +++ b/platform/linux-generic/odp_crypto_ipsecmb.c @@ -32,7 +32,6 @@ /* Length in bytes */ #define IPSEC_MB_CRYPTO_MAX_CIPHER_KEY_LENGTH 32 #define IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH 32 -#define IPSEC_MB_CRYPTO_MAX_IV_LENGTH 32 #define IPSEC_MB_CRYPTO_MAX_DATA_LENGTH 65536 #define ZUC_DIGEST_LENGTH 4 @@ -85,19 +84,12 @@ struct odp_crypto_generic_session_t { odp_bool_t do_cipher_first; struct { -#if ODP_DEPRECATED_API - /* Copy of session IV data */ - uint8_t iv_data[IPSEC_MB_CRYPTO_MAX_IV_LENGTH]; -#endif uint8_t key_data[IPSEC_MB_CRYPTO_MAX_CIPHER_KEY_LENGTH]; crypto_func_t func; } cipher; struct { uint8_t key[IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH]; -#if ODP_DEPRECATED_API - uint8_t iv_data[IPSEC_MB_CRYPTO_MAX_IV_LENGTH]; -#endif crypto_func_t func; } auth; @@ -165,21 +157,11 @@ odp_crypto_alg_err_t zuc_eea3_cipher_op(odp_packet_t pkt, odp_crypto_generic_session_t *session) { IMB_MGR *mb_mgr = local.mb_mgr; - uint8_t *iv_ptr; + uint8_t *iv_ptr = param->cipher_iv_ptr; uint32_t in_pos = param->cipher_range.offset; uint32_t in_len = param->cipher_range.length; -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL); -#endif + _ODP_ASSERT(iv_ptr != NULL); uint32_t seg_len = 0; uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); @@ -242,22 +224,12 @@ odp_crypto_alg_err_t auth_zuc_eia3_gen(odp_packet_t pkt, odp_crypto_generic_session_t *session) { IMB_MGR *mb_mgr = local.mb_mgr; - uint8_t *iv_ptr; + uint8_t *iv_ptr = param->auth_iv_ptr; uint32_t in_pos = param->auth_range.offset; uint32_t in_len = param->auth_range.length; uint32_t auth_tag; -#if ODP_DEPRECATED_API - if (param->auth_iv_ptr) - iv_ptr = param->auth_iv_ptr; - else if (session->p.auth_iv.data) - iv_ptr = session->auth.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->auth_iv_ptr; - _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); -#endif + _ODP_ASSERT(iv_ptr != NULL); uint32_t seg_len = 0; uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); @@ -306,7 +278,7 @@ odp_crypto_alg_err_t auth_zuc_eia3_check(odp_packet_t pkt, odp_crypto_generic_session_t *session) { IMB_MGR *mb_mgr = local.mb_mgr; - uint8_t *iv_ptr; + uint8_t *iv_ptr = param->auth_iv_ptr; uint32_t in_pos = param->auth_range.offset; uint32_t in_len = param->auth_range.length; uint32_t bytes = ZUC_DIGEST_LENGTH; @@ -320,17 +292,7 @@ odp_crypto_alg_err_t auth_zuc_eia3_check(odp_packet_t pkt, if (odp_unlikely(session->p.hash_result_in_auth_range)) _odp_packet_set_data(pkt, param->hash_result_offset, 0, bytes); -#if ODP_DEPRECATED_API - if (param->auth_iv_ptr) - iv_ptr = param->auth_iv_ptr; - else if (session->p.auth_iv.data) - iv_ptr = session->auth.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; -#else - iv_ptr = param->auth_iv_ptr; - _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL); -#endif + _ODP_ASSERT(iv_ptr != NULL); uint32_t seg_len = 0; uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); @@ -499,29 +461,6 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, session->p = *param; - if (session->p.cipher_iv_len > IPSEC_MB_CRYPTO_MAX_IV_LENGTH) { - _ODP_DBG("Maximum IV length exceeded\n"); - *status = ODP_CRYPTO_SES_ERR_CIPHER; - goto err; - } - - if (session->p.auth_iv_len > IPSEC_MB_CRYPTO_MAX_IV_LENGTH) { - _ODP_DBG("Maximum auth IV length exceeded\n"); - *status = ODP_CRYPTO_SES_ERR_CIPHER; - goto err; - } - -#if ODP_DEPRECATED_API - /* Copy IV data */ - if (session->p.cipher_iv.data) - memcpy(session->cipher.iv_data, session->p.cipher_iv.data, - session->p.cipher_iv.length); - - if (session->p.auth_iv.data) - memcpy(session->auth.iv_data, session->p.auth_iv.data, - session->p.auth_iv.length); -#endif - /* Derive order */ if (ODP_CRYPTO_OP_ENCODE == param->op) session->do_cipher_first = param->auth_cipher_text; -- cgit v1.2.3 From 0fe16f26c1b7a32f1d3f97cf82b77146a97e2f07 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 23 Dec 2022 16:47:22 +0200 Subject: linux-gen: crypto: arm: remove deprecated per-session IVs Stop supporting deprecated per-session IVs. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Anoob Joseph --- .../linux-generic/arch/aarch64/odp_crypto_armv8.c | 54 +++------------------- 1 file changed, 6 insertions(+), 48 deletions(-) diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index a1c2c96d2..6e266e0c8 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -115,18 +115,11 @@ struct odp_crypto_generic_session_t { odp_crypto_session_param_t p; struct { -#if ODP_DEPRECATED_API - /* Copy of session IV data */ - uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; -#endif uint8_t key_data[ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH]; } cipher; struct { uint8_t key[ARM_CRYPTO_MAX_AUTH_KEY_LENGTH]; -#if ODP_DEPRECATED_API - uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; -#endif } auth; crypto_func_t func; @@ -232,7 +225,6 @@ void aes_gcm_encrypt(odp_packet_t pkt, } }; uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; - uint8_t *iv_ptr; uint64_t iv_bit_length = AES_GCM_IV_LEN * 8; uint64_t plaintext_bit_length = param->cipher_range.length * 8; uint64_t aad_bit_length = session->p.auth_aad_len * 8; @@ -249,24 +241,13 @@ void aes_gcm_encrypt(odp_packet_t pkt, goto err; } -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - goto err; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(iv_ptr != NULL); -#endif /* The crypto lib may read 16 bytes. Copy to a big enough buffer */ - memcpy(iv_data, iv_ptr, AES_GCM_IV_LEN); - iv_ptr = iv_data; + _ODP_ASSERT(param->cipher_iv_ptr != NULL); + memcpy(iv_data, param->cipher_iv_ptr, AES_GCM_IV_LEN); cs.constants = &session->cc; - rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs); + rc = armv8_aes_gcm_set_counter(iv_data, iv_bit_length, &cs); if (odp_unlikely(rc)) { _ODP_DBG("ARM Crypto: Failure while setting nonce\n"); goto err; @@ -336,7 +317,6 @@ void aes_gcm_decrypt(odp_packet_t pkt, } }; uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; - uint8_t *iv_ptr; uint8_t tag[AES_GCM_TAG_LEN]; uint64_t iv_bit_length = AES_GCM_IV_LEN * 8; uint64_t plaintext_bit_length = param->cipher_range.length * 8; @@ -353,24 +333,13 @@ void aes_gcm_decrypt(odp_packet_t pkt, goto err; } -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - iv_ptr = param->cipher_iv_ptr; - else if (session->p.cipher_iv.data) - iv_ptr = session->cipher.iv_data; - else - goto err; -#else - iv_ptr = param->cipher_iv_ptr; - _ODP_ASSERT(iv_ptr != NULL); -#endif /* The crypto lib may read 16 bytes. Copy to a big enough buffer */ - memcpy(iv_data, iv_ptr, AES_GCM_IV_LEN); - iv_ptr = iv_data; + _ODP_ASSERT(param->cipher_iv_ptr != NULL); + memcpy(iv_data, param->cipher_iv_ptr, AES_GCM_IV_LEN); cs.constants = &session->cc; - rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs); + rc = armv8_aes_gcm_set_counter(iv_data, iv_bit_length, &cs); if (odp_unlikely(rc)) { _ODP_DBG("ARM Crypto: Failure while setting nonce\n"); goto err; @@ -581,17 +550,6 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, goto err; } -#if ODP_DEPRECATED_API - /* Copy IV data */ - if (session->p.cipher_iv.data) - memcpy(session->cipher.iv_data, session->p.cipher_iv.data, - session->p.cipher_iv.length); - - if (session->p.auth_iv.data) - memcpy(session->auth.iv_data, session->p.auth_iv.data, - session->p.auth_iv.length); -#endif - /* Process based on cipher */ switch (param->cipher_alg) { case ODP_CIPHER_ALG_NULL: -- cgit v1.2.3 From ab179ac6e4cff11c9bb319cec86e306f1227a878 Mon Sep 17 00:00:00 2001 From: Pavan Nikhilesh Date: Fri, 6 Jan 2023 01:46:44 +0530 Subject: validation: time: reset count before reading ts Reset busy loop count to zero before reading the next set of timestamps. Signed-off-by: Pavan Nikhilesh Reviewed-by: Matias Elo --- test/validation/api/time/time.c | 1 + 1 file changed, 1 insertion(+) diff --git a/test/validation/api/time/time.c b/test/validation/api/time/time.c index 45bfc8264..f285bf8e4 100644 --- a/test/validation/api/time/time.c +++ b/test/validation/api/time/time.c @@ -139,6 +139,7 @@ static void time_test_monotony(void) lns_t2 = odp_time_local_ns(); gns_t2 = odp_time_global_ns(); + count = 0; while (count < BUSY_LOOP_CNT) { count++; }; -- cgit v1.2.3 From f198108bd0c6210b4dc49df4a3fe5429f526d028 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Fri, 3 Feb 2023 09:46:20 +0000 Subject: validation: dma: validate user context pointer value In addition to checking that DMA implementation passes the user context pointer of completion parameters as is back to application (in poll and event completion cases), check also that the value pointed to by it remains unchanged. Signed-off-by: Tuomas Taipale Reviewed-by: Petri Savolainen --- test/validation/api/dma/dma.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/validation/api/dma/dma.c b/test/validation/api/dma/dma.c index 8eb75b172..f3e967193 100644 --- a/test/validation/api/dma/dma.c +++ b/test/validation/api/dma/dma.c @@ -19,6 +19,7 @@ #define TRAILER 10 #define MULTI 1 #define RESULT 1 +#define USER_DATA 0xdeadbeef typedef struct global_t { odp_dma_capability_t dma_capa; @@ -385,12 +386,13 @@ static int do_transfer_async(odp_dma_t dma, const odp_dma_transfer_param_t *trs_ odp_dma_compl_param_t compl_param; odp_event_t ev; odp_dma_compl_t compl; - int i, ret, done, dummy; + int i, ret, done; + uint32_t user_data = USER_DATA; odp_dma_result_t result; odp_dma_transfer_id_t transfer_id = ODP_DMA_TRANSFER_ID_INVALID; uint64_t wait_ns = 500 * ODP_TIME_MSEC_IN_NS; uint64_t sched_wait = odp_schedule_wait_time(wait_ns); - void *user_ptr = &dummy; + void *user_ptr = &user_data; odp_dma_compl_param_init(&compl_param); compl_param.compl_mode = compl_mode; @@ -452,6 +454,7 @@ static int do_transfer_async(odp_dma_t dma, const odp_dma_transfer_param_t *trs_ CU_ASSERT(done == 1); CU_ASSERT(result.success); CU_ASSERT(result.user_ptr == user_ptr); + CU_ASSERT(user_data == USER_DATA); odp_dma_transfer_id_free(dma, transfer_id); @@ -479,6 +482,7 @@ static int do_transfer_async(odp_dma_t dma, const odp_dma_transfer_param_t *trs_ CU_ASSERT(odp_dma_compl_result(compl, &result) == 0); CU_ASSERT(result.success); CU_ASSERT(result.user_ptr == user_ptr); + CU_ASSERT(user_data == USER_DATA); /* Test also without result struct output */ CU_ASSERT(odp_dma_compl_result(compl, NULL) == 0); -- cgit v1.2.3 From 6b798ec547fda27329259f8c4f4695e4606a081f Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 19 Dec 2022 15:25:39 +0200 Subject: api: pktio: remove deprecated in_unknown_protos statistics counter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove deprecated 'in_unknown_protos' field from odp_pktio_stats_t. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- include/odp/api/spec/packet_io_stats.h | 15 --------------- platform/linux-generic/pktio/stats/packet_io_stats.c | 5 ----- 2 files changed, 20 deletions(-) diff --git a/include/odp/api/spec/packet_io_stats.h b/include/odp/api/spec/packet_io_stats.h index c6b782d5a..156a3a95e 100644 --- a/include/odp/api/spec/packet_io_stats.h +++ b/include/odp/api/spec/packet_io_stats.h @@ -19,7 +19,6 @@ extern "C" { #endif -#include #include /** @addtogroup odp_packet_io @@ -68,20 +67,6 @@ typedef struct odp_pktio_stats_t { * configuration, packets with errors may be dropped or not. */ uint64_t in_errors; - /** - * For packet-oriented interfaces, the number of packets received via - * the interface which were discarded because of an unknown or - * unsupported protocol. For character-oriented or fixed-length - * interfaces that support protocol multiplexing the number of - * transmission units received via the interface which were discarded - * because of an unknown or unsupported protocol. For any interface - * that does not support protocol multiplexing, this counter will always - * be 0. See ifInUnknownProtos in RFC 2863, RFC 3635. - * - * @deprecated This counter has been deprecated. - */ - uint64_t ODP_DEPRECATE(in_unknown_protos); - /** Number of octets in successfully transmitted packets. In case of * Ethernet, packet size includes MAC header. */ uint64_t out_octets; diff --git a/platform/linux-generic/pktio/stats/packet_io_stats.c b/platform/linux-generic/pktio/stats/packet_io_stats.c index ac61c0343..280aca250 100644 --- a/platform/linux-generic/pktio/stats/packet_io_stats.c +++ b/platform/linux-generic/pktio/stats/packet_io_stats.c @@ -5,7 +5,6 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#include #include #include #include @@ -77,10 +76,6 @@ int _odp_sock_stats_fd(pktio_entry_t *pktio_entry, pktio_entry->stats.in_discards; stats->in_errors = cur_stats.in_errors - pktio_entry->stats.in_errors; -#if ODP_DEPRECATED_API - stats->in_unknown_protos = cur_stats.in_unknown_protos - - pktio_entry->stats.in_unknown_protos; -#endif stats->out_octets = cur_stats.out_octets - pktio_entry->stats.out_octets; stats->out_packets = cur_stats.out_packets - -- cgit v1.2.3 From 740bcb143e570901ff67da3df0c7e53fde975eb5 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 19 Dec 2022 15:29:57 +0200 Subject: api: pktio: remove deprecated odp_pktin_ts_res() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove deprecated odp_pktin_ts_res() function. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- include/odp/api/spec/packet_io.h | 16 ---------------- platform/linux-generic/odp_packet_io.c | 5 ----- 2 files changed, 21 deletions(-) diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h index e8f0cd0b9..a23f2d9ac 100644 --- a/include/odp/api/spec/packet_io.h +++ b/include/odp/api/spec/packet_io.h @@ -832,22 +832,6 @@ int odp_pktio_info(odp_pktio_t pktio, odp_pktio_info_t *info); */ int odp_pktio_link_info(odp_pktio_t pktio, odp_pktio_link_info_t *info); -/** - * Packet input timestamp resolution in hertz - * - * @deprecated Use odp_pktio_ts_res() instead, which returns resolution for - * both packet input and output timestamps. - * - * This is the resolution of packet input timestamps. Returns zero on a failure - * or when timestamping is disabled. - * - * @param pktio Packet IO handle - * - * @return Packet input timestamp resolution in hertz - * @retval 0 on failure - */ -uint64_t ODP_DEPRECATE(odp_pktin_ts_res)(odp_pktio_t pktio); - /** * Convert nanoseconds to packet input time * diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 663b31759..d3e51aa01 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -1613,11 +1613,6 @@ odp_time_t odp_pktio_time(odp_pktio_t hdl, odp_time_t *global_ts) return ts; } -uint64_t ODP_DEPRECATE(odp_pktin_ts_res)(odp_pktio_t hdl) -{ - return odp_pktio_ts_res(hdl); -} - odp_time_t ODP_DEPRECATE(odp_pktin_ts_from_ns)(odp_pktio_t hdl, uint64_t ns) { return odp_pktio_ts_from_ns(hdl, ns); -- cgit v1.2.3 From 67e7bc21ded69243e636b949397591399d0f9a1f Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 19 Dec 2022 15:31:04 +0200 Subject: api: pktio: remove deprecated odp_pktin_ts_from_ns() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove deprecated odp_pktin_ts_from_ns() function. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- include/odp/api/spec/packet_io.h | 17 ----------------- include/odp/api/spec/packet_io_types.h | 1 - platform/linux-generic/odp_packet_io.c | 5 ----- 3 files changed, 23 deletions(-) diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h index a23f2d9ac..9a6117cee 100644 --- a/include/odp/api/spec/packet_io.h +++ b/include/odp/api/spec/packet_io.h @@ -20,7 +20,6 @@ extern "C" { #endif #include -#include #include #include #include @@ -832,22 +831,6 @@ int odp_pktio_info(odp_pktio_t pktio, odp_pktio_info_t *info); */ int odp_pktio_link_info(odp_pktio_t pktio, odp_pktio_link_info_t *info); -/** - * Convert nanoseconds to packet input time - * - * @deprecated Use odp_pktio_ts_from_ns() instead, which can be used with both - * packet input and output timestamps. - * - * Packet input time source is used for timestamping incoming packets. - * This function is used convert nanosecond time to packet input timestamp time. - * - * @param pktio Packet IO handle - * @param ns Time in nanoseconds - * - * @return Packet input timestamp - */ -odp_time_t ODP_DEPRECATE(odp_pktin_ts_from_ns)(odp_pktio_t pktio, uint64_t ns); - /** * Packet IO timestamp resolution in hertz * diff --git a/include/odp/api/spec/packet_io_types.h b/include/odp/api/spec/packet_io_types.h index fe86f6f12..a46668c30 100644 --- a/include/odp/api/spec/packet_io_types.h +++ b/include/odp/api/spec/packet_io_types.h @@ -19,7 +19,6 @@ extern "C" { #endif -#include #include #include #include diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index d3e51aa01..5a6c0f460 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -1613,11 +1613,6 @@ odp_time_t odp_pktio_time(odp_pktio_t hdl, odp_time_t *global_ts) return ts; } -odp_time_t ODP_DEPRECATE(odp_pktin_ts_from_ns)(odp_pktio_t hdl, uint64_t ns) -{ - return odp_pktio_ts_from_ns(hdl, ns); -} - void odp_pktio_print(odp_pktio_t hdl) { pktio_entry_t *entry; -- cgit v1.2.3 From 721e08c6f9cac1e03065e8f3653b616cf0261ca4 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 19 Dec 2022 15:34:47 +0200 Subject: api: shm: remove deprecated ODP_SHM_SW_ONLY define MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove deprecated ODP_SHM_SW_ONLY define. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- include/odp/api/spec/shared_memory.h | 13 ------------- platform/linux-generic/odp_shared_memory.c | 9 +-------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/include/odp/api/spec/shared_memory.h b/include/odp/api/spec/shared_memory.h index 36b38782f..eebf288c9 100644 --- a/include/odp/api/spec/shared_memory.h +++ b/include/odp/api/spec/shared_memory.h @@ -50,19 +50,6 @@ extern "C" { * Invalid physical address */ -/** - * Application SW only, no HW access - * - * @deprecated When set, application will not share the reserved memory with HW - * accelerators. However, leaving this flag to zero does not guarantee that - * the reserved memory can be accessed from HW, and thus usage of this flag is - * considered deprecated. If HW accessible memory is required, set - * ODP_SHM_HW_ACCESS instead. - * - * This flag must not be combined with ODP_SHM_HW_ACCESS. - */ -#define ODP_SHM_SW_ONLY 0x1 - /** * Share with external processes */ diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c index a0e822a53..ef4df3a33 100644 --- a/platform/linux-generic/odp_shared_memory.c +++ b/platform/linux-generic/odp_shared_memory.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -18,14 +17,8 @@ #include /* Supported ODP_SHM_* flags */ -#if ODP_DEPRECATED_API - #define DEPRECATED_SHM_FLAGS (ODP_SHM_SW_ONLY) -#else - #define DEPRECATED_SHM_FLAGS 0 -#endif - #define SUPPORTED_SHM_FLAGS (ODP_SHM_PROC | ODP_SHM_SINGLE_VA | ODP_SHM_EXPORT | \ - ODP_SHM_HP | ODP_SHM_NO_HP | DEPRECATED_SHM_FLAGS) + ODP_SHM_HP | ODP_SHM_NO_HP) static inline uint32_t from_handle(odp_shm_t shm) { -- cgit v1.2.3 From 59ba9ac933713ba6f4cb92d21880343793e662e2 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 19 Dec 2022 15:43:20 +0200 Subject: api: tm: remove deprecated odp_tm_capabilities() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove deprecated odp_tm_capabilities() function. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- include/odp/api/spec/traffic_mngr.h | 33 ------------------------------- platform/linux-generic/odp_traffic_mngr.c | 6 ------ 2 files changed, 39 deletions(-) diff --git a/include/odp/api/spec/traffic_mngr.h b/include/odp/api/spec/traffic_mngr.h index f634cfb44..a550bd28f 100644 --- a/include/odp/api/spec/traffic_mngr.h +++ b/include/odp/api/spec/traffic_mngr.h @@ -745,39 +745,6 @@ void odp_tm_requirements_init(odp_tm_requirements_t *requirements); */ void odp_tm_egress_init(odp_tm_egress_t *egress); -/** Query All TM Capabilities - * - * @deprecated Use odp_tm_egress_capabilities() instead that also additionally - * takes egress as input to provide capabilities specific to a given egress. - * - * This function returns the set of TM capabilities that are common for all - * egresses. The reason that this returns a SET of capabilities and not just - * one, is because it is expected that many HW based implementations may have - * one set of limits for the HW and also support a SW TM implementation with a - * (presumably larger) different set of limits. There are also cases where - * there could be more than one SW implementation (one supporting say tens of - * thousands of tm_queues and a variant supporting tens of millions of - * tm_queues). It returns capabilities that are valid for all egresses. - * The caller passes in an array of odp_tm_capabilities_t records and the - * maximum number of such records to output. If number of such records - * implementation supports is larger than caller requested number, then - * only caller requested number of records are written and return value is - * max number of records implementation supports. - * Caller then may again call with larger number of records to be returned. - * - * @param[out] capabilities An array of odp_tm_capabilities_t records to - * be filled in. - * @param capabilities_size The number of odp_tm_capabilities_t records - * in the capabilities array. - * @return Returns < 0 upon failure. Returns N > 0, - * where N is the maximum number of different - * odp_tm_capabilities_t records that the - * implementations supports. *NOTE* that this - * number can be > capabilities_size! - */ -int ODP_DEPRECATE(odp_tm_capabilities)(odp_tm_capabilities_t capabilities[], - uint32_t capabilities_size); - /** Query TM Capabilities specific to an egress * * The function returns the set of TM limits supported by this implementation diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 284a6f7f5..c8d8bc5c0 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -2665,12 +2665,6 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[], return 1; } -int ODP_DEPRECATE(odp_tm_capabilities)(odp_tm_capabilities_t capabilities[], - uint32_t capabilities_size) -{ - return tm_capabilities(capabilities, capabilities_size); -} - int odp_tm_egress_capabilities(odp_tm_capabilities_t *capabilities, const odp_tm_egress_t *egress) { -- cgit v1.2.3 From 6fdef020b8f15d4b0a5b7ff94255a8702fdada7d Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 19 Dec 2022 15:47:33 +0200 Subject: api: tm: remove deprecated commit_bps shaper parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove deprecated 'commit_bps' field from odp_tm_shaper_params_t. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- include/odp/api/spec/traffic_mngr.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/include/odp/api/spec/traffic_mngr.h b/include/odp/api/spec/traffic_mngr.h index a550bd28f..6f05d59f9 100644 --- a/include/odp/api/spec/traffic_mngr.h +++ b/include/odp/api/spec/traffic_mngr.h @@ -1061,11 +1061,7 @@ typedef struct { * for this integer is in bits per second when packet_mode is * not TRUE while packets per second when packet mode is TRUE. */ - union { - /** @deprecated Use commit_rate instead */ - uint64_t ODP_DEPRECATE(commit_bps); - uint64_t commit_rate; /**< Commit information rate */ - }; + uint64_t commit_rate; /** The peak information rate for this shaper profile. The units for * this integer is in bits per second when packet_mode is -- cgit v1.2.3 From a9e0425f9a0a25e5a173f06612ed31188ca005bf Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 19 Dec 2022 15:48:54 +0200 Subject: api: tm: remove deprecated peak_bps shaper parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove deprecated 'peak_bps' field from odp_tm_shaper_params_t. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- include/odp/api/spec/traffic_mngr.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/include/odp/api/spec/traffic_mngr.h b/include/odp/api/spec/traffic_mngr.h index 6f05d59f9..9bd735a7a 100644 --- a/include/odp/api/spec/traffic_mngr.h +++ b/include/odp/api/spec/traffic_mngr.h @@ -1068,11 +1068,7 @@ typedef struct { * not TRUE while in packets per second when packet mode is TRUE. * This field is ignored when dual_rate is FALSE. */ - union { - /** @deprecated Use peak_rate instead */ - uint64_t ODP_DEPRECATE(peak_bps); - uint64_t peak_rate; /**< Peak information rate */ - }; + uint64_t peak_rate; /** The commit burst tolerance for this shaper profile. The units for * this field is bits when packet_mode is not TRUE and packets when -- cgit v1.2.3 From d74fabe0960815c17a1746bbc9edc4378dc40fcb Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 22 Dec 2022 10:27:44 +0200 Subject: api: timer: use deprecate macro for odp_timer_set_rel() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use ODP_DEPRECATE() macro for odp_timer_set_rel() function. Previously, the deprecation was only mentioned in the function documentation. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- include/odp/api/spec/timer.h | 5 +++-- platform/linux-generic/odp_timer.c | 9 +++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/include/odp/api/spec/timer.h b/include/odp/api/spec/timer.h index edfa423e7..0e1f07f09 100644 --- a/include/odp/api/spec/timer.h +++ b/include/odp/api/spec/timer.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include #include #include #include @@ -387,7 +388,7 @@ int odp_timer_periodic_cancel(odp_timer_t timer); * * @deprecated Use odp_timer_start() or odp_timer_restart() instead */ -int odp_timer_set_abs(odp_timer_t timer, uint64_t abs_tick, odp_event_t *tmo_ev); +int ODP_DEPRECATE(odp_timer_set_abs)(odp_timer_t timer, uint64_t abs_tick, odp_event_t *tmo_ev); /** * Set (or reset) a timer with relative expiration time @@ -416,7 +417,7 @@ int odp_timer_set_abs(odp_timer_t timer, uint64_t abs_tick, odp_event_t *tmo_ev) * * @deprecated Use odp_timer_start() or odp_timer_restart() instead */ -int odp_timer_set_rel(odp_timer_t timer, uint64_t rel_tick, odp_event_t *tmo_ev); +int ODP_DEPRECATE(odp_timer_set_rel)(odp_timer_t timer, uint64_t rel_tick, odp_event_t *tmo_ev); /** * Cancel a timer diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 05660eb7a..8597c3f4e 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -1494,9 +1495,7 @@ odp_event_t odp_timer_free(odp_timer_t hdl) return timer_free(tp, idx); } -int odp_timer_set_abs(odp_timer_t hdl, - uint64_t abs_tck, - odp_event_t *tmo_ev) +int ODP_DEPRECATE(odp_timer_set_abs)(odp_timer_t hdl, uint64_t abs_tck, odp_event_t *tmo_ev) { timer_pool_t *tp = handle_to_tp(hdl); uint64_t cur_tick = current_nsec(tp); @@ -1512,9 +1511,7 @@ int odp_timer_set_abs(odp_timer_t hdl, return ODP_TIMER_FAIL; } -int odp_timer_set_rel(odp_timer_t hdl, - uint64_t rel_tck, - odp_event_t *tmo_ev) +int ODP_DEPRECATE(odp_timer_set_rel)(odp_timer_t hdl, uint64_t rel_tck, odp_event_t *tmo_ev) { timer_pool_t *tp = handle_to_tp(hdl); uint64_t cur_tick = current_nsec(tp); -- cgit v1.2.3 From 8139d878964e34dc62242a27c33ef8043d430922 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 23 Dec 2022 09:43:49 +0200 Subject: api: timer: deprecate old timer pool clock sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deprecate old timer pool clock sources ODP_CLOCK_CPU and ODP_CLOCK_EXT, which have been replaced by ODP_CLOCK_DEFAULT and ODP_CLOCK_SRC_1. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- include/odp/api/spec/timer_types.h | 19 ++++++++++++++----- test/validation/api/timer/timer.c | 2 ++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/include/odp/api/spec/timer_types.h b/include/odp/api/spec/timer_types.h index 754c831b2..bbc7ef8dc 100644 --- a/include/odp/api/spec/timer_types.h +++ b/include/odp/api/spec/timer_types.h @@ -20,8 +20,9 @@ extern "C" { #endif -#include +#include #include +#include /** @defgroup odp_timer ODP TIMER * Timer generating timeout events. @@ -245,13 +246,21 @@ typedef enum { /** The default clock source */ #define ODP_CLOCK_DEFAULT ODP_CLOCK_SRC_0 -/** For backwards compatibility, ODP_CLOCK_CPU is synonym of ODP_CLOCK_DEFAULT. - * This will be deprecated in the future. */ +#if ODP_DEPRECATED_API +/** + * For backwards compatibility, ODP_CLOCK_CPU is synonym of ODP_CLOCK_DEFAULT. + * + * @deprecated Use #ODP_CLOCK_DEFAULT instead. + */ #define ODP_CLOCK_CPU ODP_CLOCK_DEFAULT -/** For backwards compatibility, ODP_CLOCK_EXT is synonym of ODP_CLOCK_SRC_1. - * This will be deprecated in the future. */ +/** + * For backwards compatibility, ODP_CLOCK_EXT is synonym of ODP_CLOCK_SRC_1. + * + * @deprecated Use #ODP_CLOCK_SRC_1 instead. + */ #define ODP_CLOCK_EXT ODP_CLOCK_SRC_1 +#endif /** * Timer expiration mode diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index 5f57fdaae..0e3919b73 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -329,8 +329,10 @@ static void timer_test_capa(void) CU_ASSERT_FATAL(ODP_CLOCK_SRC_0 + 4 == ODP_CLOCK_SRC_4); CU_ASSERT_FATAL(ODP_CLOCK_SRC_0 + 5 == ODP_CLOCK_SRC_5); CU_ASSERT_FATAL(ODP_CLOCK_SRC_5 + 1 == ODP_CLOCK_NUM_SRC); +#if ODP_DEPRECATED_API CU_ASSERT_FATAL(ODP_CLOCK_CPU == ODP_CLOCK_DEFAULT); CU_ASSERT_FATAL(ODP_CLOCK_EXT == ODP_CLOCK_SRC_1); +#endif for (i = 0; i < ODP_CLOCK_NUM_SRC; i++) { clk_src = ODP_CLOCK_SRC_0 + i; -- cgit v1.2.3 From 0bc1fae4fa50777932582ea72cb0167bd4e17878 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 23 Dec 2022 09:50:33 +0200 Subject: api: timer: deprecate old timer set return values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deprecate old timer set return values ODP_TIMER_TOOEARLY, ODP_TIMER_TOOLATE, and ODP_TIMER_NOEVENT, which have been replaced by ODP_TIMER_TOO_NEAR, ODP_TIMER_TOO_FAR, and ODP_TIMER_FAIL. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- include/odp/api/spec/timer_types.h | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/include/odp/api/spec/timer_types.h b/include/odp/api/spec/timer_types.h index bbc7ef8dc..d61992fba 100644 --- a/include/odp/api/spec/timer_types.h +++ b/include/odp/api/spec/timer_types.h @@ -523,17 +523,28 @@ typedef enum { } odp_timer_set_t; -/** For backwards compatibility, ODP_TIMER_TOOEARLY is synonym of ODP_TIMER_TOO_NEAR. - * This will be deprecated in the future. */ +#if ODP_DEPRECATED_API +/** + * For backwards compatibility, ODP_TIMER_TOOEARLY is synonym of ODP_TIMER_TOO_NEAR. + * + * @deprecated Use #ODP_TIMER_TOO_NEAR instead. + */ #define ODP_TIMER_TOOEARLY ODP_TIMER_TOO_NEAR -/** For backwards compatibility, ODP_TIMER_TOOLATE is synonym of ODP_TIMER_TOO_FAR. - * This will be deprecated in the future. */ +/** + * For backwards compatibility, ODP_TIMER_TOOLATE is synonym of ODP_TIMER_TOO_FAR. + * + * @deprecated Use #ODP_TIMER_TOO_FAR instead. + */ #define ODP_TIMER_TOOLATE ODP_TIMER_TOO_FAR -/** For backwards compatibility, ODP_TIMER_NOEVENT is synonym of ODP_TIMER_FAIL. - * This will be deprecated in the future. */ +/** + * For backwards compatibility, ODP_TIMER_NOEVENT is synonym of ODP_TIMER_FAIL. + * + * @deprecated Use #ODP_TIMER_FAIL instead. + */ #define ODP_TIMER_NOEVENT ODP_TIMER_FAIL +#endif /** * Timer tick information -- cgit v1.2.3 From 03e204911bb9ad80d2f2eab56461a2d9efd54722 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 4 Jan 2023 15:16:30 +0200 Subject: api: packet: specify which packet flags cannot be set simultaneously MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Specify which packet metadata flags cannot be set simultaneously using odp_packet_has_XX_set() functions. Signed-off-by: Matias Elo Reviewed-by: Kiran Kumar K Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- include/odp/api/spec/packet_flags.h | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/include/odp/api/spec/packet_flags.h b/include/odp/api/spec/packet_flags.h index 98bf2b7bf..c6ff11054 100644 --- a/include/odp/api/spec/packet_flags.h +++ b/include/odp/api/spec/packet_flags.h @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -22,7 +23,13 @@ extern "C" { #include /** @addtogroup odp_packet - * Operations on packet metadata flags. + * @par Operations on packet metadata flags + * + * If user sets multiple conflicting packet metadata flags + * using odp_packet_has_XX_set() functions, only the last set flag value is + * guaranteed to hold. The values of other conflicting flags are implementation + * specific. The conflicting flag combinations are defined in function + * documentations. * @{ */ @@ -401,6 +408,8 @@ void odp_packet_has_jumbo_set(odp_packet_t pkt, int val); /** * Set flag for VLAN * + * Only one VLAN flag (VLAN/VLAN QinQ) can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -409,6 +418,8 @@ void odp_packet_has_vlan_set(odp_packet_t pkt, int val); /** * Set flag for VLAN QinQ (stacked VLAN) * + * Only one VLAN flag (VLAN/VLAN QinQ) can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -417,6 +428,8 @@ void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val); /** * Set flag for ARP * + * Only one of ARP/IPv4/IPv6 flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -425,6 +438,8 @@ void odp_packet_has_arp_set(odp_packet_t pkt, int val); /** * Set flag for IPv4 * + * Only one of ARP/IPv4/IPv6 flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -433,6 +448,8 @@ void odp_packet_has_ipv4_set(odp_packet_t pkt, int val); /** * Set flag for IPv6 * + * Only one of ARP/IPv4/IPv6 flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -441,6 +458,8 @@ void odp_packet_has_ipv6_set(odp_packet_t pkt, int val); /** * Set flag for IP broadcast address * + * Only one of IP broadcast/multicast flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -449,6 +468,8 @@ void odp_packet_has_ip_bcast_set(odp_packet_t pkt, int val); /** * Set flag for IP multicast address * + * Only one of IP broadcast/multicast flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -481,6 +502,8 @@ void odp_packet_has_ipsec_set(odp_packet_t pkt, int val); /** * Set flag for UDP * + * Only one of TCP/UDP/SCTP/ICMP flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -489,6 +512,8 @@ void odp_packet_has_udp_set(odp_packet_t pkt, int val); /** * Set flag for TCP * + * Only one of TCP/UDP/SCTP/ICMP flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -497,6 +522,8 @@ void odp_packet_has_tcp_set(odp_packet_t pkt, int val); /** * Set flag for SCTP * + * Only one of TCP/UDP/SCTP/ICMP flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ @@ -505,6 +532,8 @@ void odp_packet_has_sctp_set(odp_packet_t pkt, int val); /** * Set flag for ICMP * + * Only one of TCP/UDP/SCTP/ICMP flags can be set simultaneously. + * * @param pkt Packet handle * @param val Value */ -- cgit v1.2.3 From 495a9f770480cdbe9e6ed7b0bff6194b0f1f29dd Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 9 Jan 2023 13:44:53 +0200 Subject: api: packet: clarify packer has vlan and vlan q-in-q specifications MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clarify odp_packet_has_vlan() and odp_packet_has_vlan_qinq() specifications. Signed-off-by: Matias Elo Reviewed-by: Kiran Kumar K Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- include/odp/api/spec/packet_flags.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/include/odp/api/spec/packet_flags.h b/include/odp/api/spec/packet_flags.h index c6ff11054..c62e50885 100644 --- a/include/odp/api/spec/packet_flags.h +++ b/include/odp/api/spec/packet_flags.h @@ -179,6 +179,8 @@ int odp_packet_has_jumbo(odp_packet_t pkt); /** * Check for VLAN * + * Check if packet contains normal or QinQ VLAN header. + * * @param pkt Packet handle * * @retval non-zero Packet contains a VLAN header @@ -189,6 +191,8 @@ int odp_packet_has_vlan(odp_packet_t pkt); /** * Check for VLAN QinQ (stacked VLAN) * + * Check if packet contains QinQ VLAN header. + * * @param pkt Packet handle * * @retval non-zero Packet contains a VLAN QinQ header @@ -408,7 +412,8 @@ void odp_packet_has_jumbo_set(odp_packet_t pkt, int val); /** * Set flag for VLAN * - * Only one VLAN flag (VLAN/VLAN QinQ) can be set simultaneously. + * Set when packet contains normal VLAN header. Only one VLAN flag + * (VLAN/VLAN QinQ) can be set simultaneously. * * @param pkt Packet handle * @param val Value @@ -418,7 +423,8 @@ void odp_packet_has_vlan_set(odp_packet_t pkt, int val); /** * Set flag for VLAN QinQ (stacked VLAN) * - * Only one VLAN flag (VLAN/VLAN QinQ) can be set simultaneously. + * Set when packet contains QinQ VLAN header. Only one VLAN flag + * (VLAN/VLAN QinQ) can be set simultaneously. * * @param pkt Packet handle * @param val Value -- cgit v1.2.3 From a057dd7dbb5216a06ecbc0b286ecd88a1502148b Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 10 Jan 2023 15:31:17 +0200 Subject: linux-gen: packet: update packet vlan flag set function implementations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update packet VLAN/VLAN QinQ flag set functions to match the updated packet API. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- platform/linux-generic/odp_packet_flags.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c index ef542a2cd..777da12ae 100644 --- a/platform/linux-generic/odp_packet_flags.c +++ b/platform/linux-generic/odp_packet_flags.c @@ -72,10 +72,12 @@ void odp_packet_has_jumbo_set(odp_packet_t pkt, int val) void odp_packet_has_vlan_set(odp_packet_t pkt, int val) { setflag(pkt, input_flags.vlan, val); + setflag(pkt, input_flags.vlan_qinq, 0); } void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val) { + setflag(pkt, input_flags.vlan, val); setflag(pkt, input_flags.vlan_qinq, val); } -- cgit v1.2.3 From 8076020175dc801eb235ddf1f98c3b925ec13af8 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 9 Jan 2023 14:13:42 +0200 Subject: validation: cls: set packet has vlan flag correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow the new packet API specification and set either odp_packet_has_vlan() or odp_packet_has_vlan_qinq() flag, not both. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- test/validation/api/classification/odp_classification_common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/validation/api/classification/odp_classification_common.c b/test/validation/api/classification/odp_classification_common.c index 4c24099a5..fe7677044 100644 --- a/test/validation/api/classification/odp_classification_common.c +++ b/test/validation/api/classification/odp_classification_common.c @@ -399,10 +399,10 @@ odp_packet_t create_packet(cls_packet_info_t pkt_info) vlan_hdr->type = odp_cpu_to_be_16(ODPH_ETHTYPE_VLAN); vlan_hdr++; } else { + odp_packet_has_vlan_set(pkt, 1); ethhdr->type = odp_cpu_to_be_16(ODPH_ETHTYPE_VLAN); } /* Default vlan header */ - odp_packet_has_vlan_set(pkt, 1); vlan_hdr->tci = odp_cpu_to_be_16(0); vlan_hdr->type = odp_cpu_to_be_16(eth_type); } else { -- cgit v1.2.3 From cc6020bdb78b2d7a10508834378cb49b3eada4ce Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 9 Jan 2023 16:02:02 +0200 Subject: validation: packet: remove conflicting packet metadata flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop setting conflicting packet metadata flags. Signed-off-by: Matias Elo Reviewed-by: Kiran Kumar K Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- test/validation/api/packet/packet.c | 59 +++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index 545ecb5c0..36be808ba 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -329,7 +329,8 @@ static int packet_suite_term(void) return 0; } -static void packet_set_inflags(odp_packet_t pkt, int val) +/* Set all non-conflicting metadata flags */ +static void packet_set_inflags_common(odp_packet_t pkt, int val) { odp_packet_has_l2_set(pkt, val); odp_packet_has_l3_set(pkt, val); @@ -339,23 +340,37 @@ static void packet_set_inflags(odp_packet_t pkt, int val) odp_packet_has_eth_mcast_set(pkt, val); odp_packet_has_jumbo_set(pkt, val); odp_packet_has_vlan_set(pkt, val); - odp_packet_has_vlan_qinq_set(pkt, val); - odp_packet_has_arp_set(pkt, val); odp_packet_has_ipv4_set(pkt, val); - odp_packet_has_ipv6_set(pkt, val); odp_packet_has_ip_bcast_set(pkt, val); - odp_packet_has_ip_mcast_set(pkt, val); odp_packet_has_ipfrag_set(pkt, val); odp_packet_has_ipopt_set(pkt, val); odp_packet_has_ipsec_set(pkt, val); odp_packet_has_udp_set(pkt, val); - odp_packet_has_tcp_set(pkt, val); - odp_packet_has_sctp_set(pkt, val); - odp_packet_has_icmp_set(pkt, val); odp_packet_user_flag_set(pkt, val); } -static void packet_check_inflags(odp_packet_t pkt, int val) +/* Check all non-conflicting metadata flags */ +static void packet_check_inflags_common(odp_packet_t pkt, int val) +{ + CU_ASSERT(odp_packet_has_l2(pkt) == !!val); + CU_ASSERT(odp_packet_has_l3(pkt) == !!val); + CU_ASSERT(odp_packet_has_l4(pkt) == !!val); + CU_ASSERT(odp_packet_has_eth(pkt) == !!val); + CU_ASSERT(odp_packet_has_eth_bcast(pkt) == !!val); + CU_ASSERT(odp_packet_has_eth_mcast(pkt) == !!val); + CU_ASSERT(odp_packet_has_jumbo(pkt) == !!val); + CU_ASSERT(odp_packet_has_vlan(pkt) == !!val); + CU_ASSERT(odp_packet_has_ipv4(pkt) == !!val); + CU_ASSERT(odp_packet_has_ip_bcast(pkt) == !!val); + CU_ASSERT(odp_packet_has_ipfrag(pkt) == !!val); + CU_ASSERT(odp_packet_has_ipopt(pkt) == !!val); + CU_ASSERT(odp_packet_has_ipsec(pkt) == !!val); + CU_ASSERT(odp_packet_has_udp(pkt) == !!val); + CU_ASSERT(odp_packet_user_flag(pkt) == !!val); +} + +/* Check all metadata flags */ +static void packet_check_inflags_all(odp_packet_t pkt, int val) { CU_ASSERT(odp_packet_has_l2(pkt) == !!val); CU_ASSERT(odp_packet_has_l3(pkt) == !!val); @@ -418,7 +433,7 @@ static void packet_test_alloc_free(void) CU_ASSERT(odp_packet_user_ptr(packet) == NULL); /* Packet flags should be zero */ - packet_check_inflags(packet, 0); + packet_check_inflags_all(packet, 0); /* Pool should have only one packet */ CU_ASSERT_FATAL(odp_packet_alloc(pool, packet_len) @@ -937,10 +952,10 @@ static void packet_test_reset(void) CU_ASSERT(odp_packet_reset(pkt, len) == 0); CU_ASSERT(odp_packet_len(pkt) == len); - packet_set_inflags(pkt, 1); - packet_check_inflags(pkt, 1); + packet_set_inflags_common(pkt, 1); + packet_check_inflags_common(pkt, 1); CU_ASSERT(odp_packet_reset(pkt, len) == 0); - packet_check_inflags(pkt, 0); + packet_check_inflags_all(pkt, 0); CU_ASSERT(odp_packet_reset(pkt, len - 1) == 0); CU_ASSERT(odp_packet_len(pkt) == (len - 1)); @@ -1374,10 +1389,10 @@ static void packet_test_in_flags(void) { odp_packet_t pkt = test_packet; - packet_set_inflags(pkt, 0); - packet_check_inflags(pkt, 0); - packet_set_inflags(pkt, 1); - packet_check_inflags(pkt, 1); + packet_set_inflags_common(pkt, 0); + packet_check_inflags_common(pkt, 0); + packet_set_inflags_common(pkt, 1); + packet_check_inflags_common(pkt, 1); TEST_INFLAG(pkt, has_l2); TEST_INFLAG(pkt, has_l3); @@ -1402,8 +1417,8 @@ static void packet_test_in_flags(void) TEST_INFLAG(pkt, has_icmp); TEST_INFLAG(pkt, user_flag); - packet_set_inflags(pkt, 0); - packet_check_inflags(pkt, 0); + packet_set_inflags_common(pkt, 0); + packet_check_inflags_common(pkt, 0); } static void packet_test_error_flags(void) @@ -1628,15 +1643,15 @@ static void packet_test_meta_data_copy(void) pkt = odp_packet_alloc(pool, packet_len); CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); - packet_check_inflags(pkt, 0); + packet_check_inflags_all(pkt, 0); CU_ASSERT(odp_packet_input(pkt) == ODP_PKTIO_INVALID); CU_ASSERT(odp_packet_l3_offset(pkt) == ODP_PACKET_OFFSET_INVALID); CU_ASSERT(odp_packet_l4_offset(pkt) == ODP_PACKET_OFFSET_INVALID); CU_ASSERT(odp_packet_payload_offset(pkt) == ODP_PACKET_OFFSET_INVALID); - packet_set_inflags(pkt, 1); - packet_check_inflags(pkt, 1); + packet_set_inflags_common(pkt, 1); + packet_check_inflags_common(pkt, 1); odp_packet_input_set(pkt, pktio); odp_packet_user_ptr_set(pkt, (void *)(uintptr_t)0xdeadbeef); -- cgit v1.2.3 From 45198e57cdddb07987e2a9c4479f93230e30b20f Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 10 Jan 2023 17:01:00 +0200 Subject: validation: packet: add test for vlan flag functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add new tests for VLAN/VLAN QinQ input flags. Signed-off-by: Matias Elo Reviewed-by: Kiran Kumar K Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- test/validation/api/packet/packet.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index 36be808ba..a8b0e99f9 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -1421,6 +1421,30 @@ static void packet_test_in_flags(void) packet_check_inflags_common(pkt, 0); } +static void packet_test_vlan_flags(void) +{ + odp_packet_t pkt = test_packet; + + odp_packet_reset(pkt, odp_packet_len(test_packet)); + + CU_ASSERT(!odp_packet_has_vlan(pkt)); + CU_ASSERT(!odp_packet_has_vlan_qinq(pkt)); + + odp_packet_has_vlan_qinq_set(pkt, 1); + CU_ASSERT(odp_packet_has_vlan(pkt)); + CU_ASSERT(odp_packet_has_vlan_qinq(pkt)); + + odp_packet_has_vlan_qinq_set(pkt, 0); + CU_ASSERT(!odp_packet_has_vlan(pkt)); + CU_ASSERT(!odp_packet_has_vlan_qinq(pkt)); + + odp_packet_has_vlan_set(pkt, 1); + CU_ASSERT(odp_packet_has_vlan(pkt)); + CU_ASSERT(!odp_packet_has_vlan_qinq(pkt)); + + odp_packet_reset(pkt, odp_packet_len(test_packet)); +} + static void packet_test_error_flags(void) { odp_packet_t pkt = test_packet; @@ -4448,6 +4472,7 @@ odp_testinfo_t packet_suite[] = { ODP_TEST_INFO(packet_test_layer_offsets), ODP_TEST_INFO(packet_test_segment_last), ODP_TEST_INFO(packet_test_in_flags), + ODP_TEST_INFO(packet_test_vlan_flags), ODP_TEST_INFO(packet_test_error_flags), ODP_TEST_INFO(packet_test_add_rem_data), ODP_TEST_INFO(packet_test_meta_data_copy), -- cgit v1.2.3 From dbc552e8b45b645d428b7f274946538bbb449d04 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 24 Jan 2023 15:57:24 +0200 Subject: api: crypto: fix EIA2 IV length in API text EIA2 uses 64-bit IV, not 128-bit as mentioned in the current API text. Fix the remark about IV length and fix also a couple of small typos. Signed-off-by: Janne Peltonen Reviewed-by: Anoob Joseph --- include/odp/api/spec/crypto_types.h | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/include/odp/api/spec/crypto_types.h b/include/odp/api/spec/crypto_types.h index 01947c532..b581f451e 100644 --- a/include/odp/api/spec/crypto_types.h +++ b/include/odp/api/spec/crypto_types.h @@ -134,7 +134,7 @@ typedef enum { * * SNOW 3G-based UEA2 algorithm (also known as 128-EEA1). * - * IV (128 bit) should be formatted according to the ETSI/SAGE + * IV (128 bits) should be formatted according to the ETSI/SAGE * UEA2 & UIA2 specification: * COUNT || BEARER || DIRECTION || 0...0 || * COUNT || BEARER || DIRECTION || 0...0 || @@ -145,10 +145,9 @@ typedef enum { * * AES-CTR-based 128-EEA2 algorithm. * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * 128-EA2 & 128-EIA2 specification: - * COUNT || BEARER || - * DIRECTION || 0....0 + * IV (128 bits) should be formatted according to the ETSI/SAGE + * 128-EAA2 & 128-EIA2 specification: + * COUNT || BEARER || DIRECTION || 0....0 */ ODP_CIPHER_ALG_AES_EEA2, @@ -156,7 +155,7 @@ typedef enum { * * 128-EEA3/128-NEA3 algorithm when key length is 128 bits. * - * IV (128 bit) should be formatted according to the ETSI/SAGE + * IV (128 bits) should be formatted according to the ETSI/SAGE * 128-EEA3 & 128-EIA3 specification: * COUNT || BEARER || DIRECTION || 0...0 || * COUNT || BEARER || DIRECTION || 0...0 || @@ -292,7 +291,7 @@ typedef enum { /** Integrity UIA2 algorithm (128-EIA1) * * SNOW 3G-based UIA2 algorithm (also known as 128-EIA1). - * IV (128 bit) should be formatted according to the ETSI/SAGE + * IV (128 bits) should be formatted according to the ETSI/SAGE * UEA2 & UIA2 specification: * COUNT || FRESH || * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 || @@ -304,10 +303,9 @@ typedef enum { * * AES_CMAC-based 128-EIA2 algorithm. * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * 128-EA2 & 128-EIA2 specification: - * COUNT || BEARER || - * DIRECTION || 0....0 + * IV (64 bits) should be formatted according to the ETSI/SAGE + * 128-EEA2 & 128-EIA2 specification: + * COUNT || BEARER || DIRECTION || 0....0 */ ODP_AUTH_ALG_AES_EIA2, @@ -315,8 +313,8 @@ typedef enum { * * 128-EIA3/128-NIA3 algorithm when key length is 128 bits. * - * IV (128 bit) should be formatted according to the ETSI/SAGE - * 128-EA3 & 128-EIA2 specification: + * IV (128 bits) should be formatted according to the ETSI/SAGE + * 128-EEA3 & 128-EIA2 specification: * COUNT || BEARER || * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 || * BEARER || 0...0 || DIRECTION || 0...0 -- cgit v1.2.3 From ccdcd3cea974e4af3323c1bd59b1b3037f5e8ee8 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 16 Aug 2022 12:39:52 +0300 Subject: api: crypto: clarify that input packet is fully copied to output packet The semantics of the crypto op functions are somewhat ambiguous regarding how an explicitly given output packet is handled. In particular, the API does not state how the provided output packet affects the final output (e.g. whether some data is preserved). Clarify the API based on how the linux gen reference implementation works and on how the older odp_crypto_operation() function behaves: - All packet data and metadata are copied from the input packet to the output packet, even when an output packet is given explicitly (i.e. the old packet data and metadata of an explicitly given output packet are lost. - Memory layout (e.g. data pointers, segmentation) of the output packet may change during crypto operations. Signed-off-by: Janne Peltonen Reviewed-by: Anoob Joseph Reviewed-by: Petri Savolainen --- include/odp/api/spec/crypto.h | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index 2118cb589..ee84cd537 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -275,12 +275,27 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, * Crypto packet operation * * Performs the SYNC cryptographic operations specified during session creation - * on the packets. Caller should initialize pkt_out either with desired output - * packet handles or with ODP_PACKET_INVALID to make ODP allocate new packets - * from provided pool. All arrays should be of num_pkt size. + * on the packets. Caller should initialize each element of pkt_out either with + * the desired output packet handle or with ODP_PACKET_INVALID to make ODP + * allocate a new packet from provided pool. All arrays should be of num_pkt + * size. + * + * All packet data and metadata are copied from the input packet to the output + * packet before the requested crypto operation is performed to the output + * packet. If an output packet is given to the operation, it must be at least + * as long as the input packet and, in encode operations, long enough for the + * hash result to be fully inside the packet data. Memory layout of the output + * packet may change during the crypto operation. If the output packet is + * longer than needed, it is not truncated and the extra data bytes retain + * their content. + * + * It is ok to pass the same packet handle as both the input packet and the + * output packet for the same crypto operation. In that case the input packet + * is consumed but returned as the output packet (with possibly different + * memory layout). * * @param pkt_in Packets to be processed - * @param[in,out] pkt_out Packet handle array specifying resulting packets + * @param[in,out] pkt_out Packet handle array for resulting packets * @param param Operation parameters array * @param num_pkt Number of packets to be processed * @@ -295,14 +310,12 @@ int odp_crypto_op(const odp_packet_t pkt_in[], /** * Crypto packet operation * - * Performs the ASYNC cryptographic operations specified during session creation - * on the packets. Caller should initialize pkt_out either with desired output - * packet handles or with ODP_PACKET_INVALID to make ODP allocate new packets - * from provided pool. All arrays should be of num_pkt size. Resulting packets - * are returned through events. + * Performs the ASYNC cryptographic operations specified during session + * creation on the packets. Behaves otherwise like odp_crypto_op() but + * returns output packets through events. * * @param pkt_in Packets to be processed - * @param pkt_out Packet handle array specifying resulting packets + * @param pkt_out Packet handle array for resulting packets * @param param Operation parameters array * @param num_pkt Number of packets to be processed * -- cgit v1.2.3 From ff6a30c022ddbfafad92b7c1e56d597f3cbea0f2 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 16 Aug 2022 12:39:57 +0300 Subject: api: crypto: introduce a session parameter to control output packet handling Introduce new "operation type" session parameter that controls how the crypto operations interpret their parameters and handle the output packet. - Legacy type is the default and provides backward compatibility. - Basic type provides a slightly simplified interface Other types can be introduced later. In the basic operation type the pool session parameter must be set to ODP_POOL_INVALID. This is to enable backward compatibility for a possible future API change that would allow caller to optionally specify the pool of the output packet instead of having ODP use the pool of the input packet. Signed-off-by: Janne Peltonen Reviewed-by: Anoob Joseph Reviewed-by: Petri Savolainen --- include/odp/api/spec/crypto.h | 33 ++++++++++++++++++++++++---- include/odp/api/spec/crypto_types.h | 43 ++++++++++++++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index ee84cd537..674f476b0 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -156,6 +156,9 @@ void odp_crypto_compl_free(odp_crypto_compl_t completion_event); * * @deprecated Use odp_crypto_op() or odp_crypto_op_enq() instead. * + * This function may be called only for sessions configured with + * the ODP_CRYPTO_OP_TYPE_LEGACY operation type. + * * Performs the cryptographic operations specified during session creation * on the packet. If the operation is performed synchronously, "posted" * will return FALSE and the result of the operation is immediately available. @@ -275,10 +278,16 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, * Crypto packet operation * * Performs the SYNC cryptographic operations specified during session creation - * on the packets. Caller should initialize each element of pkt_out either with - * the desired output packet handle or with ODP_PACKET_INVALID to make ODP - * allocate a new packet from provided pool. All arrays should be of num_pkt - * size. + * on the packets. All arrays should be of num_pkt size. + * + * Use of the pkt_out parameter depends on the configured crypto operation + * type as described below. + * + * ODP_CRYPTO_OP_TYPE_LEGACY: + * + * Caller should initialize each element of pkt_out either with the desired + * output packet handle or with ODP_PACKET_INVALID to make ODP allocate a new + * packet from provided pool. * * All packet data and metadata are copied from the input packet to the output * packet before the requested crypto operation is performed to the output @@ -294,6 +303,19 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, * is consumed but returned as the output packet (with possibly different * memory layout). * + * ODP_CRYPTO_OP_TYPE_BASIC: + * + * ODP allocates the output packet from the pool from which the input + * packet was allocated. The processed input packet is consumed. All + * packet data and metadata are copied from the input packet to the output + * packet before the requested crypto operation is applied to the output + * packet. Memory layout (including packet data pointers, head and tail room, + * segmentation) of the output packet may differ from that of the input + * packet. + * + * The value of pktout[n] is ignored as pktout[n] is used purely as an + * output parameter that returns the handle of the newly allocated packet. + * * @param pkt_in Packets to be processed * @param[in,out] pkt_out Packet handle array for resulting packets * @param param Operation parameters array @@ -314,6 +336,9 @@ int odp_crypto_op(const odp_packet_t pkt_in[], * creation on the packets. Behaves otherwise like odp_crypto_op() but * returns output packets through events. * + * All arrays should be of num_pkt size, except that pkt_out parameter + * is ignored when the crypto operation type is ODP_CRYPTO_OP_TYPE_BASIC. + * * @param pkt_in Packets to be processed * @param pkt_out Packet handle array for resulting packets * @param param Operation parameters array diff --git a/include/odp/api/spec/crypto_types.h b/include/odp/api/spec/crypto_types.h index b581f451e..205bdb700 100644 --- a/include/odp/api/spec/crypto_types.h +++ b/include/odp/api/spec/crypto_types.h @@ -53,7 +53,7 @@ typedef enum { } odp_crypto_op_mode_t; /** - * Crypto API operation type + * Crypto API operation */ typedef enum { /** Encrypt and/or compute authentication ICV */ @@ -509,6 +509,28 @@ typedef struct odp_crypto_key { } odp_crypto_key_t; +/** + * Type of odp_crypto_op()/odp_crypto_op_enq() calls. + */ +typedef enum odp_crypto_op_type_t { + /** + * Input packet data and metadata are copied in the output packet + * and then processed. Output packet is allocated by the caller + * or by ODP. odp_crypto_op(), odp_crypto_op_enq() and + * odp_crypto_operation() can be used. + * + * This is the default value but will be deprecated in the future. + */ + ODP_CRYPTO_OP_TYPE_LEGACY, + + /** + * Input packet data and metadata are copied in the output packet + * and then processed. Output packet is allocated by ODP. + * odp_crypto_op() and odp_crypto_op_enq() can be used. + */ + ODP_CRYPTO_OP_TYPE_BASIC, +} odp_crypto_op_type_t; + /** * Crypto API session creation parameters */ @@ -519,6 +541,18 @@ typedef struct odp_crypto_session_param_t { */ odp_crypto_op_t op; + /** Crypto operation type + * + * This field defines how the crypto operation functions are + * to be called and what they return. In particular, this field + * specifies the interpretation of the output packet parameter, + * how output packets are allocated and what data and metadata + * they contain. + * + * The default value is ODP_CRYPTO_OP_TYPE_LEGACY. + */ + odp_crypto_op_type_t op_type; + /** Authenticate cipher vs. plain text * * Controls ordering of authentication and cipher operations, @@ -632,8 +666,11 @@ typedef struct odp_crypto_session_param_t { /** Output pool * * When the output packet is not specified during the call to - * crypto operation, the output packet will be allocated - * from this pool. + * crypto operation in the legacy operation type, the output + * packet will be allocated from this pool. + * + * In ODP_CRYPTO_OP_TYPE_BASIC operation type this must be set to + * ODP_POOL_INVALID. */ odp_pool_t output_pool; -- cgit v1.2.3 From f2131a81ad507d06a5152dc3b6fbeeeb96a98979 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Wed, 14 Sep 2022 11:25:15 +0300 Subject: api: crypto: add an out-of-place type operation type Add ODP_CRYPTO_OP_TYPE_OP that writes the output of the crypto operation into a caller provided output packet and does not consume the input packet. Signed-off-by: Janne Peltonen Reviewed-by: Anoob Joseph Reviewed-by: Petri Savolainen --- include/odp/api/spec/crypto.h | 34 +++++++++++++++++++++- include/odp/api/spec/crypto_types.h | 56 ++++++++++++++++++++++++++++++++----- 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index 674f476b0..05ab514a0 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -316,12 +316,39 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, * The value of pktout[n] is ignored as pktout[n] is used purely as an * output parameter that returns the handle of the newly allocated packet. * + * ODP_CRYPTO_OP_TYPE_OOP: + * + * Writes the output bytes of the crypto operation in a caller provided + * output packet passed through pkt_out[n]. Input packets are not consumed + * nor modified. Memory layout (including packet data pointers, head and + * tail room, segmentation) of the output packet may change during the + * operation. + * + * Crypto output is the processed crypto_range, auth_range and + * MAC/digest (in encode sessions) of the input packet. The operation + * behaves as if crypto range and auth range were first copied from the + * input packet to the output packet and then the crypto operation + * was applied to the output packet. + * + * Crypto range and auth range of null cipher and auth algorithms are + * ignored, i.e. not copied in the output packet. Auth range of (AEAD) + * algorithms that ignore auth range is not copied. + * + * The offset of the crypto range and auth range in the output packet is + * the same as in the input packet, adjusted by dst_offset_shift operation + * parameter. + * + * pkt_out[n] must be a valid handle to a packet that is long enough to + * contain the shifted crypto range, auth range and, in encode sessions, + * the MAC/digest result. pkt_out[n] must not be the same as any input + * packet or any other output packet. + * * @param pkt_in Packets to be processed * @param[in,out] pkt_out Packet handle array for resulting packets * @param param Operation parameters array * @param num_pkt Number of packets to be processed * - * @return Number of input packets consumed (0 ... num_pkt) + * @return Number of input packets processed (0 ... num_pkt) * @retval <0 on failure */ int odp_crypto_op(const odp_packet_t pkt_in[], @@ -336,6 +363,11 @@ int odp_crypto_op(const odp_packet_t pkt_in[], * creation on the packets. Behaves otherwise like odp_crypto_op() but * returns output packets through events. * + * With ODP_CRYPTO_OP_TYPE_OOP, an enqueued input packet is consumed but + * returned back unmodified after the crypto operation is complete. The + * caller may not access the input packet until getting the handle back + * through odp_crypto_result(). + * * All arrays should be of num_pkt size, except that pkt_out parameter * is ignored when the crypto operation type is ODP_CRYPTO_OP_TYPE_BASIC. * diff --git a/include/odp/api/spec/crypto_types.h b/include/odp/api/spec/crypto_types.h index 205bdb700..574f56445 100644 --- a/include/odp/api/spec/crypto_types.h +++ b/include/odp/api/spec/crypto_types.h @@ -529,6 +529,22 @@ typedef enum odp_crypto_op_type_t { * odp_crypto_op() and odp_crypto_op_enq() can be used. */ ODP_CRYPTO_OP_TYPE_BASIC, + + /** + * Out-of-place crypto operation. Output packet is provided by + * the caller and the input packet is not consumed nor modified. + * + * Output of the crypto operation is written in the caller provided + * output packet without affecting other data and metadata of the + * output packet. Memory layout of the output packet may change + * during the operation. + * + * Crypto output is the processed crypto_range, auth_range and + * MAC/digest (in encode sessions) of the input packet. + * + * odp_crypto_op() and odp_crypto_op_enq() can be used. + */ + ODP_CRYPTO_OP_TYPE_OOP, } odp_crypto_op_type_t; /** @@ -669,8 +685,8 @@ typedef struct odp_crypto_session_param_t { * crypto operation in the legacy operation type, the output * packet will be allocated from this pool. * - * In ODP_CRYPTO_OP_TYPE_BASIC operation type this must be set to - * ODP_POOL_INVALID. + * In ODP_CRYPTO_OP_TYPE_BASIC and ODP_CRYPTO_OP_TYPE_OOP + * operation types this must be set to ODP_POOL_INVALID. */ odp_pool_t output_pool; @@ -769,16 +785,23 @@ typedef struct odp_crypto_packet_op_param_t { /** Offset from start of packet for hash result * * In case of decode sessions, the expected hash will be read from - * this offset and compared with the calculated hash. After the - * operation the hash bytes will have undefined values. + * this offset from the input packet and compared with the calculated + * hash. After the operation the hash bytes will have undefined + * values except with out-of-place sessions (ODP_CRYPTO_OP_TYPE_OOP + * operation type). + * + * With out-of-place decode sessions the input packet is not modified + * but if the hash location overlaps the cipher range or the auth + * range, then the corresponding location in the output packet will + * have undefined content. * * In case of encode sessions the calculated hash will be stored in - * this offset. + * this offset in the output packet. * * If the hash_result_in_auth_range session parameter is true, * the hash result location may overlap auth_range. In that case the - * result location will be zeroed in decode sessions before hash - * calculation. Zeroing is not done in encode sessions. + * result location will be treated as containing zero bytes for the + * purpose of hash calculation in decode sessions. */ uint32_t hash_result_offset; @@ -793,6 +816,19 @@ typedef struct odp_crypto_packet_op_param_t { /** Data range to authenticate */ odp_packet_data_range_t auth_range; + /** Shift of the output offsets with ODP_CRYPTO_OP_TYPE_OOP + * + * The processed crypto range and auth range of the input packet + * will be written in the output packet at the offset specified + * in the ranges (i.e. the same as in the input packet), shifted + * by this many bytes. This allows directing the output to + * a different packet offset than the offset of the input data. + * + * This is ignored if the crypto operation type is not + * ODP_CRYPTO_OP_TYPE_OOP. + */ + int32_t dst_offset_shift; + } odp_crypto_packet_op_param_t; /** @@ -914,6 +950,12 @@ typedef struct odp_crypto_packet_result_t { /** Request completed successfully */ odp_bool_t ok; + /** Input packet passed to odp_crypo_op_enq() when the operation + * type of the session is ODP_CRYPTO_OP_TYPE_OOP. In other cases + * this field does not have a valid value. + */ + odp_packet_t pkt_in; + /** Cipher status */ odp_crypto_op_status_t cipher_status; -- cgit v1.2.3 From 0a314d35e6c0ae6c60e6c8f819440f4e5c232bb1 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Wed, 4 Jan 2023 14:07:56 +0200 Subject: api: crypto: allow post-processing in odp_crypto_result() Require that odp_crypto_result() is called before packet data of async processed packets can be assumed to be valid. This makes it possible for an ODP implementation to postpone part of crypto processing to odp_crypto_result() after completion event delivery. For backward compatibility, do not change the semantics of the legacy operation mode. I.e. this change applies only to the newly introduced operation modes. Signed-off-by: Janne Peltonen Reviewed-by: Anoob Joseph Reviewed-by: Petri Savolainen --- include/odp/api/spec/crypto.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index 05ab514a0..4292b78c1 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -257,15 +257,15 @@ odp_packet_t odp_crypto_packet_from_event(odp_event_t ev); odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt); /** - * Get crypto operation results from an crypto processed packet + * Get crypto operation results from a crypto processed packet * * Successful crypto operations of all types (SYNC and ASYNC) produce packets * which contain crypto result metadata. This function copies the operation - * results from an crypto processed packet. Event subtype of this kind of + * results from a crypto processed packet. Event subtype of this kind of * packet is ODP_EVENT_PACKET_CRYPTO. Results are undefined if a non-crypto * processed packet is passed as input. * - * @param packet An crypto processed packet (ODP_EVENT_PACKET_CRYPTO) + * @param packet A crypto processed packet (ODP_EVENT_PACKET_CRYPTO) * @param[out] result Pointer to operation result for output * * @retval 0 On success @@ -363,6 +363,10 @@ int odp_crypto_op(const odp_packet_t pkt_in[], * creation on the packets. Behaves otherwise like odp_crypto_op() but * returns output packets through events. * + * With operation types other than ODP_CRYPTO_OP_TYPE_LEGACY, packet + * data of processed packets may not be valid before odp_crypto_result() + * has been called. + * * With ODP_CRYPTO_OP_TYPE_OOP, an enqueued input packet is consumed but * returned back unmodified after the crypto operation is complete. The * caller may not access the input packet until getting the handle back -- cgit v1.2.3 From 6ef048aceded7c06a568834dc6f8fb0ac2b6c505 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Mon, 30 Jan 2023 20:21:25 +0200 Subject: linux-gen: crypto: null: use new session creation error names Stop using the old session creation error names slated for deprecation. Signed-off-by: Janne Peltonen Reviewed-by: Petri Savolainen Reviewed-by: Anoob Joseph --- platform/linux-generic/odp_crypto_null.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index 50d4b5c6b..88ca2b8ce 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -201,7 +201,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, _ODP_ERR("Crypto is disabled\n"); /* Dummy output to avoid compiler warning about uninitialized * variables */ - *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *status = ODP_CRYPTO_SES_ERR_ENOMEM; *session_out = ODP_CRYPTO_SESSION_INVALID; return -1; } @@ -209,7 +209,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Allocate memory for this session */ session = alloc_session(); if (NULL == session) { - *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM; + *status = ODP_CRYPTO_SES_ERR_ENOMEM; goto err; } @@ -227,7 +227,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Check result */ if (rc) { - *status = ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER; + *status = ODP_CRYPTO_SES_ERR_CIPHER; goto err; } @@ -242,13 +242,13 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Check result */ if (rc) { - *status = ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH; + *status = ODP_CRYPTO_SES_ERR_AUTH; goto err; } /* We're happy */ *session_out = (intptr_t)session; - *status = ODP_CRYPTO_SES_CREATE_ERR_NONE; + *status = ODP_CRYPTO_SES_ERR_NONE; return 0; err: -- cgit v1.2.3 From 2280e91a1273f9e37e06054a5db5c3053eb2ec85 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Mon, 30 Jan 2023 20:21:26 +0200 Subject: linux-gen: crypto: implement basic crypto operation type Add support for ODP_CRYPTO_OP_TYPE_BASIC operation type. Signed-off-by: Janne Peltonen Reviewed-by: Petri Savolainen Reviewed-by: Anoob Joseph --- platform/linux-generic/arch/aarch64/odp_crypto_armv8.c | 13 +++++++++++-- platform/linux-generic/odp_crypto_ipsecmb.c | 13 +++++++++++-- platform/linux-generic/odp_crypto_null.c | 13 +++++++++++-- platform/linux-generic/odp_crypto_openssl.c | 13 +++++++++++-- 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index 6e266e0c8..0754cfdd5 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -1,6 +1,6 @@ /* Copyright (c) 2014-2018, Linaro Limited * Copyright (c) 2021, ARM Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -667,6 +667,10 @@ odp_crypto_operation(odp_crypto_op_param_t *param, odp_crypto_op_result_t local_result; int rc; + if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type != + ODP_CRYPTO_OP_TYPE_LEGACY) + return -1; + packet_param.session = param->session; packet_param.cipher_iv_ptr = param->cipher_iv_ptr; packet_param.auth_iv_ptr = param->auth_iv_ptr; @@ -867,6 +871,9 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio { int rc; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) + return pkt_in; + if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -951,7 +958,9 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); - pkt = pkt_out[i]; + if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC) + pkt = pkt_out[i]; + rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); if (rc < 0) break; diff --git a/platform/linux-generic/odp_crypto_ipsecmb.c b/platform/linux-generic/odp_crypto_ipsecmb.c index 3532be908..27367d063 100644 --- a/platform/linux-generic/odp_crypto_ipsecmb.c +++ b/platform/linux-generic/odp_crypto_ipsecmb.c @@ -1,6 +1,6 @@ /* Copyright (c) 2014-2018, Linaro Limited * Copyright (c) 2021, ARM Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -537,6 +537,10 @@ odp_crypto_operation(odp_crypto_op_param_t *param, odp_crypto_op_result_t local_result; int rc; + if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type != + ODP_CRYPTO_OP_TYPE_LEGACY) + return -1; + packet_param.session = param->session; packet_param.cipher_iv_ptr = param->cipher_iv_ptr; packet_param.auth_iv_ptr = param->auth_iv_ptr; @@ -749,6 +753,9 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio { int rc; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) + return pkt_in; + if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -850,7 +857,9 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); - pkt = pkt_out[i]; + if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC) + pkt = pkt_out[i]; + rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); if (rc < 0) break; diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index 88ca2b8ce..8a1e2ff03 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -281,6 +281,10 @@ odp_crypto_operation(odp_crypto_op_param_t *param, odp_crypto_op_result_t local_result; int rc; + if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type != + ODP_CRYPTO_OP_TYPE_LEGACY) + return -1; + packet_param.session = param->session; packet_param.cipher_iv_ptr = param->cipher_iv_ptr; packet_param.auth_iv_ptr = param->auth_iv_ptr; @@ -477,6 +481,9 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio { int rc; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) + return pkt_in; + if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -566,7 +573,9 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); - pkt = pkt_out[i]; + if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC) + pkt = pkt_out[i]; + rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); if (rc < 0) break; diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index f629ddf67..abfab8612 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -2345,6 +2345,10 @@ odp_crypto_operation(odp_crypto_op_param_t *param, odp_crypto_op_result_t local_result; int rc; + if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type != + ODP_CRYPTO_OP_TYPE_LEGACY) + return -1; + packet_param.session = param->session; packet_param.cipher_iv_ptr = param->cipher_iv_ptr; packet_param.auth_iv_ptr = param->auth_iv_ptr; @@ -2621,6 +2625,9 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio { int rc; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) + return pkt_in; + if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -2739,7 +2746,9 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); - pkt = pkt_out[i]; + if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC) + pkt = pkt_out[i]; + rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); if (rc < 0) break; -- cgit v1.2.3 From 5054ec2cad030c0a41bf421869c56f921f42cf65 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Mon, 30 Jan 2023 20:21:28 +0200 Subject: linux-gen: crypto: implement out-of-place crypto operation type Add support for ODP_CRYPTO_OP_TYPE_OOP operation type. In openssl crypto, implement out-of-place operations as combinations of packet data copies and in-place operations. This is done for simplicity and to ensure correct semantics for API validation test development. In other crypto modules, do not support out-of-place operations yet, but fail gracefully by returning ODP_CRYPTO_SES_ERR_PARAMS error in session creation if creation of an out-of-place session is attempted. Signed-off-by: Janne Peltonen Reviewed-by: Petri Savolainen Reviewed-by: Anoob Joseph --- .../linux-generic/arch/aarch64/odp_crypto_armv8.c | 6 + platform/linux-generic/odp_crypto_ipsecmb.c | 6 + platform/linux-generic/odp_crypto_null.c | 6 + platform/linux-generic/odp_crypto_openssl.c | 130 ++++++++++++++++++++- 4 files changed, 146 insertions(+), 2 deletions(-) diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index 0754cfdd5..67ae6a389 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -528,6 +528,12 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, return -1; } + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + *status = ODP_CRYPTO_SES_ERR_PARAMS; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + /* Allocate memory for this session */ session = alloc_session(); if (NULL == session) { diff --git a/platform/linux-generic/odp_crypto_ipsecmb.c b/platform/linux-generic/odp_crypto_ipsecmb.c index 27367d063..a7ce6077a 100644 --- a/platform/linux-generic/odp_crypto_ipsecmb.c +++ b/platform/linux-generic/odp_crypto_ipsecmb.c @@ -453,6 +453,12 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, return -1; } + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + *status = ODP_CRYPTO_SES_ERR_PARAMS; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + session = alloc_session(); if (NULL == session) { *status = ODP_CRYPTO_SES_ERR_ENOMEM; diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index 8a1e2ff03..3ca27ca00 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -206,6 +206,12 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, return -1; } + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + *status = ODP_CRYPTO_SES_ERR_PARAMS; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + /* Allocate memory for this session */ session = alloc_session(); if (NULL == session) { diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index abfab8612..19925fc09 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -217,6 +217,9 @@ struct odp_crypto_generic_session_t { odp_crypto_session_param_t p; odp_bool_t do_cipher_first; + uint8_t cipher_bit_mode : 1; + uint8_t cipher_range_used : 1; + uint8_t auth_range_used : 1; struct { uint8_t key_data[EVP_MAX_KEY_LENGTH]; @@ -1196,6 +1199,7 @@ static int process_cipher_param_bits(odp_crypto_generic_session_t *session, session->p.cipher_iv_len) return -1; + session->cipher_bit_mode = 1; session->cipher.evp_cipher = cipher; memcpy(session->cipher.key_data, session->p.cipher_key.data, @@ -2048,6 +2052,10 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, /* Copy parameters */ session->p = *param; + session->cipher_bit_mode = 0; + session->auth_range_used = 1; + session->cipher_range_used = 1; + if (session->p.cipher_iv_len > EVP_MAX_IV_LENGTH) { _ODP_DBG("Maximum IV length exceeded\n"); *status = ODP_CRYPTO_SES_ERR_CIPHER; @@ -2071,6 +2079,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, case ODP_CIPHER_ALG_NULL: session->cipher.func = null_crypto_routine; session->cipher.init = null_crypto_init_routine; + session->cipher_range_used = 0; rc = 0; break; case ODP_CIPHER_ALG_3DES_CBC: @@ -2191,6 +2200,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, case ODP_AUTH_ALG_NULL: session->auth.func = null_crypto_routine; session->auth.init = null_crypto_init_routine; + session->auth_range_used = 0; rc = 0; break; case ODP_AUTH_ALG_MD5_HMAC: @@ -2224,6 +2234,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, } else { rc = -1; } + session->auth_range_used = 0; break; case ODP_AUTH_ALG_AES_GMAC: if (param->auth_key.length == 16) @@ -2245,6 +2256,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, } else { rc = -1; } + session->auth_range_used = 0; break; case ODP_AUTH_ALG_AES_CMAC: if (param->auth_key.length == 16) @@ -2270,6 +2282,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, } else { rc = -1; } + session->auth_range_used = 0; break; #endif case ODP_AUTH_ALG_AES_EIA2: @@ -2711,6 +2724,113 @@ out: return 0; } +/* + * Copy cipher range and auth range from src to dst, + * with shifting by dst_offset_shift. + */ +static void copy_ranges(odp_packet_t dst, + odp_packet_t src, + const odp_crypto_generic_session_t *session, + const odp_crypto_packet_op_param_t *param) +{ + odp_packet_data_range_t c_range = param->cipher_range; + odp_packet_data_range_t a_range = param->auth_range; + int32_t shift = param->dst_offset_shift; + int rc; + + if (session->cipher_bit_mode) { + c_range.offset /= 8; + c_range.length = (c_range.length + 7) / 8; + } + + if (session->cipher_range_used) { + rc = odp_packet_copy_from_pkt(dst, c_range.offset + shift, + src, c_range.offset, + c_range.length); + if (rc) { + _ODP_ERR("cipher range copying failed\n"); + return; + } + } + if (session->auth_range_used) { + rc = odp_packet_copy_from_pkt(dst, a_range.offset + shift, + src, a_range.offset, + a_range.length); + if (rc) { + _ODP_ERR("auth range copying failed\n"); + return; + } + } +} + +static int crypto_int_oop_encode(odp_packet_t pkt_in, + odp_packet_t *pkt_out, + const odp_crypto_generic_session_t *session, + const odp_crypto_packet_op_param_t *param) +{ + odp_crypto_packet_op_param_t new_param = *param; + const uint32_t scale = session->cipher_bit_mode ? 8 : 1; + + copy_ranges(*pkt_out, pkt_in, session, param); + + new_param.cipher_range.offset += param->dst_offset_shift * scale; + new_param.auth_range.offset += param->dst_offset_shift; + + return crypto_int(*pkt_out, pkt_out, &new_param); +} + +static int crypto_int_oop_decode(odp_packet_t pkt_in, + odp_packet_t *pkt_out, + const odp_crypto_generic_session_t *session, + const odp_crypto_packet_op_param_t *param) +{ + odp_packet_t copy; + int rc; + + copy = odp_packet_copy(pkt_in, odp_packet_pool(pkt_in)); + if (copy == ODP_PACKET_INVALID) + return -1; + + rc = crypto_int(copy, ©, param); + if (rc < 0) { + odp_packet_free(copy); + return rc; + } + + copy_ranges(*pkt_out, copy, session, param); + + packet_subtype_set(*pkt_out, ODP_EVENT_PACKET_CRYPTO); + packet_hdr(*pkt_out)->crypto_op_result = packet_hdr(copy)->crypto_op_result; + odp_packet_free(copy); + + return 0; +} + +/* + * Slow out-of-place operation implemented using copying and in-place operation + */ +static int crypto_int_oop(odp_packet_t pkt_in, + odp_packet_t *pkt_out, + const odp_crypto_packet_op_param_t *param) +{ + odp_crypto_generic_session_t *session; + int rc; + + session = (odp_crypto_generic_session_t *)(intptr_t)param->session; + + if (session->p.op == ODP_CRYPTO_OP_ENCODE) + rc = crypto_int_oop_encode(pkt_in, pkt_out, session, param); + else + rc = crypto_int_oop_decode(pkt_in, pkt_out, session, param); + if (rc) + return rc; + + if (session->p.op_mode == ODP_CRYPTO_ASYNC) + packet_hdr(*pkt_out)->crypto_op_result.pkt_in = pkt_in; + + return 0; +} + int odp_crypto_op(const odp_packet_t pkt_in[], odp_packet_t pkt_out[], const odp_crypto_packet_op_param_t param[], @@ -2723,7 +2843,10 @@ int odp_crypto_op(const odp_packet_t pkt_in[], session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; _ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); - rc = crypto_int(pkt_in[i], &pkt_out[i], ¶m[i]); + if (odp_unlikely(session->p.op_type == ODP_CRYPTO_OP_TYPE_OOP)) + rc = crypto_int_oop(pkt_in[i], &pkt_out[i], ¶m[i]); + else + rc = crypto_int(pkt_in[i], &pkt_out[i], ¶m[i]); if (rc < 0) break; } @@ -2749,7 +2872,10 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC) pkt = pkt_out[i]; - rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); + if (odp_unlikely(session->p.op_type == ODP_CRYPTO_OP_TYPE_OOP)) + rc = crypto_int_oop(pkt_in[i], &pkt, ¶m[i]); + else + rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); if (rc < 0) break; -- cgit v1.2.3 From 24ebb09302f738a25dd3043650b3de34f8af4162 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Wed, 1 Feb 2023 19:54:32 +0200 Subject: validation: crypto: add ODP_CRYPTO_OP_TYPE_BASIC testing Add support for the operation type parameter and add testing of ODP_CRYPTO_OP_TYPE_BASIC. Signed-off-by: Janne Peltonen Reviewed-by: Anoob Joseph --- test/validation/api/crypto/odp_crypto_test_inp.c | 123 +++++++++++++++++------ 1 file changed, 94 insertions(+), 29 deletions(-) diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index 802a419e2..acbf6779d 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -13,6 +13,7 @@ #define PKT_POOL_NUM 64 #define PKT_POOL_LEN (1 * 1024) +#define UAREA_SIZE 8 #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) @@ -36,6 +37,7 @@ static void test_defaults(uint8_t fill) odp_crypto_session_param_init(¶m); CU_ASSERT_EQUAL(param.op, ODP_CRYPTO_OP_ENCODE); + CU_ASSERT_EQUAL(param.op_type, ODP_CRYPTO_OP_TYPE_LEGACY); CU_ASSERT_EQUAL(param.auth_cipher_text, false); #if ODP_DEPRECATED_API CU_ASSERT_EQUAL(param.pref_mode, ODP_CRYPTO_SYNC); @@ -277,9 +279,11 @@ static int alg_op(odp_packet_t pkt, } #endif -static int alg_packet_op(odp_packet_t pkt, +static int alg_packet_op(odp_packet_t pkt_in, + odp_packet_t *pkt_out, odp_bool_t *ok, odp_crypto_session_t session, + odp_crypto_op_type_t op_type, uint8_t *cipher_iv_ptr, uint8_t *auth_iv_ptr, odp_packet_data_range_t *cipher_range, @@ -292,7 +296,11 @@ static int alg_packet_op(odp_packet_t pkt, odp_crypto_packet_result_t result; odp_crypto_packet_op_param_t op_params; odp_event_subtype_t subtype; - odp_packet_t out_pkt = pkt; + + if (op_type == ODP_CRYPTO_OP_TYPE_LEGACY) + *pkt_out = pkt_in; + else + *pkt_out = ODP_PACKET_INVALID; /* Prepare input/output params */ memset(&op_params, 0, sizeof(op_params)); @@ -310,13 +318,18 @@ static int alg_packet_op(odp_packet_t pkt, op_params.hash_result_offset = hash_result_offset; if (suite_context.op_mode == ODP_CRYPTO_SYNC) { - rc = odp_crypto_op(&pkt, &out_pkt, &op_params, 1); + rc = odp_crypto_op(&pkt_in, pkt_out, &op_params, 1); if (rc <= 0) { CU_FAIL("Failed odp_crypto_packet_op()"); return rc; } } else { - rc = odp_crypto_op_enq(&pkt, &pkt, &op_params, 1); + odp_packet_t *out_param = pkt_out; + + if (op_type == ODP_CRYPTO_OP_TYPE_BASIC) + out_param = NULL; + + rc = odp_crypto_op_enq(&pkt_in, out_param, &op_params, 1); if (rc <= 0) { CU_FAIL("Failed odp_crypto_op_enq()"); return rc; @@ -333,33 +346,37 @@ static int alg_packet_op(odp_packet_t pkt, CU_ASSERT(ODP_EVENT_PACKET == odp_event_types(event, &subtype)); CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == subtype); - pkt = odp_crypto_packet_from_event(event); + *pkt_out = odp_crypto_packet_from_event(event); } - CU_ASSERT(out_pkt == pkt); + if (op_type != ODP_CRYPTO_OP_TYPE_BASIC) + CU_ASSERT(*pkt_out == pkt_in); CU_ASSERT(ODP_EVENT_PACKET == - odp_event_type(odp_packet_to_event(pkt))); + odp_event_type(odp_packet_to_event(*pkt_out))); CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == - odp_event_subtype(odp_packet_to_event(pkt))); + odp_event_subtype(odp_packet_to_event(*pkt_out))); CU_ASSERT(ODP_EVENT_PACKET == - odp_event_types(odp_packet_to_event(pkt), &subtype)); + odp_event_types(odp_packet_to_event(*pkt_out), &subtype)); CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == subtype); - CU_ASSERT(odp_packet_subtype(pkt) == ODP_EVENT_PACKET_CRYPTO); + CU_ASSERT(odp_packet_subtype(*pkt_out) == ODP_EVENT_PACKET_CRYPTO); - rc = odp_crypto_result(&result, pkt); + rc = odp_crypto_result(&result, *pkt_out); if (rc < 0) { CU_FAIL("Failed odp_crypto_packet_result()"); return rc; } + CU_ASSERT(rc == 0); *ok = result.ok; return 0; } -static int crypto_op(odp_packet_t pkt, +static int crypto_op(odp_packet_t pkt_in, + odp_packet_t *pkt_out, odp_bool_t *ok, odp_crypto_session_t session, + odp_crypto_op_type_t op_type, uint8_t *cipher_iv, uint8_t *auth_iv, odp_packet_data_range_t *cipher_range, @@ -369,23 +386,25 @@ static int crypto_op(odp_packet_t pkt, { int rc; - if (!suite_context.packet) + if (!suite_context.packet) { #if ODP_DEPRECATED_API - rc = alg_op(pkt, ok, session, + rc = alg_op(pkt_in, ok, session, cipher_iv, auth_iv, cipher_range, auth_range, aad, hash_result_offset); + *pkt_out = pkt_in; #else rc = -1; #endif - else - rc = alg_packet_op(pkt, ok, session, + } else { + rc = alg_packet_op(pkt_in, pkt_out, ok, session, op_type, cipher_iv, auth_iv, cipher_range, auth_range, aad, hash_result_offset); + } if (rc < 0) - odp_packet_free(pkt); + odp_packet_free(pkt_in); return rc; } @@ -467,6 +486,7 @@ typedef enum crypto_test { typedef struct alg_test_param_t { odp_crypto_session_t session; odp_crypto_op_t op; + odp_crypto_op_type_t op_type; odp_cipher_alg_t cipher_alg; odp_auth_alg_t auth_alg; crypto_test_reference_t *ref; @@ -561,7 +581,8 @@ static void alg_test_execute(const alg_test_param_t *param) test_packet_set_md(pkt); test_packet_get_md(pkt, &md_in); - if (crypto_op(pkt, &ok, param->session, + if (crypto_op(pkt, &pkt, &ok, param->session, + param->op_type, cipher_iv, auth_iv, &cipher_range, &auth_range, ref->aad, digest_offset)) @@ -622,6 +643,7 @@ typedef enum { } hash_test_mode_t; static odp_crypto_session_t session_create(odp_crypto_op_t op, + odp_crypto_op_type_t op_type, odp_cipher_alg_t cipher_alg, odp_auth_alg_t auth_alg, crypto_test_reference_t *ref, @@ -648,6 +670,7 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, /* Create a crypto session */ odp_crypto_session_param_init(&ses_params); ses_params.op = op; + ses_params.op_type = op_type; ses_params.auth_cipher_text = false; ses_params.op_mode = suite_context.op_mode; #if ODP_DEPRECATED_API @@ -702,13 +725,14 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, return session; } -static void alg_test(odp_crypto_op_t op, - odp_cipher_alg_t cipher_alg, - odp_auth_alg_t auth_alg, - crypto_test_reference_t *ref, - uint32_t digest_offset, - odp_bool_t is_bit_mode_cipher, - odp_bool_t is_bit_mode_auth) +static void alg_test_ses(odp_crypto_op_t op, + odp_crypto_op_type_t op_type, + odp_cipher_alg_t cipher_alg, + odp_auth_alg_t auth_alg, + crypto_test_reference_t *ref, + uint32_t digest_offset, + odp_bool_t is_bit_mode_cipher, + odp_bool_t is_bit_mode_auth) { unsigned int initial_num_failures = CU_get_number_of_failures(); const uint32_t reflength = ref_length_in_bytes(ref); @@ -720,13 +744,14 @@ static void alg_test(odp_crypto_op_t op, uint32_t max_shift; alg_test_param_t test_param; - session = session_create(op, cipher_alg, auth_alg, ref, hash_mode); + session = session_create(op, op_type, cipher_alg, auth_alg, ref, hash_mode); if (session == ODP_CRYPTO_SESSION_INVALID) return; memset(&test_param, 0, sizeof(test_param)); test_param.session = session; test_param.op = op; + test_param.op_type = op_type; test_param.cipher_alg = cipher_alg; test_param.auth_alg = auth_alg; test_param.ref = ref; @@ -768,6 +793,34 @@ static void alg_test(odp_crypto_op_t op, CU_ASSERT(!rc); } +static void alg_test(odp_crypto_op_t op, + odp_cipher_alg_t cipher_alg, + odp_auth_alg_t auth_alg, + crypto_test_reference_t *ref, + uint32_t digest_offset, + odp_bool_t is_bit_mode_cipher, + odp_bool_t is_bit_mode_auth) +{ + odp_crypto_op_type_t op_types[] = { + ODP_CRYPTO_OP_TYPE_LEGACY, + ODP_CRYPTO_OP_TYPE_BASIC, + }; + + for (unsigned int n = 0; n < ARRAY_SIZE(op_types); n++) { + if (!suite_context.packet && + op_types[n] != ODP_CRYPTO_OP_TYPE_LEGACY) + continue; + alg_test_ses(op, + op_types[n], + cipher_alg, + auth_alg, + ref, + digest_offset, + is_bit_mode_cipher, + is_bit_mode_auth); + } +} + static odp_bool_t aad_len_ok(const odp_crypto_auth_capability_t *capa, uint32_t len) { if (len < capa->aad_len.min || len > capa->aad_len.max) @@ -1183,12 +1236,15 @@ static int create_hash_test_reference(odp_auth_alg_t auth, rc = odp_packet_copy_from_mem(pkt, 0, auth_bytes, ref->plaintext); CU_ASSERT(rc == 0); - session = session_create(ODP_CRYPTO_OP_ENCODE, ODP_CIPHER_ALG_NULL, + session = session_create(ODP_CRYPTO_OP_ENCODE, + ODP_CRYPTO_OP_TYPE_LEGACY, + ODP_CIPHER_ALG_NULL, auth, ref, HASH_NO_OVERLAP); if (session == ODP_CRYPTO_SESSION_INVALID) return -1; - rc = crypto_op(pkt, &ok, session, ref->cipher_iv, ref->auth_iv, + rc = crypto_op(pkt, &pkt, &ok, session, ODP_CRYPTO_OP_TYPE_LEGACY, + ref->cipher_iv, ref->auth_iv, &cipher_range, &auth_range, ref->aad, enc_digest_offset); CU_ASSERT(rc == 0); CU_ASSERT(ok); @@ -2460,6 +2516,15 @@ static int crypto_init(odp_instance_t *inst) params.pkt.num = PKT_POOL_NUM; params.type = ODP_POOL_PACKET; + /* + * Let's have a user area so that we can check that its + * content gets copied along with other metadata when needed. + */ + if (pool_capa.pkt.max_uarea_size >= UAREA_SIZE) + params.pkt.uarea_size = UAREA_SIZE; + else + printf("Warning: could not request packet user area\n"); + if (pool_capa.pkt.max_seg_len && PKT_POOL_LEN > pool_capa.pkt.max_seg_len) { fprintf(stderr, "Warning: small packet segment length\n"); -- cgit v1.2.3 From 62e3476419c8eee9ca8007542e6ba1069cb7629b Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Wed, 1 Feb 2023 19:54:34 +0200 Subject: validation: crypto: refactor alg_test_execute() Move the iteration across normal, repeat and wrong digest tests away from alg_test_execute() so that alg_test_execute() can concentrate on invoking and verifying single crypto operation. Move most of test packet building and result verification from alg_test_execute() to separate functions to make the code cleaner. Signed-off-by: Janne Peltonen Reviewed-by: Anoob Joseph --- test/validation/api/crypto/odp_crypto_test_inp.c | 261 ++++++++++++----------- 1 file changed, 140 insertions(+), 121 deletions(-) diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index acbf6779d..cbd499ceb 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -476,13 +476,6 @@ static void do_header_and_trailer(odp_packet_t pkt, } } -typedef enum crypto_test { - NORMAL_TEST = 0, /**< Plain execution */ - REPEAT_TEST, /**< Rerun without reinitializing the session */ - WRONG_DIGEST_TEST, /**< Check against wrong digest */ - MAX_TEST, /**< Final mark */ -} crypto_test; - typedef struct alg_test_param_t { odp_crypto_session_t session; odp_crypto_op_t op; @@ -494,147 +487,173 @@ typedef struct alg_test_param_t { odp_bool_t is_bit_mode_cipher; odp_bool_t is_bit_mode_auth; odp_bool_t adjust_segmentation; + odp_bool_t wrong_digest; uint32_t first_seg_len; uint32_t header_len; uint32_t trailer_len; } alg_test_param_t; -static void alg_test_execute(const alg_test_param_t *param) +static void prepare_crypto_ranges(const alg_test_param_t *param, + odp_packet_data_range_t *cipher_range, + odp_packet_data_range_t *auth_range) { - odp_bool_t ok = false; - int iteration; - crypto_test_reference_t *ref = param->ref; - uint32_t reflength = ref_length_in_bytes(ref); odp_packet_data_range_t zero_range = {.offset = 0, .length = 0}; - odp_packet_data_range_t cipher_range; - odp_packet_data_range_t auth_range; - uint8_t *cipher_iv = ref->cipher_iv; - uint8_t *auth_iv = ref->auth_iv; - test_packet_md_t md_in, md_out; - cipher_range.offset = param->header_len; - cipher_range.length = reflength; - auth_range.offset = param->header_len; - auth_range.length = reflength; + cipher_range->offset = param->header_len; + cipher_range->length = ref_length_in_bytes(param->ref); + auth_range->offset = param->header_len; + auth_range->length = ref_length_in_bytes(param->ref); if (param->is_bit_mode_cipher) { - cipher_range.offset *= 8; - cipher_range.length = ref_length_in_bits(ref); + cipher_range->offset *= 8; + cipher_range->length = ref_length_in_bits(param->ref); } if (param->is_bit_mode_auth) { - auth_range.offset *= 8; - auth_range.length = ref_length_in_bits(ref); + auth_range->offset *= 8; + auth_range->length = ref_length_in_bits(param->ref); } /* * We did not check the bit mode of the null algorithms, so let's * not pass potentially invalid ranges to them. */ if (param->cipher_alg == ODP_CIPHER_ALG_NULL) - cipher_range = zero_range; + *cipher_range = zero_range; if (param->auth_alg == ODP_AUTH_ALG_NULL) - auth_range = zero_range; - for (iteration = NORMAL_TEST; iteration < MAX_TEST; iteration++) { - odp_packet_t pkt; - uint32_t digest_offset = param->digest_offset; - uint32_t pkt_len; + *auth_range = zero_range; +} - /* - * Test detection of wrong digest value in input packet - * only when decoding and using non-null auth algorithm. - */ - if (iteration == WRONG_DIGEST_TEST && - (param->auth_alg == ODP_AUTH_ALG_NULL || - param->op == ODP_CRYPTO_OP_ENCODE)) - continue; +static int prepare_input_packet(const alg_test_param_t *param, + odp_packet_t *pkt_in) +{ + crypto_test_reference_t *ref = param->ref; + uint32_t reflength = ref_length_in_bytes(ref); + odp_packet_t pkt; + uint32_t digest_offset = param->digest_offset; + uint32_t pkt_len; - pkt_len = param->header_len + reflength + param->trailer_len; - if (param->digest_offset == param->header_len + reflength) - pkt_len += ref->digest_length; + pkt_len = param->header_len + reflength + param->trailer_len; + if (param->digest_offset == param->header_len + reflength) + pkt_len += ref->digest_length; - pkt = odp_packet_alloc(suite_context.pool, pkt_len); + pkt = odp_packet_alloc(suite_context.pool, pkt_len); - CU_ASSERT(pkt != ODP_PACKET_INVALID); - if (pkt == ODP_PACKET_INVALID) - continue; + CU_ASSERT(pkt != ODP_PACKET_INVALID); + if (pkt == ODP_PACKET_INVALID) + return -1; - if (param->adjust_segmentation) - adjust_segments(&pkt, param->first_seg_len); - - do_header_and_trailer(pkt, param->header_len, param->trailer_len, false); - - if (param->op == ODP_CRYPTO_OP_ENCODE) { - odp_packet_copy_from_mem(pkt, param->header_len, - reflength, ref->plaintext); - } else { - odp_packet_copy_from_mem(pkt, param->header_len, - reflength, ref->ciphertext); - odp_packet_copy_from_mem(pkt, digest_offset, - ref->digest_length, - ref->digest); - if (iteration == WRONG_DIGEST_TEST) { - uint8_t byte = ~ref->digest[0]; - - odp_packet_copy_from_mem(pkt, digest_offset, - 1, &byte); - } + if (param->adjust_segmentation) + adjust_segments(&pkt, param->first_seg_len); + + do_header_and_trailer(pkt, param->header_len, param->trailer_len, false); + + if (param->op == ODP_CRYPTO_OP_ENCODE) { + odp_packet_copy_from_mem(pkt, param->header_len, + reflength, ref->plaintext); + } else { + odp_packet_copy_from_mem(pkt, param->header_len, + reflength, ref->ciphertext); + odp_packet_copy_from_mem(pkt, digest_offset, + ref->digest_length, + ref->digest); + if (param->wrong_digest) { + uint8_t byte = ~ref->digest[0]; + + odp_packet_copy_from_mem(pkt, digest_offset, 1, &byte); } + } + *pkt_in = pkt; + return 0; +} - test_packet_set_md(pkt); - test_packet_get_md(pkt, &md_in); +static void check_output_packet_data(odp_packet_t pkt, + const alg_test_param_t *param) +{ + crypto_test_reference_t *ref = param->ref; + uint8_t *expected_bytes; - if (crypto_op(pkt, &pkt, &ok, param->session, - param->op_type, - cipher_iv, auth_iv, - &cipher_range, &auth_range, - ref->aad, digest_offset)) - break; + /* crypto output is undefined if authentication fails */ + if (param->wrong_digest) + return; + + do_header_and_trailer(pkt, param->header_len, param->trailer_len, true); + if (param->op == ODP_CRYPTO_OP_ENCODE) { + CU_ASSERT(!packet_cmp_mem_bytes(pkt, param->digest_offset, + ref->digest, + ref->digest_length)); + expected_bytes = ref->ciphertext; + } else { /* - * API is not explicit about whether a failed crypto op - * sets the has_error packet flag or leaves it unchanged. - * Let's allow both behaviours. + * Hash result in the packet is left to undefined + * values. Restore it from the plaintext packet + * to make the subsequent comparison work even + * if the hash result is within the auth_range. */ - test_packet_get_md(pkt, &md_out); - if (iteration == WRONG_DIGEST_TEST) - md_out.has_error = 0; - CU_ASSERT(test_packet_is_md_equal(&md_in, &md_out)); - - if (iteration == WRONG_DIGEST_TEST) { - CU_ASSERT(!ok); - odp_packet_free(pkt); - continue; - } - - CU_ASSERT(ok); - - do_header_and_trailer(pkt, param->header_len, param->trailer_len, true); - - if (param->op == ODP_CRYPTO_OP_ENCODE) { - CU_ASSERT(!packet_cmp_mem(pkt, param->header_len, - ref->ciphertext, - ref->length, - ref->is_length_in_bits)); - CU_ASSERT(!packet_cmp_mem_bytes(pkt, digest_offset, - ref->digest, - ref->digest_length)); - } else { - /* - * Hash result in the packet is left to undefined - * values. Restore it from the plaintext packet - * to make the subsequent comparison work even - * if the hash result is within the auth_range. - */ - odp_packet_copy_from_mem(pkt, digest_offset, - ref->digest_length, - ref->plaintext + - digest_offset - param->header_len); - - CU_ASSERT(!packet_cmp_mem(pkt, param->header_len, - ref->plaintext, - ref->length, - ref->is_length_in_bits)); - } - odp_packet_free(pkt); + odp_packet_copy_from_mem(pkt, param->digest_offset, + ref->digest_length, + ref->plaintext + + param->digest_offset - param->header_len); + expected_bytes = ref->plaintext; } + CU_ASSERT(!packet_cmp_mem(pkt, param->header_len, + expected_bytes, + ref->length, + ref->is_length_in_bits)); +} + +static void alg_test_execute(const alg_test_param_t *param) +{ + odp_bool_t ok = false; + odp_packet_data_range_t cipher_range; + odp_packet_data_range_t auth_range; + odp_packet_t pkt; + test_packet_md_t md_in, md_out; + + /* + * Test detection of wrong digest value in input packet + * only when decoding and using non-null auth algorithm. + */ + if (param->wrong_digest && + (param->auth_alg == ODP_AUTH_ALG_NULL || + param->op == ODP_CRYPTO_OP_ENCODE)) + return; + + prepare_crypto_ranges(param, &cipher_range, &auth_range); + if (prepare_input_packet(param, &pkt)) + return; + + test_packet_set_md(pkt); + test_packet_get_md(pkt, &md_in); + + if (crypto_op(pkt, &pkt, &ok, param->session, + param->op_type, + param->ref->cipher_iv, + param->ref->auth_iv, + &cipher_range, &auth_range, + param->ref->aad, param->digest_offset)) + return; + + /* + * API is not explicit about whether a failed crypto op + * sets the has_error packet flag or leaves it unchanged. + * Let's allow both behaviours. + */ + test_packet_get_md(pkt, &md_out); + if (param->wrong_digest) + md_out.has_error = 0; + CU_ASSERT(test_packet_is_md_equal(&md_in, &md_out)); + + CU_ASSERT(!!ok == !param->wrong_digest); + check_output_packet_data(pkt, param); + odp_packet_free(pkt); +} + +static void alg_test_op(alg_test_param_t *param) +{ + param->wrong_digest = false; + alg_test_execute(param); + alg_test_execute(param); /* rerun with the same parameters */ + param->wrong_digest = true; + alg_test_execute(param); } typedef enum { @@ -759,7 +778,7 @@ static void alg_test_ses(odp_crypto_op_t op, test_param.is_bit_mode_auth = is_bit_mode_auth; test_param.digest_offset = digest_offset; - alg_test_execute(&test_param); + alg_test_op(&test_param); max_shift = reflength + ref->digest_length; @@ -780,13 +799,13 @@ static void alg_test_ses(odp_crypto_op_t op, test_param.header_len = 0; test_param.trailer_len = 0; test_param.digest_offset = digest_offset; - alg_test_execute(&test_param); + alg_test_op(&test_param); /* Test partial packet crypto with odd alignment. */ test_param.header_len = 3; test_param.trailer_len = 32; test_param.digest_offset = test_param.header_len + digest_offset; - alg_test_execute(&test_param); + alg_test_op(&test_param); } rc = odp_crypto_session_destroy(session); -- cgit v1.2.3 From 1692a726261538be5cc1908f11e8cbbdc20b8320 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Wed, 1 Feb 2023 19:54:35 +0200 Subject: validation: crypto: change packet data checking approach Change the method of verifying the data content of crypto result packets to such that expected packet data is constructed and then compared with the actual packet data. This is to keep the verification more readable and better maintainable when more complex tests are added later. Signed-off-by: Janne Peltonen Reviewed-by: Anoob Joseph --- test/validation/api/crypto/odp_crypto_test_inp.c | 268 +++++++++++++++-------- 1 file changed, 172 insertions(+), 96 deletions(-) diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index cbd499ceb..9fae5c06c 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -56,53 +56,6 @@ static void test_default_values(void) test_defaults(0xff); } -static int packet_cmp_mem_bits(odp_packet_t pkt, uint32_t offset, - uint8_t *s, uint32_t len) -{ - int rc = -1; - uint32_t len_bytes = ((len + 7) / 8); - uint8_t leftover_bits = len % 8; - uint8_t buf[len_bytes]; - - odp_packet_copy_to_mem(pkt, offset, len_bytes, buf); - - /* Compare till the last full byte */ - rc = memcmp(buf, s, leftover_bits ? len_bytes - 1 : len_bytes); - - if (rc == 0 && leftover_bits) { - /* Do masked comparison for the leftover bits */ - uint8_t mask = 0xff << (8 - leftover_bits); - - rc = !((mask & buf[len_bytes - 1]) == - (mask & s[len_bytes - 1])); - } - - return rc; -} - -static int packet_cmp_mem_bytes(odp_packet_t pkt, uint32_t offset, - uint8_t *s, uint32_t len) -{ - uint8_t buf[len]; - - odp_packet_copy_to_mem(pkt, offset, len, buf); - - return memcmp(buf, s, len); -} - -static int packet_cmp_mem(odp_packet_t pkt, uint32_t offset, - uint8_t *s, uint32_t len, odp_bool_t bit_mode) -{ - int rc = -1; - - if (bit_mode) - rc = packet_cmp_mem_bits(pkt, offset, s, len); - else - rc = packet_cmp_mem_bytes(pkt, offset, s, len); - - return rc; -} - static const char *auth_alg_name(odp_auth_alg_t auth) { switch (auth) { @@ -447,12 +400,8 @@ static void fill_with_pattern(uint8_t *buf, uint32_t len) buf[n] = n; } -/* - * Generate or verify header and trailer bytes - */ -static void do_header_and_trailer(odp_packet_t pkt, - uint32_t header_len, uint32_t trailer_len, - odp_bool_t check) +static void write_header_and_trailer(odp_packet_t pkt, + uint32_t header_len, uint32_t trailer_len) { uint32_t trailer_offset = odp_packet_len(pkt) - trailer_len; uint32_t max_len = header_len > trailer_len ? header_len : trailer_len; @@ -461,19 +410,10 @@ static void do_header_and_trailer(odp_packet_t pkt, fill_with_pattern(buffer, sizeof(buffer)); - if (check) { - CU_ASSERT(!packet_cmp_mem_bytes(pkt, 0, - buffer, header_len)); - CU_ASSERT(!packet_cmp_mem_bytes(pkt, trailer_offset, - buffer, trailer_len)); - } else { - rc = odp_packet_copy_from_mem(pkt, 0, - header_len, buffer); - CU_ASSERT(rc == 0); - rc = odp_packet_copy_from_mem(pkt, trailer_offset, - trailer_len, buffer); - CU_ASSERT(rc == 0); - } + rc = odp_packet_copy_from_mem(pkt, 0, header_len, buffer); + CU_ASSERT(rc == 0); + rc = odp_packet_copy_from_mem(pkt, trailer_offset, trailer_len, buffer); + CU_ASSERT(rc == 0); } typedef struct alg_test_param_t { @@ -543,7 +483,7 @@ static int prepare_input_packet(const alg_test_param_t *param, if (param->adjust_segmentation) adjust_segments(&pkt, param->first_seg_len); - do_header_and_trailer(pkt, param->header_len, param->trailer_len, false); + write_header_and_trailer(pkt, param->header_len, param->trailer_len); if (param->op == ODP_CRYPTO_OP_ENCODE) { odp_packet_copy_from_mem(pkt, param->header_len, @@ -564,40 +504,168 @@ static int prepare_input_packet(const alg_test_param_t *param, return 0; } -static void check_output_packet_data(odp_packet_t pkt, - const alg_test_param_t *param) -{ - crypto_test_reference_t *ref = param->ref; - uint8_t *expected_bytes; +#define MAX_IGNORED_RANGES 3 - /* crypto output is undefined if authentication fails */ - if (param->wrong_digest) +/* + * Output packet parts that we ignore since they have undefined values + */ +typedef struct ignore_t { + uint32_t byte_offset; /* offset to a byte which has bits to be ignored */ + uint32_t byte_mask; /* mask of ignored bits in the byte */ + struct { + uint32_t offset; + uint32_t length; + } ranges[MAX_IGNORED_RANGES]; /* byte ranges to be ignored */ + uint32_t num_ranges; +} ignore_t; + +static void add_ignored_range(ignore_t *ign, uint32_t offs, uint32_t len) +{ + if (len == 0) return; + CU_ASSERT_FATAL(ign->num_ranges < MAX_IGNORED_RANGES); + ign->ranges[ign->num_ranges].offset = offs; + ign->ranges[ign->num_ranges].length = len; + ign->num_ranges++; +} + +static void clear_ignored_data(const ignore_t *ign, uint8_t *data, uint32_t data_len) +{ + CU_ASSERT_FATAL(ign->byte_offset < data_len); + data[ign->byte_offset] &= ~ign->byte_mask; + + for (uint32_t n = 0; n < ign->num_ranges; n++) { + uint32_t offset = ign->ranges[n].offset; + uint32_t length = ign->ranges[n].length; + + CU_ASSERT(offset + length <= data_len); + memset(data + offset, 0, length); + } +} + +static void prepare_ignore_info(const alg_test_param_t *param, + uint32_t cipher_offset, + uint32_t cipher_len, + ignore_t *ignore) +{ + memset(ignore, 0, sizeof(*ignore)); + + /* + * Leftover bits in the last byte of the cipher range of bit mode + * ciphers have undefined values. + */ + if (param->is_bit_mode_cipher && + param->cipher_alg != ODP_CIPHER_ALG_NULL) { + uint8_t leftover_bits = param->ref->length % 8; - do_header_and_trailer(pkt, param->header_len, param->trailer_len, true); + ignore->byte_offset = cipher_offset + cipher_len - 1; + ignore->byte_mask = ~(0xff << (8 - leftover_bits)); + } + + /* + * In decode sessions the bytes in the hash location have + * undefined values. + */ + if (param->auth_alg != ODP_AUTH_ALG_NULL && + param->op == ODP_CRYPTO_OP_DECODE) { + add_ignored_range(ignore, + param->digest_offset, + param->ref->digest_length); + } +} + +/* Add room for bytes that are not included in ref->length */ +#define MAX_EXP_DATA_LEN (MAX_DATA_LEN + 200) + +/* + * Expected packet data + */ +typedef struct expected_t { + uint8_t data[MAX_EXP_DATA_LEN]; + uint32_t len; + ignore_t ignore; +} expected_t; + +static void prepare_expected_data(const alg_test_param_t *param, + const odp_packet_data_range_t *cipher_range, + odp_packet_t pkt_in, + expected_t *ex) +{ + uint32_t digest_offset = param->digest_offset; + uint32_t cipher_offset = cipher_range->offset; + uint32_t cipher_len = cipher_range->length; + int rc; + + if (param->is_bit_mode_cipher) { + cipher_offset /= 8; + cipher_len = (cipher_len + 7) / 8; + } + if (param->cipher_alg == ODP_CIPHER_ALG_NULL) + cipher_len = 0; + + /* copy all data from input packet */ + ex->len = odp_packet_len(pkt_in); + CU_ASSERT_FATAL(ex->len <= sizeof(ex->data)); + rc = odp_packet_copy_to_mem(pkt_in, 0, ex->len, ex->data); + CU_ASSERT(rc == 0); if (param->op == ODP_CRYPTO_OP_ENCODE) { - CU_ASSERT(!packet_cmp_mem_bytes(pkt, param->digest_offset, - ref->digest, - ref->digest_length)); - expected_bytes = ref->ciphertext; - } else { + /* copy hash first */ + memcpy(ex->data + digest_offset, + param->ref->digest, + param->ref->digest_length); /* - * Hash result in the packet is left to undefined - * values. Restore it from the plaintext packet - * to make the subsequent comparison work even - * if the hash result is within the auth_range. + * Copy ciphertext, possibly overwriting hash. + * The other order (hash overwriting some cipher + * text) does not work in any real use case anyway. */ - odp_packet_copy_from_mem(pkt, param->digest_offset, - ref->digest_length, - ref->plaintext + - param->digest_offset - param->header_len); - expected_bytes = ref->plaintext; + memcpy(ex->data + cipher_offset, + param->ref->ciphertext, + cipher_len); + } else { + memcpy(ex->data + cipher_offset, + param->ref->plaintext, + cipher_len); + } + + prepare_ignore_info(param, + cipher_offset, cipher_len, + &ex->ignore); +} + +static void print_data(const char *title, uint8_t *data, uint32_t len) +{ + static uint64_t limit; + + if (limit++ > 20) + return; + + printf("%s\n", title); + for (uint32_t n = 0; n < len ; n++) { + printf(" %02x", data[n]); + if ((n + 1) % 16 == 0) + printf("\n"); + } + printf("\n"); +} + +static void check_output_packet_data(odp_packet_t pkt, expected_t *ex) +{ + int rc; + uint8_t pkt_data[ex->len]; + + CU_ASSERT(odp_packet_len(pkt) == ex->len); + rc = odp_packet_copy_to_mem(pkt, 0, ex->len, pkt_data); + CU_ASSERT(rc == 0); + + clear_ignored_data(&ex->ignore, pkt_data, sizeof(pkt_data)); + clear_ignored_data(&ex->ignore, ex->data, sizeof(ex->data)); + + if (memcmp(pkt_data, ex->data, ex->len)) { + CU_FAIL("packet data does not match expected data"); + print_data("packet:", pkt_data, ex->len); + print_data("expected:", ex->data, ex->len); } - CU_ASSERT(!packet_cmp_mem(pkt, param->header_len, - expected_bytes, - ref->length, - ref->is_length_in_bits)); } static void alg_test_execute(const alg_test_param_t *param) @@ -607,6 +675,7 @@ static void alg_test_execute(const alg_test_param_t *param) odp_packet_data_range_t auth_range; odp_packet_t pkt; test_packet_md_t md_in, md_out; + expected_t expected; /* * Test detection of wrong digest value in input packet @@ -621,6 +690,8 @@ static void alg_test_execute(const alg_test_param_t *param) if (prepare_input_packet(param, &pkt)) return; + prepare_expected_data(param, &cipher_range, pkt, &expected); + test_packet_set_md(pkt); test_packet_get_md(pkt, &md_in); @@ -642,8 +713,13 @@ static void alg_test_execute(const alg_test_param_t *param) md_out.has_error = 0; CU_ASSERT(test_packet_is_md_equal(&md_in, &md_out)); - CU_ASSERT(!!ok == !param->wrong_digest); - check_output_packet_data(pkt, param); + if (param->wrong_digest) { + CU_ASSERT(!ok); + /* output packet data is undefined, skip check */ + } else { + CU_ASSERT(ok); + check_output_packet_data(pkt, &expected); + } odp_packet_free(pkt); } @@ -1278,11 +1354,11 @@ static int create_hash_test_reference(odp_auth_alg_t auth, /* copy the calculated digest in the ciphertext packet in ref */ rc = odp_packet_copy_to_mem(pkt, enc_digest_offset, ref->digest_length, &ref->ciphertext[digest_offset]); + CU_ASSERT(rc == 0); /* copy the calculated digest the digest field in ref */ rc = odp_packet_copy_to_mem(pkt, enc_digest_offset, ref->digest_length, &ref->digest); - CU_ASSERT(rc == 0); odp_packet_free(pkt); -- cgit v1.2.3 From a9823897f0d5dfdbebc42ce5cffca9b450928031 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Wed, 1 Feb 2023 19:54:37 +0200 Subject: validation: crypto: add ODP_CRYPTO_OP_TYPE_OOP testing Add testing of the out-of-place crypto operation type. Signed-off-by: Janne Peltonen Reviewed-by: Anoob Joseph --- test/validation/api/crypto/odp_crypto_test_inp.c | 220 +++++++++++++++++++---- 1 file changed, 188 insertions(+), 32 deletions(-) diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index 9fae5c06c..7a27092c4 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -5,6 +5,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include #include #include #include @@ -237,6 +238,7 @@ static int alg_packet_op(odp_packet_t pkt_in, odp_bool_t *ok, odp_crypto_session_t session, odp_crypto_op_type_t op_type, + int32_t oop_shift, uint8_t *cipher_iv_ptr, uint8_t *auth_iv_ptr, odp_packet_data_range_t *cipher_range, @@ -249,11 +251,13 @@ static int alg_packet_op(odp_packet_t pkt_in, odp_crypto_packet_result_t result; odp_crypto_packet_op_param_t op_params; odp_event_subtype_t subtype; + odp_packet_t orig_pkt_out; if (op_type == ODP_CRYPTO_OP_TYPE_LEGACY) *pkt_out = pkt_in; - else + else if (op_type == ODP_CRYPTO_OP_TYPE_BASIC) *pkt_out = ODP_PACKET_INVALID; + orig_pkt_out = *pkt_out; /* Prepare input/output params */ memset(&op_params, 0, sizeof(op_params)); @@ -261,6 +265,7 @@ static int alg_packet_op(odp_packet_t pkt_in, op_params.cipher_range = *cipher_range; op_params.auth_range = *auth_range; + op_params.dst_offset_shift = oop_shift; if (cipher_iv_ptr) op_params.cipher_iv_ptr = cipher_iv_ptr; if (auth_iv_ptr) @@ -303,7 +308,7 @@ static int alg_packet_op(odp_packet_t pkt_in, } if (op_type != ODP_CRYPTO_OP_TYPE_BASIC) - CU_ASSERT(*pkt_out == pkt_in); + CU_ASSERT(*pkt_out == orig_pkt_out); CU_ASSERT(ODP_EVENT_PACKET == odp_event_type(odp_packet_to_event(*pkt_out))); CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == @@ -320,6 +325,10 @@ static int alg_packet_op(odp_packet_t pkt_in, } CU_ASSERT(rc == 0); + if (op_type == ODP_CRYPTO_OP_TYPE_OOP && + suite_context.op_mode == ODP_CRYPTO_ASYNC) + CU_ASSERT(result.pkt_in == pkt_in); + *ok = result.ok; return 0; @@ -330,6 +339,7 @@ static int crypto_op(odp_packet_t pkt_in, odp_bool_t *ok, odp_crypto_session_t session, odp_crypto_op_type_t op_type, + int32_t oop_shift, uint8_t *cipher_iv, uint8_t *auth_iv, odp_packet_data_range_t *cipher_range, @@ -350,7 +360,8 @@ static int crypto_op(odp_packet_t pkt_in, rc = -1; #endif } else { - rc = alg_packet_op(pkt_in, pkt_out, ok, session, op_type, + rc = alg_packet_op(pkt_in, pkt_out, ok, session, + op_type, oop_shift, cipher_iv, auth_iv, cipher_range, auth_range, aad, hash_result_offset); @@ -420,6 +431,7 @@ typedef struct alg_test_param_t { odp_crypto_session_t session; odp_crypto_op_t op; odp_crypto_op_type_t op_type; + int32_t oop_shift; odp_cipher_alg_t cipher_alg; odp_auth_alg_t auth_alg; crypto_test_reference_t *ref; @@ -504,6 +516,54 @@ static int prepare_input_packet(const alg_test_param_t *param, return 0; } +static void prepare_oop_output_packet(const alg_test_param_t *param, + odp_packet_t *pkt_out, + uint32_t pkt_len) +{ + uint32_t reflength = ref_length_in_bytes(param->ref); + const uint32_t oop_extra_len = 5; + uint32_t trl_len; + uint32_t hdr_len; + uint32_t oop_len; + + oop_len = pkt_len + param->oop_shift + oop_extra_len; + *pkt_out = odp_packet_alloc(suite_context.pool, oop_len); + CU_ASSERT_FATAL(*pkt_out != ODP_PACKET_INVALID); + + uint8_t buf[oop_len]; + + memset(buf, 0x55, sizeof(buf)); + odp_packet_copy_from_mem(*pkt_out, 0, sizeof(buf), buf); + + hdr_len = param->header_len + param->oop_shift; + trl_len = oop_len - hdr_len - reflength; + + write_header_and_trailer(*pkt_out, hdr_len, trl_len); + + /* have different metadata than in the input packet */ + memset(odp_packet_user_area(*pkt_out), 0xab, + odp_packet_user_area_size(*pkt_out)); +} + +static int is_packet_data_equal(odp_packet_t pkt_1, odp_packet_t pkt_2) +{ + uint32_t len = odp_packet_len(pkt_1); + uint8_t buf_1[len]; + uint8_t buf_2[len]; + + if (len != odp_packet_len(pkt_2) || + odp_packet_copy_to_mem(pkt_1, 0, len, buf_1) || + odp_packet_copy_to_mem(pkt_2, 0, len, buf_2)) + return 0; + + return !memcmp(buf_1, buf_2, len); +} + +static int is_in_range(uint32_t offs, uint32_t range_offs, uint32_t range_len) +{ + return offs >= range_offs && offs < range_offs + range_len; +} + #define MAX_IGNORED_RANGES 3 /* @@ -544,8 +604,11 @@ static void clear_ignored_data(const ignore_t *ign, uint8_t *data, uint32_t data } static void prepare_ignore_info(const alg_test_param_t *param, + uint32_t shift, uint32_t cipher_offset, uint32_t cipher_len, + uint32_t auth_offset, + uint32_t auth_len, ignore_t *ignore) { memset(ignore, 0, sizeof(*ignore)); @@ -558,7 +621,7 @@ static void prepare_ignore_info(const alg_test_param_t *param, param->cipher_alg != ODP_CIPHER_ALG_NULL) { uint8_t leftover_bits = param->ref->length % 8; - ignore->byte_offset = cipher_offset + cipher_len - 1; + ignore->byte_offset = cipher_offset + cipher_len - 1 + shift; ignore->byte_mask = ~(0xff << (8 - leftover_bits)); } @@ -568,9 +631,15 @@ static void prepare_ignore_info(const alg_test_param_t *param, */ if (param->auth_alg != ODP_AUTH_ALG_NULL && param->op == ODP_CRYPTO_OP_DECODE) { - add_ignored_range(ignore, - param->digest_offset, - param->ref->digest_length); + uint32_t offs = param->digest_offset; + + if (param->op_type != ODP_CRYPTO_OP_TYPE_OOP || + is_in_range(offs, cipher_offset, cipher_len) || + is_in_range(offs, auth_offset, auth_len)) { + add_ignored_range(ignore, + param->digest_offset + shift, + param->ref->digest_length); + } } } @@ -588,27 +657,54 @@ typedef struct expected_t { static void prepare_expected_data(const alg_test_param_t *param, const odp_packet_data_range_t *cipher_range, + const odp_packet_data_range_t *auth_range, odp_packet_t pkt_in, + odp_packet_t pkt_out, expected_t *ex) { uint32_t digest_offset = param->digest_offset; uint32_t cipher_offset = cipher_range->offset; uint32_t cipher_len = cipher_range->length; + uint32_t auth_offset = auth_range->offset; + uint32_t auth_len = auth_range->length; + const int32_t shift = param->op_type == ODP_CRYPTO_OP_TYPE_OOP ? param->oop_shift : 0; + const odp_packet_t base_pkt = param->op_type == ODP_CRYPTO_OP_TYPE_OOP ? pkt_out : pkt_in; int rc; + if (param->op == ODP_CRYPTO_OP_ENCODE) + digest_offset += shift; + if (param->is_bit_mode_cipher) { cipher_offset /= 8; cipher_len = (cipher_len + 7) / 8; } + if (param->is_bit_mode_auth) { + auth_offset /= 8; + auth_len = (auth_len + 7) / 8; + } if (param->cipher_alg == ODP_CIPHER_ALG_NULL) cipher_len = 0; + if (param->auth_alg == ODP_AUTH_ALG_NULL || + param->auth_alg == ODP_AUTH_ALG_AES_GCM || + param->auth_alg == ODP_AUTH_ALG_AES_CCM || + param->auth_alg == ODP_AUTH_ALG_CHACHA20_POLY1305) { + /* auth range is ignored with null and AEAD algorithms */ + auth_len = 0; + } - /* copy all data from input packet */ - ex->len = odp_packet_len(pkt_in); + /* copy all data from base packet */ + ex->len = odp_packet_len(base_pkt); CU_ASSERT_FATAL(ex->len <= sizeof(ex->data)); - rc = odp_packet_copy_to_mem(pkt_in, 0, ex->len, ex->data); + rc = odp_packet_copy_to_mem(base_pkt, 0, ex->len, ex->data); CU_ASSERT(rc == 0); + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP && auth_len > 0) { + /* copy auth range from input packet */ + rc = odp_packet_copy_to_mem(pkt_in, auth_offset, auth_len, + ex->data + auth_offset + shift); + CU_ASSERT(rc == 0); + } + if (param->op == ODP_CRYPTO_OP_ENCODE) { /* copy hash first */ memcpy(ex->data + digest_offset, @@ -619,17 +715,18 @@ static void prepare_expected_data(const alg_test_param_t *param, * The other order (hash overwriting some cipher * text) does not work in any real use case anyway. */ - memcpy(ex->data + cipher_offset, + memcpy(ex->data + cipher_offset + shift, param->ref->ciphertext, cipher_len); } else { - memcpy(ex->data + cipher_offset, + memcpy(ex->data + cipher_offset + shift, param->ref->plaintext, cipher_len); } - prepare_ignore_info(param, + prepare_ignore_info(param, shift, cipher_offset, cipher_len, + auth_offset, auth_len, &ex->ignore); } @@ -674,7 +771,10 @@ static void alg_test_execute(const alg_test_param_t *param) odp_packet_data_range_t cipher_range; odp_packet_data_range_t auth_range; odp_packet_t pkt; - test_packet_md_t md_in, md_out; + odp_packet_t pkt_copy = ODP_PACKET_INVALID; + odp_packet_t pkt_out = ODP_PACKET_INVALID; + uint32_t digest_offset = param->digest_offset; + test_packet_md_t md_in, md_out, md_out_orig; expected_t expected; /* @@ -690,17 +790,35 @@ static void alg_test_execute(const alg_test_param_t *param) if (prepare_input_packet(param, &pkt)) return; - prepare_expected_data(param, &cipher_range, pkt, &expected); + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + prepare_oop_output_packet(param, &pkt_out, odp_packet_len(pkt)); + + pkt_copy = odp_packet_copy(pkt, suite_context.pool); + CU_ASSERT_FATAL(pkt_copy != ODP_PACKET_INVALID); + test_packet_get_md(pkt_out, &md_out_orig); + } + + prepare_expected_data(param, &cipher_range, &auth_range, + pkt, pkt_out, &expected); + + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP && + param->op == ODP_CRYPTO_OP_ENCODE) { + /* + * In this type of sessions digest offset is an offset to the output + * packet, so apply the shift. + */ + digest_offset += param->oop_shift; + } test_packet_set_md(pkt); test_packet_get_md(pkt, &md_in); - if (crypto_op(pkt, &pkt, &ok, param->session, - param->op_type, + if (crypto_op(pkt, &pkt_out, &ok, param->session, + param->op_type, param->oop_shift, param->ref->cipher_iv, param->ref->auth_iv, &cipher_range, &auth_range, - param->ref->aad, param->digest_offset)) + param->ref->aad, digest_offset)) return; /* @@ -708,30 +826,58 @@ static void alg_test_execute(const alg_test_param_t *param) * sets the has_error packet flag or leaves it unchanged. * Let's allow both behaviours. */ - test_packet_get_md(pkt, &md_out); + test_packet_get_md(pkt_out, &md_out); if (param->wrong_digest) md_out.has_error = 0; - CU_ASSERT(test_packet_is_md_equal(&md_in, &md_out)); + + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + test_packet_md_t md; + + /* check that input packet has not changed */ + CU_ASSERT(is_packet_data_equal(pkt, pkt_copy)); + odp_packet_free(pkt_copy); + test_packet_get_md(pkt, &md); + CU_ASSERT(test_packet_is_md_equal(&md, &md_in)); + odp_packet_free(pkt); + + /* check that metadata of output packet has not changed */ + CU_ASSERT(test_packet_is_md_equal(&md_out, &md_out_orig)); + } else { + CU_ASSERT(test_packet_is_md_equal(&md_out, &md_in)); + } if (param->wrong_digest) { CU_ASSERT(!ok); /* output packet data is undefined, skip check */ } else { CU_ASSERT(ok); - check_output_packet_data(pkt, &expected); + check_output_packet_data(pkt_out, &expected); } - odp_packet_free(pkt); + odp_packet_free(pkt_out); } static void alg_test_op(alg_test_param_t *param) { - param->wrong_digest = false; - alg_test_execute(param); - alg_test_execute(param); /* rerun with the same parameters */ - param->wrong_digest = true; - alg_test_execute(param); + int32_t oop_shifts[] = {0, 3, 130, -10}; + + for (uint32_t n = 0; n < ARRAY_SIZE(oop_shifts); n++) { + if (oop_shifts[n] != 0 && + param->op_type != ODP_CRYPTO_OP_TYPE_OOP) + continue; + if ((int32_t)param->header_len + oop_shifts[n] < 0) + continue; + param->oop_shift = oop_shifts[n]; + + param->wrong_digest = false; + alg_test_execute(param); + alg_test_execute(param); /* rerun with the same parameters */ + param->wrong_digest = true; + alg_test_execute(param); + } } +static int oop_warning_shown; + typedef enum { HASH_NO_OVERLAP, HASH_OVERLAP, @@ -795,14 +941,21 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, auth_alg_name(auth_alg)); return ODP_CRYPTO_SESSION_INVALID; } + + /* For now, allow out-of-place sessions not to be supported. */ + if (rc < 0 && status == ODP_CRYPTO_SES_ERR_PARAMS && + op_type == ODP_CRYPTO_OP_TYPE_OOP) { + if (!oop_warning_shown) + printf("\n Skipping ODP_CRYPTO_OP_TYPE_OOP tests\n"); + oop_warning_shown = 1; + return ODP_CRYPTO_SESSION_INVALID; + } + /* * We do not allow ODP_CRYPTO_SES_ERR_ALG_ORDER since we do * not combine individual non-null crypto and auth algorithms * with each other in the tests. Both orders should work when * only one algorithm is used (i.e. the other one is null). - * - * We do not allow ODP_CRYPTO_SES_ERR_PARAMS until needed for - * some ODP implementation. */ CU_ASSERT_FATAL(!rc); CU_ASSERT(status == ODP_CRYPTO_SES_ERR_NONE); @@ -878,7 +1031,7 @@ static void alg_test_ses(odp_crypto_op_t op, alg_test_op(&test_param); /* Test partial packet crypto with odd alignment. */ - test_param.header_len = 3; + test_param.header_len = 13; test_param.trailer_len = 32; test_param.digest_offset = test_param.header_len + digest_offset; alg_test_op(&test_param); @@ -899,6 +1052,7 @@ static void alg_test(odp_crypto_op_t op, odp_crypto_op_type_t op_types[] = { ODP_CRYPTO_OP_TYPE_LEGACY, ODP_CRYPTO_OP_TYPE_BASIC, + ODP_CRYPTO_OP_TYPE_OOP, }; for (unsigned int n = 0; n < ARRAY_SIZE(op_types); n++) { @@ -961,6 +1115,8 @@ static void check_alg(odp_crypto_op_t op, memset(cipher_tested, 0, sizeof(cipher_tested)); memset(auth_tested, 0, sizeof(auth_tested)); + oop_warning_shown = 0; /* allow OOP-unsupported warning again */ + for (idx = 0; idx < count; idx++) { int cipher_idx = -1, auth_idx = -1; odp_bool_t bit_mode_needed = false; @@ -1338,7 +1494,7 @@ static int create_hash_test_reference(odp_auth_alg_t auth, if (session == ODP_CRYPTO_SESSION_INVALID) return -1; - rc = crypto_op(pkt, &pkt, &ok, session, ODP_CRYPTO_OP_TYPE_LEGACY, + rc = crypto_op(pkt, &pkt, &ok, session, ODP_CRYPTO_OP_TYPE_LEGACY, 0, ref->cipher_iv, ref->auth_iv, &cipher_range, &auth_range, ref->aad, enc_digest_offset); CU_ASSERT(rc == 0); -- cgit v1.2.3 From 32a684b42a99d55c9b2840ac12911e1bd71dded6 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Wed, 1 Feb 2023 19:54:39 +0200 Subject: validation: crypto: check output packet better after failed auth Always check the parts of packets that are not supposed to be affected by crypto operations. Do the checking now also after failed authentication. Signed-off-by: Janne Peltonen Reviewed-by: Anoob Joseph --- test/validation/api/crypto/odp_crypto_test_inp.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index 7a27092c4..7d5e50c71 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -641,6 +641,15 @@ static void prepare_ignore_info(const alg_test_param_t *param, param->ref->digest_length); } } + + /* Decrypted bytes are undefined if authentication fails. */ + if (param->op == ODP_CRYPTO_OP_DECODE && + param->wrong_digest) { + add_ignored_range(ignore, cipher_offset + shift, cipher_len); + /* In OOP case, auth range may not get copied */ + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) + add_ignored_range(ignore, auth_offset + shift, auth_len); + } } /* Add room for bytes that are not included in ref->length */ @@ -848,11 +857,10 @@ static void alg_test_execute(const alg_test_param_t *param) if (param->wrong_digest) { CU_ASSERT(!ok); - /* output packet data is undefined, skip check */ } else { CU_ASSERT(ok); - check_output_packet_data(pkt_out, &expected); } + check_output_packet_data(pkt_out, &expected); odp_packet_free(pkt_out); } -- cgit v1.2.3 From 30930db550a640df1a25847c7623e5ecfe670b9b Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 7 Feb 2023 15:35:10 +0200 Subject: api: increment ODP API version to 1.40.0 Increment API version number to reflect the following changes: Backward incompatible: - packet: specify which packet metadata flags cannot be set simultaneously using odp_packet_has_XX_set() functions - timer: use ODP_DEPRECATE() macro for odp_timer_set_abs() and odp_timer_set_rel() functions - timer: deprecate old timer pool clock sources ODP_CLOCK_CPU and ODP_CLOCK_EXT - timer: deprecate old timer set return values ODP_TIMER_TOOEARLY, ODP_TIMER_TOOLATE, and ODP_TIMER_NOEVENT Backward compatible: - crypto: add new operation type session parameter odp_crypto_session_param_t.op_type - crypto: add ODP_CRYPTO_OP_TYPE_BASIC operation type - crypto: add ODP_CRYPTO_OP_TYPE_OOP operation type - crypto: clarify odp_crypto_op() documentation regarding packet data and metadata copy - crypto: require that odp_crypto_result() is called before accessing packet data of asynchronously processed packets in non-legacy modes - crypto: fix EIA2 IV length in API documentation - packet: clarify odp_packet_has_vlan() and odp_packet_has_vlan_qinq() specifications Removed deprecated APIs: - crypto: per-session IV configuration - pktio: in_unknown_protos field from odp_pktio_stats_t - pktio: odp_pktin_ts_res() function - pktio: odp_pktin_ts_from_ns() function - shm: ODP_SHM_SW_ONLY define - tm: odp_tm_capabilities() function - tm: commit_bps field from odp_tm_shaper_params_t - tm: peak_bps field from odp_tm_shaper_params_t Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- CHANGELOG | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ configure.ac | 2 +- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 863b41002..8d6a88ddf 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,71 @@ +== OpenDataPlane (1.40.0.0) + +=== Backward incompatible API changes +==== Packet +* Specify which packet metadata flags cannot be set simultaneously using +`odp_packet_has_XX_set()` functions. + +==== Timer +* Use `ODP_DEPRECATE()` macro for `odp_timer_set_abs()` and +`odp_timer_set_rel()` functions. Previously, the deprecation was only mentioned +in the function documentation. +* Deprecate old timer pool clock sources `ODP_CLOCK_CPU` and `ODP_CLOCK_EXT`, +which have been replaced by `ODP_CLOCK_DEFAULT` and `ODP_CLOCK_SRC_1`. +* Deprecate old timer set return values `ODP_TIMER_TOOEARLY`, +`ODP_TIMER_TOOLATE`, and `ODP_TIMER_NOEVENT`, which have been replaced by +`ODP_TIMER_TOO_NEAR`, `ODP_TIMER_TOO_FAR`, and `ODP_TIMER_FAIL`. + +=== Backward compatible API changes +==== Crypto +* Add new operation type session parameter +(`odp_crypto_session_param_t.op_type`) that controls how crypto operations +interpret their parameters and handle output packets. Defaults to backward +compatible `ODP_CRYPTO_OP_TYPE_LEGACY` mode. +* Add `ODP_CRYPTO_OP_TYPE_BASIC` operation type with simplified interface +compared to `ODP_CRYPTO_OP_TYPE_LEGACY`. +* Add `ODP_CRYPTO_OP_TYPE_OOP` operation type that writes the output of the +crypto operation into a caller provided output packet and does not consume the +input packet. +* Clarify that `odp_crypto_op()` copies all packet data and metadata from the +input packet to the output packet in `ODP_CRYPTO_OP_TYPE_LEGACY` and +`ODP_CRYPTO_OP_TYPE_BASIC` modes. +* Require that `odp_crypto_result()` is called before packet data of +asynchronously processed packets can be assumed to be valid in non-legacy modes. +* Fix EIA2 IV length in API documentation. EIA2 uses 64-bit IV, not 128-bit as +previously mentioned in the API text. + +==== Packet +* Clarify `odp_packet_has_vlan()` and `odp_packet_has_vlan_qinq()` +specifications. + +=== Remove deprecated APIs +==== Crypto +* Remove deprecated per-session IV configuration. + +==== Packet IO +* Remove deprecated `in_unknown_protos` field from `odp_pktio_stats_t`. +* Remove deprecated `odp_pktin_ts_res()` function. +* Remove deprecated `odp_pktin_ts_from_ns()` function. + +==== Shared Memory +* Remove deprecated `ODP_SHM_SW_ONLY` define. + +==== Traffic Manager +* Remove deprecated `odp_tm_capabilities()` function. +* Remove deprecated `commit_bps` field from `odp_tm_shaper_params_t`. +* Remove deprecated `peak_bps` field from `odp_tm_shaper_params_t`. + +=== Implementation +==== Crypto +* Add Multi-Buffer Crypto for IPsec library (Arm optimized) based crypto +implementation. See `DEPENDENCIES` for additional information. + +==== Debug +* Add support for runtime event validation (buffer endmark checking). Event +validation can be enabled during configure with +`--enable-event-validation [warn/abort]` or with `--enabled-debug=full`. See +`README` for additional information. + == OpenDataPlane (1.39.0.0) === Backward incompatible API changes diff --git a/configure.ac b/configure.ac index 43446c4f3..d496438a8 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ([2.5]) # ODP API version ########################################################################## m4_define([odp_version_generation], [1]) -m4_define([odp_version_major], [39]) +m4_define([odp_version_major], [40]) m4_define([odp_version_minor], [0]) m4_define([odp_version_patch], [0]) -- cgit v1.2.3 From 7b024450c1a1234358fdd8abe0aa67ca15b61ac2 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 23 Jan 2023 10:28:54 +0200 Subject: Port 7f42e20b0 "linux-gen: sched: add config option for adjusting ordered queue stash size" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- config/odp-linux-dpdk.conf | 11 ++++++++++- platform/linux-dpdk/m4/odp_libconfig.m4 | 2 +- platform/linux-dpdk/test/alternate-timer.conf | 2 +- platform/linux-dpdk/test/crypto.conf | 2 +- platform/linux-dpdk/test/process-mode.conf | 2 +- platform/linux-dpdk/test/sched-basic.conf | 3 ++- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/config/odp-linux-dpdk.conf b/config/odp-linux-dpdk.conf index d28f728f2..f0bd808c2 100644 --- a/config/odp-linux-dpdk.conf +++ b/config/odp-linux-dpdk.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.18" +config_file_version = "0.1.19" # System options system: { @@ -192,6 +192,15 @@ sched_basic: { worker = 1 control = 1 } + + # Ordered queue reorder stash size + # + # Number of events each thread can stash internally before having to + # wait for the right order context. Reorder stash can improve + # performance if threads process events in bursts. If 'order_stash_size' + # > 0, events may be dropped by the implementation if the target queue + # is full. To prevent this set 'order_stash_size' to 0. + order_stash_size = 512 } sched_eventdev: { diff --git a/platform/linux-dpdk/m4/odp_libconfig.m4 b/platform/linux-dpdk/m4/odp_libconfig.m4 index 84d331b8a..72bb291ff 100644 --- a/platform/linux-dpdk/m4/odp_libconfig.m4 +++ b/platform/linux-dpdk/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [18]) +m4_define([_odp_config_version_minor], [19]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-dpdk/test/alternate-timer.conf b/platform/linux-dpdk/test/alternate-timer.conf index 0b326f259..cacee9da4 100644 --- a/platform/linux-dpdk/test/alternate-timer.conf +++ b/platform/linux-dpdk/test/alternate-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.18" +config_file_version = "0.1.19" timer: { # Enable alternate DPDK timer implementation diff --git a/platform/linux-dpdk/test/crypto.conf b/platform/linux-dpdk/test/crypto.conf index f3d642963..3b8eec3ff 100644 --- a/platform/linux-dpdk/test/crypto.conf +++ b/platform/linux-dpdk/test/crypto.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.18" +config_file_version = "0.1.19" system: { # One crypto queue pair is required per thread for lockless operation diff --git a/platform/linux-dpdk/test/process-mode.conf b/platform/linux-dpdk/test/process-mode.conf index b95f50ea2..2ded5d706 100644 --- a/platform/linux-dpdk/test/process-mode.conf +++ b/platform/linux-dpdk/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.18" +config_file_version = "0.1.19" dpdk: { process_mode_memory_mb = 1024 diff --git a/platform/linux-dpdk/test/sched-basic.conf b/platform/linux-dpdk/test/sched-basic.conf index 181136c33..fb894bd53 100644 --- a/platform/linux-dpdk/test/sched-basic.conf +++ b/platform/linux-dpdk/test/sched-basic.conf @@ -1,9 +1,10 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.18" +config_file_version = "0.1.19" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { prio_spread = 3 load_balance = 0 + order_stash_size = 0 } -- cgit v1.2.3 From b088933227d1bda95840ca24250bea6cedf71adf Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 23 Jan 2023 10:34:36 +0200 Subject: Port 648c27bfc "api: crypto: split header files" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/Makefile.am | 1 + platform/linux-dpdk/include-abi/odp/api/abi/crypto_types.h | 1 + 2 files changed, 2 insertions(+) create mode 120000 platform/linux-dpdk/include-abi/odp/api/abi/crypto_types.h diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 3bd599bcd..c231dd7af 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -73,6 +73,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/comp.h \ include-abi/odp/api/abi/cpumask.h \ include-abi/odp/api/abi/crypto.h \ + include-abi/odp/api/abi/crypto_types.h \ include-abi/odp/api/abi/debug.h \ include-abi/odp/api/abi/dma_types.h \ include-abi/odp/api/abi/errno.h \ diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/crypto_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/crypto_types.h new file mode 120000 index 000000000..ce973bb2d --- /dev/null +++ b/platform/linux-dpdk/include-abi/odp/api/abi/crypto_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include-abi/odp/api/abi/crypto_types.h \ No newline at end of file -- cgit v1.2.3 From 5739f0286394a35307a4fa088531f3f196711c7e Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 23 Jan 2023 10:48:29 +0200 Subject: Port e2eb952cc "linux-gen: crypto: inline event conversion functions" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/Makefile.am | 2 ++ .../linux-dpdk/include/odp/api/plat/crypto_inlines.h | 1 + platform/linux-dpdk/odp_crypto.c | 16 +--------------- 3 files changed, 4 insertions(+), 15 deletions(-) create mode 120000 platform/linux-dpdk/include/odp/api/plat/crypto_inlines.h diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index c231dd7af..58d1fc1d9 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -32,6 +32,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/buffer_inline_types.h \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ + include/odp/api/plat/crypto_inlines.h \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ @@ -250,6 +251,7 @@ __LIB__libodp_dpdk_la_SOURCES += \ ../linux-generic/odp_buffer_api.c \ ../linux-generic/odp_byteorder_api.c \ ../linux-generic/odp_cpu_api.c \ + ../linux-generic/odp_crypto_api.c \ ../linux-generic/odp_event_api.c \ ../linux-generic/odp_hash_api.c \ ../linux-generic/odp_ipsec_api.c \ diff --git a/platform/linux-dpdk/include/odp/api/plat/crypto_inlines.h b/platform/linux-dpdk/include/odp/api/plat/crypto_inlines.h new file mode 120000 index 000000000..0c1935696 --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/crypto_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/crypto_inlines.h \ No newline at end of file diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 076de1b0e..ea2073029 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -1,5 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited - * Copyright (c) 2018-2022, Nokia + * Copyright (c) 2018-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -1623,20 +1623,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -odp_packet_t odp_crypto_packet_from_event(odp_event_t ev) -{ - /* This check not mandated by the API specification */ - _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET); - _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO); - - return odp_packet_from_event(ev); -} - -odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) -{ - return odp_packet_to_event(pkt); -} - static odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) { -- cgit v1.2.3 From e92112f79ea18f7148b418320ec3215d259e0794 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 23 Jan 2023 10:56:26 +0200 Subject: Port 76e996322 "linux-gen: crypto: inline odp_crypto_result() implementation" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .../include/odp/api/plat/packet_inline_types.h | 3 ++- platform/linux-dpdk/odp_crypto.c | 24 +--------------------- platform/linux-dpdk/odp_packet.c | 3 ++- 3 files changed, 5 insertions(+), 25 deletions(-) diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h index 9a72a70ef..c6ede5d3a 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -44,6 +44,7 @@ typedef struct _odp_packet_inline_offset_t { uint16_t subtype; uint16_t cls_mark; uint16_t ipsec_ctx; + uint16_t crypto_op; uint16_t buf_addr; uint16_t data; uint16_t pkt_len; diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index ea2073029..81643eea7 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -1623,28 +1623,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) return (uint64_t)hdl; } -static -odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) -{ - odp_packet_hdr_t *hdr = packet_hdr(pkt); - - return &hdr->crypto_op_result; -} - -int odp_crypto_result(odp_crypto_packet_result_t *result, - odp_packet_t packet) -{ - odp_crypto_packet_result_t *op_result; - - _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == ODP_EVENT_PACKET_CRYPTO); - - op_result = get_op_result_from_packet(packet); - - memcpy(result, op_result, sizeof(*result)); - - return 0; -} - static uint8_t *crypto_prepare_digest(const crypto_session_entry_t *session, odp_packet_t pkt, const odp_crypto_packet_op_param_t *param, @@ -2131,7 +2109,7 @@ static void op_finish(crypto_op_t *op) /* Fill in result */ packet_subtype_set(pkt, ODP_EVENT_PACKET_CRYPTO); - op_result = get_op_result_from_packet(pkt); + op_result = &packet_hdr(pkt)->crypto_op_result; op_result->cipher_status.alg_err = rc_cipher; op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; op_result->auth_status.alg_err = rc_auth; diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 16b940f2f..ba0b6c908 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -66,6 +66,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .subtype = offsetof(odp_packet_hdr_t, subtype), .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), + .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result), .buf_addr = offsetof(odp_packet_hdr_t, event_hdr.mb.buf_addr), .data = offsetof(odp_packet_hdr_t, event_hdr.mb.data_off), .pkt_len = offsetof(odp_packet_hdr_t, event_hdr.mb.pkt_len), -- cgit v1.2.3 From 34f5c4d86a89dbc084215b4a089b920df1eceaf8 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 23 Jan 2023 10:59:43 +0200 Subject: Port 77d2986ac "linux-gen: loop: add multi-queue support" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/include/odp_packet_io_internal.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index 78b1f343a..2efa52b67 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -53,11 +53,7 @@ ODP_STATIC_ASSERT(PKTIO_LSO_PROFILES < UINT8_MAX, "PKTIO_LSO_PROFILES_ERROR"); /* Forward declaration */ struct pktio_if_ops; -#if ODP_CACHE_LINE_SIZE == 128 -#define PKTIO_PRIVATE_SIZE 1536 -#else -#define PKTIO_PRIVATE_SIZE 1344 -#endif +#define PKTIO_PRIVATE_SIZE 9216 typedef struct ODP_ALIGNED_CACHE { const struct pktio_if_ops *ops; /**< Implementation specific methods */ -- cgit v1.2.3 From 1c2db3e8073b83905a3686db4d411c1ba24aa3ca Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 23 Jan 2023 13:07:44 +0200 Subject: Port 579d81057 "api: buffer: split header files" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/Makefile.am | 1 + platform/linux-dpdk/include-abi/odp/api/abi/buffer_types.h | 1 + platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h | 3 +-- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 120000 platform/linux-dpdk/include-abi/odp/api/abi/buffer_types.h diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 58d1fc1d9..31148168b 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -69,6 +69,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/atomic.h \ include-abi/odp/api/abi/barrier.h \ include-abi/odp/api/abi/buffer.h \ + include-abi/odp/api/abi/buffer_types.h \ include-abi/odp/api/abi/byteorder.h \ include-abi/odp/api/abi/classification.h \ include-abi/odp/api/abi/comp.h \ diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/buffer_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/buffer_types.h new file mode 120000 index 000000000..49e88bd2e --- /dev/null +++ b/platform/linux-dpdk/include-abi/odp/api/abi/buffer_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include-abi/odp/api/abi/buffer_types.h \ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h index 6265914b4..37a3bf296 100644 --- a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h @@ -7,12 +7,11 @@ #ifndef ODP_PLAT_BUFFER_INLINES_H_ #define ODP_PLAT_BUFFER_INLINES_H_ +#include #include #include #include -#include - #include #include #include -- cgit v1.2.3 From ff05084f1eb44029112b206f01bf6fa502a10c40 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 23 Jan 2023 15:31:17 +0200 Subject: Port 8782c99a6 "linux-gen: queue: header include clean-up" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/odp_queue_basic.c | 47 +++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index a83164e2f..ebe86ef1b 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -1,43 +1,46 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include #include -#include +#include #include -#include -#include + #include +#include +#include + +#include +#include +#include #include +#include +#include +#include +#include +#include #include -#include +#include +#include +#include + +#include +#include -#include #define LOCK(queue_ptr) odp_ticketlock_lock(&((queue_ptr)->lock)) #define UNLOCK(queue_ptr) odp_ticketlock_unlock(&((queue_ptr)->lock)) #define LOCK_INIT(queue_ptr) odp_ticketlock_init(&((queue_ptr)->lock)) -#include -#include - #define MIN_QUEUE_SIZE (32 - 1) #define MAX_QUEUE_SIZE ((1 * 1024 * 1024) - 1) -- cgit v1.2.3 From 3f0a93e790849b69fc3a99649880f7c6821ebf4c Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 23 Jan 2023 10:43:01 +0200 Subject: Port a735a7131 "linux-gen: event: add event validation support" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/Makefile.am | 3 ++ .../include/odp/api/plat/buffer_inlines.h | 10 ++++++- .../odp/api/plat/event_validation_external.h | 1 + .../include/odp/api/plat/packet_inlines.h | 11 ++++++-- platform/linux-dpdk/include/odp_event_internal.h | 9 +++++- platform/linux-dpdk/include/odp_pool_internal.h | 5 ++-- platform/linux-dpdk/m4/configure.m4 | 3 ++ platform/linux-dpdk/m4/odp_event_validation.m4 | 1 + platform/linux-dpdk/odp_event.c | 25 ++++++++++------ platform/linux-dpdk/odp_init.c | 16 ++++++++++- platform/linux-dpdk/odp_packet.c | 4 +++ platform/linux-dpdk/odp_pool.c | 33 +++++++++++++++------- 12 files changed, 95 insertions(+), 26 deletions(-) create mode 120000 platform/linux-dpdk/include/odp/api/plat/event_validation_external.h create mode 120000 platform/linux-dpdk/m4/odp_event_validation.m4 diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 31148168b..7f45d7683 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -36,6 +36,7 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/debug_inlines.h \ include/odp/api/plat/event_inlines.h \ include/odp/api/plat/event_inline_types.h \ + include/odp/api/plat/event_validation_external.h \ include/odp/api/plat/event_vector_inline_types.h \ include/odp/api/plat/hash_inlines.h \ include/odp/api/plat/ipsec_inlines.h \ @@ -144,6 +145,7 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_print_internal.h \ include/odp_errno_define.h \ include/odp_event_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_event_validation_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_packet_dpdk.h \ ${top_srcdir}/platform/linux-generic/include/odp_pcapng.h \ ${top_srcdir}/platform/linux-generic/include/odp_pkt_queue_internal.h \ @@ -200,6 +202,7 @@ __LIB__libodp_dpdk_la_SOURCES = \ odp_crypto.c \ odp_errno.c \ odp_event.c \ + ../linux-generic/odp_event_validation.c \ ../linux-generic/odp_hash_crc_gen.c \ odp_init.c \ ../linux-generic/odp_impl.c \ diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h index 37a3bf296..396d78559 100644 --- a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019-2022, Nokia +/* Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -94,6 +95,8 @@ _ODP_INLINE void odp_buffer_free(odp_buffer_t buf) { struct rte_mbuf *mbuf = (struct rte_mbuf *)buf; + _odp_buffer_validate(buf, _ODP_EV_BUFFER_FREE); + rte_mempool_put(mbuf->pool, mbuf); } @@ -106,6 +109,8 @@ _ODP_INLINE void odp_buffer_free_multi(const odp_buffer_t buf[], int num) if (odp_unlikely(num <= 0)) return; + _odp_buffer_validate_multi(buf, num, _ODP_EV_BUFFER_FREE_MULTI); + mbuf_tbl[0] = (struct rte_mbuf *)buf[0]; mp_pending = mbuf_tbl[0]->pool; num_pending = 1; @@ -133,6 +138,9 @@ _ODP_INLINE int odp_buffer_is_valid(odp_buffer_t buf) if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER) return 0; + if (odp_unlikely(_odp_buffer_validate(buf, _ODP_EV_BUFFER_IS_VALID))) + return 0; + return 1; } diff --git a/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h b/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h new file mode 120000 index 000000000..001662d8b --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/event_validation_external.h \ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h index 4347afb88..4ae53c7b3 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,6 +25,7 @@ extern "C" { #include #include +#include #include #include #include @@ -663,17 +664,23 @@ _ODP_INLINE odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_co _ODP_INLINE void odp_packet_free(odp_packet_t pkt) { + _odp_packet_validate(pkt, _ODP_EV_PACKET_FREE); + rte_pktmbuf_free((struct rte_mbuf *)pkt); } _ODP_INLINE void odp_packet_free_multi(const odp_packet_t pkt[], int num) { + _odp_packet_validate_multi(pkt, num, _ODP_EV_PACKET_FREE_MULTI); + rte_pktmbuf_free_bulk((struct rte_mbuf **)(uintptr_t)pkt, (unsigned int)num); } _ODP_INLINE void odp_packet_free_sp(const odp_packet_t pkt[], int num) { - odp_packet_free_multi(pkt, num); + _odp_packet_validate_multi(pkt, num, _ODP_EV_PACKET_FREE_SP); + + rte_pktmbuf_free_bulk((struct rte_mbuf **)(uintptr_t)pkt, (unsigned int)num); } _ODP_INLINE void *odp_packet_seg_data(odp_packet_t pkt ODP_UNUSED, diff --git a/platform/linux-dpdk/include/odp_event_internal.h b/platform/linux-dpdk/include/odp_event_internal.h index 320f42146..5ccf5a3ed 100644 --- a/platform/linux-dpdk/include/odp_event_internal.h +++ b/platform/linux-dpdk/include/odp_event_internal.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2021-2022, Nokia +/* Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -80,6 +80,13 @@ static inline void _odp_event_type_set(odp_event_t event, int ev) _odp_event_hdr(event)->event_type = ev; } +static inline uint64_t *_odp_event_endmark_get_ptr(odp_event_t event) +{ + struct rte_mbuf *mbuf = _odp_event_to_mbuf(event); + + return (uint64_t *)((uint8_t *)mbuf->buf_addr + mbuf->buf_len); +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index a5ffcf517..b82324a77 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -72,7 +72,7 @@ typedef struct ODP_ALIGNED_CACHE { /* Everything under this mark is memset() to zero on pool create */ uint8_t memset_mark; struct rte_mempool *rte_mempool; - uint32_t seg_len; + uint32_t seg_len; /* Initial packet segment length (excludes endmark) */ uint32_t ext_head_offset; uint32_t num; uint32_t num_populated; @@ -84,6 +84,7 @@ typedef struct ODP_ALIGNED_CACHE { odp_shm_t uarea_shm; uint64_t uarea_shm_size; uint32_t uarea_size; + uint32_t trailer_size; /* Endmark size */ uint8_t *uarea_base_addr; char name[ODP_POOL_NAME_LEN]; diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index 90d55b6f6..70295400f 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -5,11 +5,13 @@ ODP_VISIBILITY ODP_ATOMIC m4_include([platform/linux-dpdk/m4/odp_cpu.m4]) +m4_include([platform/linux-dpdk/m4/odp_event_validation.m4]) m4_include([platform/linux-dpdk/m4/odp_libconfig.m4]) m4_include([platform/linux-dpdk/m4/odp_openssl.m4]) m4_include([platform/linux-dpdk/m4/odp_pcapng.m4]) m4_include([platform/linux-dpdk/m4/odp_scheduler.m4]) +ODP_EVENT_VALIDATION ODP_PTHREAD ODP_SCHEDULER ODP_TIMER @@ -68,6 +70,7 @@ AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. AS_VAR_APPEND([PLAT_CFG_TEXT], [" + event_validation: ${enable_event_validation} openssl: ${with_openssl} openssl_rand: ${openssl_rand} pcap: ${have_pmd_pcap} diff --git a/platform/linux-dpdk/m4/odp_event_validation.m4 b/platform/linux-dpdk/m4/odp_event_validation.m4 new file mode 120000 index 000000000..0d457c6ff --- /dev/null +++ b/platform/linux-dpdk/m4/odp_event_validation.m4 @@ -0,0 +1 @@ +../../linux-generic/m4/odp_event_validation.m4 \ No newline at end of file diff --git a/platform/linux-dpdk/odp_event.c b/platform/linux-dpdk/odp_event.c index 24853f83e..625b6eefe 100644 --- a/platform/linux-dpdk/odp_event.c +++ b/platform/linux-dpdk/odp_event.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -18,6 +18,7 @@ #include #include #include +#include #include /* Inlined API functions */ @@ -42,13 +43,15 @@ _odp_event_inline_offset ODP_ALIGNED_CACHE = { #include -void odp_event_free(odp_event_t event) +static inline void event_free(odp_event_t event, _odp_ev_id_t id) { switch (odp_event_type(event)) { case ODP_EVENT_BUFFER: + _odp_buffer_validate(odp_buffer_from_event(event), id); odp_buffer_free(odp_buffer_from_event(event)); break; case ODP_EVENT_PACKET: + _odp_packet_validate(odp_packet_from_event(event), id); odp_packet_free(odp_packet_from_event(event)); break; case ODP_EVENT_PACKET_VECTOR: @@ -76,17 +79,21 @@ void odp_event_free(odp_event_t event) } } -void odp_event_free_multi(const odp_event_t event[], int num) +void odp_event_free(odp_event_t event) { - int i; + event_free(event, _ODP_EV_EVENT_FREE); +} - for (i = 0; i < num; i++) - odp_event_free(event[i]); +void odp_event_free_multi(const odp_event_t event[], int num) +{ + for (int i = 0; i < num; i++) + event_free(event[i], _ODP_EV_EVENT_FREE_MULTI); } void odp_event_free_sp(const odp_event_t event[], int num) { - odp_event_free_multi(event, num); + for (int i = 0; i < num; i++) + event_free(event[i], _ODP_EV_EVENT_FREE_SP); } uint64_t odp_event_to_u64(odp_event_t hdl) @@ -104,9 +111,9 @@ int odp_event_is_valid(odp_event_t event) switch (odp_event_type(event)) { case ODP_EVENT_BUFFER: - /* Fall through */ + return !_odp_buffer_validate(odp_buffer_from_event(event), _ODP_EV_EVENT_IS_VALID); case ODP_EVENT_PACKET: - /* Fall through */ + return !_odp_packet_validate(odp_packet_from_event(event), _ODP_EV_EVENT_IS_VALID); case ODP_EVENT_TIMEOUT: /* Fall through */ #if ODP_DEPRECATED_API diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index 329252f9c..b144835ea 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -41,6 +41,7 @@ enum init_stage { HASH_INIT, THREAD_INIT, POOL_INIT, + EVENT_VALIDATION_INIT, STASH_INIT, QUEUE_INIT, SCHED_INIT, @@ -426,6 +427,13 @@ static int term_global(enum init_stage stage) } /* Fall through */ + case EVENT_VALIDATION_INIT: + if (_odp_event_validation_term_global()) { + _ODP_ERR("ODP event validation term failed.\n"); + rc = -1; + } + /* Fall through */ + case POOL_INIT: if (_odp_pool_term_global()) { _ODP_ERR("ODP buffer pool term failed.\n"); @@ -590,6 +598,12 @@ int odp_init_global(odp_instance_t *instance, } stage = POOL_INIT; + if (_odp_event_validation_init_global()) { + _ODP_ERR("ODP event validation init failed.\n"); + goto init_failed; + } + stage = EVENT_VALIDATION_INIT; + if (_odp_stash_init_global()) { _ODP_ERR("ODP stash init failed.\n"); goto init_failed; diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index ba0b6c908..9b8755ce9 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1145,6 +1146,9 @@ int odp_packet_is_valid(odp_packet_t pkt) if (odp_event_type(ev) != ODP_EVENT_PACKET) return 0; + if (odp_unlikely(_odp_packet_validate(pkt, _ODP_EV_PACKET_IS_VALID))) + return 0; + switch (odp_event_subtype(ev)) { case ODP_EVENT_PACKET_BASIC: /* Fall through */ diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index b0df7997a..d36f197c0 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -380,6 +382,12 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp, pkt_hdr->uarea_addr = uarea; } + /* Initialize data endmark */ + if (mb_ctor_arg->type == ODP_POOL_BUFFER || mb_ctor_arg->type == ODP_POOL_PACKET) { + mb->buf_len -= _ODP_EV_ENDMARK_SIZE; + _odp_event_endmark_set(_odp_event_from_mbuf(mb)); + } + /* Initialize event vector metadata */ if (mb_ctor_arg->type == ODP_POOL_VECTOR) { odp_event_vector_hdr_t *vect_hdr = (void *)raw_mbuf; @@ -679,6 +687,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, uint32_t buf_align, blk_size, headroom, tailroom, min_seg_len; uint32_t max_len, min_align; uint32_t uarea_size = 0; + uint32_t trailer_size = 0; int8_t event_type; char pool_name[ODP_POOL_NAME_LEN]; char rte_name[RTE_MEMPOOL_NAMESIZE]; @@ -709,7 +718,8 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, switch (type) { case ODP_POOL_BUFFER: buf_align = params->buf.align; - blk_size = params->buf.size; + trailer_size = _ODP_EV_ENDMARK_SIZE; + blk_size = params->buf.size + trailer_size; cache_size = params->buf.cache_size; uarea_size = params->buf.uarea_size; @@ -734,6 +744,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, case ODP_POOL_PACKET: headroom = RTE_PKTMBUF_HEADROOM; tailroom = CONFIG_PACKET_TAILROOM; + trailer_size = _ODP_EV_ENDMARK_SIZE; min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; min_align = ODP_CONFIG_BUFFER_ALIGN_MIN; cache_size = params->pkt.cache_size; @@ -743,16 +754,16 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, blk_size = params->pkt.seg_len; if (params->pkt.len > blk_size) blk_size = params->pkt.len; - /* Make sure at least one max len packet fits in the - * pool. - */ + + /* Make sure at least one max len packet fits in the pool */ max_len = 0; if (params->pkt.max_len != 0) max_len = params->pkt.max_len; if ((max_len + blk_size) / blk_size > params->pkt.num) blk_size = (max_len + params->pkt.num) / params->pkt.num; - blk_size = _ODP_ROUNDUP_ALIGN(headroom + blk_size + tailroom, min_align); + blk_size = _ODP_ROUNDUP_ALIGN(headroom + blk_size + tailroom + trailer_size, + min_align); /* Segment size minus headroom might be rounded down by the driver (e.g. * ixgbe) to the nearest multiple of 1024. Round it up here to make sure the @@ -827,7 +838,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, mp = rte_pktmbuf_pool_create(rte_name, num, cache_size, priv_size, data_room_size, rte_socket_id()); - pool->seg_len = data_room_size; + pool->seg_len = data_room_size - trailer_size; } else { unsigned int priv_size; @@ -855,6 +866,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, pool->type = type; pool->type_2 = type_2; pool->params = *params; + pool->trailer_size = trailer_size; if (reserve_uarea(pool, uarea_size, num)) { _ODP_ERR("User area SHM reserve failed\n"); @@ -1157,7 +1169,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, capa->pkt.max_num_buf = _odp_pool_glb->config.pkt_max_num; capa->pkt.max_buf_size = MAX_SIZE; capa->pkt.odp_header_size = SIZEOF_OBJHDR + sizeof(odp_packet_hdr_t); - capa->pkt.odp_trailer_size = 0; + capa->pkt.odp_trailer_size = _ODP_EV_ENDMARK_SIZE; capa->pkt.min_mem_align = ODP_CACHE_LINE_SIZE; capa->pkt.min_buf_align = ODP_CACHE_LINE_SIZE; capa->pkt.min_head_align = EXT_MIN_HEAD_ALIGN; @@ -1326,12 +1338,13 @@ odp_pool_t odp_pool_ext_create(const char *name, pool->ext_param = *params; pool->ext_head_offset = hdr_size; + pool->trailer_size = _ODP_EV_ENDMARK_SIZE; pool->num = num; pool->num_populated = 0; pool->params.pkt.uarea_size = params->pkt.uarea_size; pool->params.type = params->type; pool->pool_ext = 1; - pool->seg_len = blk_size; + pool->seg_len = blk_size - pool->trailer_size; pool->type = params->type; strcpy(pool->name, pool_name); @@ -1449,7 +1462,7 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, mb_ctor_arg.seg_buf_offset = sizeof(odp_packet_hdr_t) + params->pkt.app_header_size; - mb_ctor_arg.seg_buf_size = pool->seg_len; + mb_ctor_arg.seg_buf_size = pool->seg_len + pool->trailer_size; mb_ctor_arg.type = params->type; mb_ctor_arg.event_type = pool->type; mb_ctor_arg.pool = pool; -- cgit v1.2.3 From 3be70c20179bffd14f479908a40fb79673490cef Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 16 Feb 2023 16:17:20 +0200 Subject: Port 8aee6094e "linux-gen: stash: reserve all memory from a single shm block" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- config/odp-linux-dpdk.conf | 13 ++++++++++++- platform/linux-dpdk/include/odp_config_internal.h | 10 +++++----- platform/linux-dpdk/m4/odp_libconfig.m4 | 2 +- platform/linux-dpdk/test/alternate-timer.conf | 2 +- platform/linux-dpdk/test/crypto.conf | 2 +- platform/linux-dpdk/test/process-mode.conf | 2 +- platform/linux-dpdk/test/sched-basic.conf | 2 +- 7 files changed, 22 insertions(+), 11 deletions(-) diff --git a/config/odp-linux-dpdk.conf b/config/odp-linux-dpdk.conf index f0bd808c2..1f868a0fe 100644 --- a/config/odp-linux-dpdk.conf +++ b/config/odp-linux-dpdk.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.19" +config_file_version = "0.1.20" # System options system: { @@ -218,6 +218,17 @@ sched_eventdev: { num_ports = 0 } +stash: { + # Maximum number of stashes + max_num = 512 + + # Maximum number of objects in a stash + # + # The value may be rounded up by the implementation. For optimal memory + # usage set value to a power of two - 1. + max_num_obj = 4095 +} + timer: { # Inline timer poll interval # diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h index 18754184e..e104b9d24 100644 --- a/platform/linux-dpdk/include/odp_config_internal.h +++ b/platform/linux-dpdk/include/odp_config_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2020-2021, Nokia + * Copyright (c) 2020-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -70,7 +70,7 @@ extern "C" { /* * Maximum number of stashes */ -#define CONFIG_MAX_STASHES 128 +#define CONFIG_MAX_STASHES 2048 /* * Maximum number of packet IO resources @@ -133,10 +133,10 @@ extern "C" { /* * Number of shared memory blocks reserved for implementation internal use. * - * Each stash and packet pool requires one SHM block, and 20 blocks are - * reserved for per ODP module global data. + * Each packet pool requires one SHM block and 20 blocks are reserved for + * ODP module global data. */ -#define CONFIG_INTERNAL_SHM_BLOCKS (CONFIG_MAX_STASHES + ODP_CONFIG_POOLS + 20) +#define CONFIG_INTERNAL_SHM_BLOCKS (ODP_CONFIG_POOLS + 20) /* * Maximum number of shared memory blocks. diff --git a/platform/linux-dpdk/m4/odp_libconfig.m4 b/platform/linux-dpdk/m4/odp_libconfig.m4 index 72bb291ff..fef0518a1 100644 --- a/platform/linux-dpdk/m4/odp_libconfig.m4 +++ b/platform/linux-dpdk/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [19]) +m4_define([_odp_config_version_minor], [20]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-dpdk/test/alternate-timer.conf b/platform/linux-dpdk/test/alternate-timer.conf index cacee9da4..c28c64f2e 100644 --- a/platform/linux-dpdk/test/alternate-timer.conf +++ b/platform/linux-dpdk/test/alternate-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.19" +config_file_version = "0.1.20" timer: { # Enable alternate DPDK timer implementation diff --git a/platform/linux-dpdk/test/crypto.conf b/platform/linux-dpdk/test/crypto.conf index 3b8eec3ff..b775d707e 100644 --- a/platform/linux-dpdk/test/crypto.conf +++ b/platform/linux-dpdk/test/crypto.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.19" +config_file_version = "0.1.20" system: { # One crypto queue pair is required per thread for lockless operation diff --git a/platform/linux-dpdk/test/process-mode.conf b/platform/linux-dpdk/test/process-mode.conf index 2ded5d706..1c0b34d89 100644 --- a/platform/linux-dpdk/test/process-mode.conf +++ b/platform/linux-dpdk/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.19" +config_file_version = "0.1.20" dpdk: { process_mode_memory_mb = 1024 diff --git a/platform/linux-dpdk/test/sched-basic.conf b/platform/linux-dpdk/test/sched-basic.conf index fb894bd53..d5f6f1f60 100644 --- a/platform/linux-dpdk/test/sched-basic.conf +++ b/platform/linux-dpdk/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.19" +config_file_version = "0.1.20" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { -- cgit v1.2.3 From 9548231374b7f9860a7dd2022015ec1606c2408a Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 16 Feb 2023 16:19:41 +0200 Subject: Port d74fabe09 "api: timer: use deprecate macro for odp_timer_set_rel()" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/odp_timer.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c index 5296a9d3c..1021a96cd 100644 --- a/platform/linux-dpdk/odp_timer.c +++ b/platform/linux-dpdk/odp_timer.c @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,6 +7,7 @@ #include +#include #include #include #include @@ -1015,14 +1016,14 @@ retry: return ODP_TIMER_SUCCESS; } -int odp_timer_set_abs(odp_timer_t timer_hdl, uint64_t abs_tick, - odp_event_t *tmo_ev) +int ODP_DEPRECATE(odp_timer_set_abs)(odp_timer_t timer_hdl, uint64_t abs_tick, + odp_event_t *tmo_ev) { return timer_set(timer_hdl, abs_tick, tmo_ev, 1); } -int odp_timer_set_rel(odp_timer_t timer_hdl, uint64_t rel_tick, - odp_event_t *tmo_ev) +int ODP_DEPRECATE(odp_timer_set_rel)(odp_timer_t timer_hdl, uint64_t rel_tick, + odp_event_t *tmo_ev) { return timer_set(timer_hdl, rel_tick, tmo_ev, 0); } -- cgit v1.2.3 From ab53938d5f4d2588bde753430ab77a64ab4f2a33 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 16 Feb 2023 16:31:33 +0200 Subject: Port 9079409fc "linux-gen: ring: add 64-bit variant of mpmc ring" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/Makefile.am | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 7f45d7683..76e205212 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -159,6 +159,8 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_ring_common.h \ ${top_srcdir}/platform/linux-generic/include/odp_ring_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_ring_mpmc_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_ring_ptr_internal.h \ include/odp_ptr_ring_mpmc_internal.h \ include/odp_ptr_ring_spsc_internal.h \ -- cgit v1.2.3 From 74bbd7ba8c0673f68d47b6b78d0012160ed78bbb Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 16 Feb 2023 16:40:30 +0200 Subject: Port 151f650e8 "linux-gen: stash: add mpmc ring based implementation" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- config/odp-linux-dpdk.conf | 8 +++++++- platform/linux-dpdk/m4/odp_libconfig.m4 | 2 +- platform/linux-dpdk/test/alternate-timer.conf | 2 +- platform/linux-dpdk/test/crypto.conf | 2 +- platform/linux-dpdk/test/process-mode.conf | 2 +- platform/linux-dpdk/test/sched-basic.conf | 2 +- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/config/odp-linux-dpdk.conf b/config/odp-linux-dpdk.conf index 1f868a0fe..8fe298e5b 100644 --- a/config/odp-linux-dpdk.conf +++ b/config/odp-linux-dpdk.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.20" +config_file_version = "0.1.21" # System options system: { @@ -227,6 +227,12 @@ stash: { # The value may be rounded up by the implementation. For optimal memory # usage set value to a power of two - 1. max_num_obj = 4095 + + # Strict size + # + # When set to 0, application can attempt to store more handles into a + # stash than it specified in the creation parameters. + strict_size = 1 } timer: { diff --git a/platform/linux-dpdk/m4/odp_libconfig.m4 b/platform/linux-dpdk/m4/odp_libconfig.m4 index fef0518a1..1d56f0a5c 100644 --- a/platform/linux-dpdk/m4/odp_libconfig.m4 +++ b/platform/linux-dpdk/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [20]) +m4_define([_odp_config_version_minor], [21]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-dpdk/test/alternate-timer.conf b/platform/linux-dpdk/test/alternate-timer.conf index c28c64f2e..d976d05fe 100644 --- a/platform/linux-dpdk/test/alternate-timer.conf +++ b/platform/linux-dpdk/test/alternate-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.20" +config_file_version = "0.1.21" timer: { # Enable alternate DPDK timer implementation diff --git a/platform/linux-dpdk/test/crypto.conf b/platform/linux-dpdk/test/crypto.conf index b775d707e..1fcaf658e 100644 --- a/platform/linux-dpdk/test/crypto.conf +++ b/platform/linux-dpdk/test/crypto.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.20" +config_file_version = "0.1.21" system: { # One crypto queue pair is required per thread for lockless operation diff --git a/platform/linux-dpdk/test/process-mode.conf b/platform/linux-dpdk/test/process-mode.conf index 1c0b34d89..7a98cef10 100644 --- a/platform/linux-dpdk/test/process-mode.conf +++ b/platform/linux-dpdk/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.20" +config_file_version = "0.1.21" dpdk: { process_mode_memory_mb = 1024 diff --git a/platform/linux-dpdk/test/sched-basic.conf b/platform/linux-dpdk/test/sched-basic.conf index d5f6f1f60..e3d9dd304 100644 --- a/platform/linux-dpdk/test/sched-basic.conf +++ b/platform/linux-dpdk/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.20" +config_file_version = "0.1.21" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { -- cgit v1.2.3 From dfb4f9cffb8d47cb8b975a8a06b56900b233daa0 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 16 Feb 2023 16:42:32 +0200 Subject: Port 213684797 "github_ci: add test case for custom stash config" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/test/stash-custom.conf | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 platform/linux-dpdk/test/stash-custom.conf diff --git a/platform/linux-dpdk/test/stash-custom.conf b/platform/linux-dpdk/test/stash-custom.conf new file mode 100644 index 000000000..5e3ea5cdf --- /dev/null +++ b/platform/linux-dpdk/test/stash-custom.conf @@ -0,0 +1,8 @@ +# Mandatory fields +odp_implementation = "linux-dpdk" +config_file_version = "0.1.21" + +# Test overflow safe stash variant +stash: { + strict_size = 0 +} -- cgit v1.2.3 From 3896744591a741bf0017d5445bc94d4e09e89a15 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 16 Feb 2023 16:58:12 +0200 Subject: Port 721e08c6f "api: shm: remove deprecated ODP_SHM_SW_ONLY define" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/odp_shared_memory.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/platform/linux-dpdk/odp_shared_memory.c b/platform/linux-dpdk/odp_shared_memory.c index 470b78eb5..871a22ed7 100644 --- a/platform/linux-dpdk/odp_shared_memory.c +++ b/platform/linux-dpdk/odp_shared_memory.c @@ -1,5 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia + * Copyright (c) 2021-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -32,14 +31,7 @@ #include /* Supported ODP_SHM_* flags */ -#if ODP_DEPRECATED_API - #define DEPRECATED_SHM_FLAGS (ODP_SHM_SW_ONLY) -#else - #define DEPRECATED_SHM_FLAGS 0 -#endif - -#define SUPPORTED_SHM_FLAGS (ODP_SHM_EXPORT | ODP_SHM_HP | \ - ODP_SHM_SINGLE_VA | DEPRECATED_SHM_FLAGS) +#define SUPPORTED_SHM_FLAGS (ODP_SHM_EXPORT | ODP_SHM_HP | ODP_SHM_SINGLE_VA) #define SHM_MAX_ALIGN (0x80000000) #define SHM_BLOCK_NAME "%" PRIu64 "-%d-%s" @@ -299,8 +291,7 @@ odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align, block->type = SHM_TYPE_LOCAL; block->size = size; - /* Note: ODP_SHM_SW_ONLY/ODP_SHM_PROC/ODP_SHM_SINGLE_VA flags are - * currently ignored. */ + /* Note: ODP_SHM_PROC/ODP_SHM_SINGLE_VA flags are currently ignored. */ shm_zone(mz)->flags = flags; odp_spinlock_unlock(&shm_tbl->lock); -- cgit v1.2.3 From f955acc7c5dbe69236f38e21710595eedba98c13 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 16 Feb 2023 17:01:23 +0200 Subject: Port a057dd7db "linux-gen: packet: update packet vlan flag set function implementations" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-dpdk/odp_packet_flags.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platform/linux-dpdk/odp_packet_flags.c b/platform/linux-dpdk/odp_packet_flags.c index 728df157a..345ac8488 100644 --- a/platform/linux-dpdk/odp_packet_flags.c +++ b/platform/linux-dpdk/odp_packet_flags.c @@ -72,10 +72,12 @@ void odp_packet_has_jumbo_set(odp_packet_t pkt, int val) void odp_packet_has_vlan_set(odp_packet_t pkt, int val) { setflag(pkt, input_flags.vlan, val); + setflag(pkt, input_flags.vlan_qinq, 0); } void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val) { + setflag(pkt, input_flags.vlan, val); setflag(pkt, input_flags.vlan_qinq, val); } -- cgit v1.2.3 From ae7c5263b0c60884918ab6be342093ad4dcf1758 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 16 Feb 2023 17:16:10 +0200 Subject: Port 5054ec2ca "linux-gen: crypto: implement out-of-place crypto operation type" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- platform/linux-dpdk/odp_crypto.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 81643eea7..159a66966 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -1395,6 +1395,13 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, return -1; } + /* ODP_CRYPTO_OP_TYPE_OOP not supported */ + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) { + *status = ODP_CRYPTO_SES_ERR_PARAMS; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + if (rte_cryptodev_count() == 0) { _ODP_ERR("No crypto devices available\n"); *status = ODP_CRYPTO_SES_ERR_ENOMEM; -- cgit v1.2.3 From 5654b4534e9c78751b7ef118aa1d5abb82b2bdc5 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 17 Feb 2023 10:50:15 +0200 Subject: Port 2280e91a1 "linux-gen: crypto: implement basic crypto operation type" Port original commit from linux-generic. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- platform/linux-dpdk/odp_crypto.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 159a66966..755d9dfff 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -1836,6 +1836,9 @@ static odp_packet_t get_output_packet(const crypto_session_entry_t *session, { int rc; + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) + return pkt_in; + if (odp_likely(pkt_in == pkt_out)) return pkt_out; @@ -2159,6 +2162,10 @@ int odp_crypto_operation(odp_crypto_op_param_t *param, odp_crypto_op_result_t local_result; int rc; + if (((crypto_session_entry_t *)(intptr_t)param->session)->p.op_type != + ODP_CRYPTO_OP_TYPE_LEGACY) + return -1; + packet_param.session = param->session; packet_param.cipher_iv_ptr = param->cipher_iv_ptr; packet_param.auth_iv_ptr = param->auth_iv_ptr; @@ -2239,7 +2246,8 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[], _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); - out_pkts[i] = pkt_out[i]; + if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC) + out_pkts[i] = pkt_out[i]; } num_pkt = odp_crypto_int(pkt_in, out_pkts, param, num_pkt); -- cgit v1.2.3 From c7c99cfd5635da8c0afc7f3803ce6e1c82c3465b Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 16 Feb 2023 16:56:15 +0200 Subject: linux-dpdk: crypto: remove deprecated per-session IVs Stop supporting deprecated per-session IVs. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- platform/linux-dpdk/odp_crypto.c | 49 ++-------------------------------------- 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index 755d9dfff..713419a14 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -83,10 +83,7 @@ typedef struct crypto_session_entry_s { unsigned int chained_bufs_ok:1; } flags; uint8_t cdev_id; -#if ODP_DEPRECATED_API - uint8_t cipher_iv_data[MAX_IV_LENGTH]; - uint8_t auth_iv_data[MAX_IV_LENGTH]; -#endif + } crypto_session_entry_t; typedef struct crypto_global_s { @@ -1503,16 +1500,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param, out_null: session->rte_session = rte_session; session->cdev_id = cdev_id; -#if ODP_DEPRECATED_API - if (param->cipher_iv.data) - memcpy(session->cipher_iv_data, - param->cipher_iv.data, - param->cipher_iv.length); - if (param->auth_iv.data) - memcpy(session->auth_iv_data, - param->auth_iv.data, - param->auth_iv.length); -#endif + /* We're happy */ *session_out = (intptr_t)session; *status = ODP_CRYPTO_SES_ERR_NONE; @@ -1689,17 +1677,8 @@ static void crypto_fill_aead_param(const crypto_session_entry_t *session, iv_ptr++; } -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) - memcpy(iv_ptr, param->cipher_iv_ptr, iv_len); - else if (session->p.cipher_iv.data) - memcpy(iv_ptr, session->cipher_iv_data, iv_len); - else - _ODP_ASSERT(iv_len == 0); -#else _ODP_ASSERT(iv_len == 0 || param->cipher_iv_ptr != NULL); memcpy(iv_ptr, param->cipher_iv_ptr, iv_len); -#endif op->sym->aead.data.offset = param->cipher_range.offset; op->sym->aead.data.length = param->cipher_range.length; @@ -1723,29 +1702,6 @@ static void crypto_fill_sym_param(const crypto_session_entry_t *session, &op->sym->auth.digest.phys_addr); } -#if ODP_DEPRECATED_API - if (param->cipher_iv_ptr) { - iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); - memcpy(iv_ptr, param->cipher_iv_ptr, cipher_iv_len); - } else if (session->p.cipher_iv.data) { - iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); - memcpy(iv_ptr, session->cipher_iv_data, cipher_iv_len); - } else { - _ODP_ASSERT(cipher_iv_len == 0); - } - - if (param->auth_iv_ptr) { - iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, - IV_OFFSET + MAX_IV_LENGTH); - memcpy(iv_ptr, param->auth_iv_ptr, auth_iv_len); - } else if (session->p.auth_iv.data) { - iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, - IV_OFFSET + MAX_IV_LENGTH); - memcpy(iv_ptr, session->auth_iv_data, auth_iv_len); - } else { - _ODP_ASSERT(auth_iv_len == 0); - } -#else _ODP_ASSERT(cipher_iv_len == 0 || param->cipher_iv_ptr != NULL); _ODP_ASSERT(auth_iv_len == 0 || param->auth_iv_ptr != NULL); iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); @@ -1755,7 +1711,6 @@ static void crypto_fill_sym_param(const crypto_session_entry_t *session, iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET + MAX_IV_LENGTH); memcpy(iv_ptr, param->auth_iv_ptr, auth_iv_len); } -#endif op->sym->cipher.data.offset = param->cipher_range.offset; op->sym->cipher.data.length = param->cipher_range.length; -- cgit v1.2.3