wolfssl-sys 4.0.0

/* linuxkm_wc_port.h
 *
 * Copyright (C) 2006-2026 wolfSSL Inc.
 *
 * This file is part of wolfSSL.
 *
 * wolfSSL is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * wolfSSL is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
 */

/* included by wolfssl/wolfcrypt/wc_port.h */

#ifndef LINUXKM_WC_PORT_H
#define LINUXKM_WC_PORT_H

    #if defined(WOLFSSL_KERNEL_VERBOSE_DEBUG) && !defined(WOLFSSL_LINUXKM_VERBOSE_DEBUG)
        #define WOLFSSL_LINUXKM_VERBOSE_DEBUG
    #endif

    #include <linux/version.h>
    #include <linux/kconfig.h>

    #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
        #error Unsupported kernel.
    #endif

    #if defined(HAVE_FIPS) && defined(LINUXKM_LKCAPI_REGISTER_AESXTS) && defined(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS)
        /* CONFIG_CRYPTO_MANAGER_EXTRA_TESTS expects AES-XTS-384 to work, even when CONFIG_CRYPTO_FIPS, but FIPS 140-3 only allows AES-XTS-256 and AES-XTS-512. */
        #error CONFIG_CRYPTO_MANAGER_EXTRA_TESTS is incompatible with FIPS wolfCrypt AES-XTS -- please reconfigure the target kernel to disable CONFIG_CRYPTO_MANAGER_EXTRA_TESTS.
    #endif

    /* The first vector set in /usr/src/linux/crypto/testmgr.h
     * ecdsa_nist_p192_tv_template[], ecdsa_nist_p256_tv_template[], and
     * ecdsa_nist_p384_tv_template[] use SHA-1 (even if CONFIG_CRYPTO_SHA1 is
     * disabled), and kernel module signatures frequently use SHA-1 until quite
     * recently (dependent on CONFIG_CRYPTO_SHA1).  If either is enabled, force
     * downgrade to 186-4.
     */
    #if defined(WC_FIPS_186_5_PLUS) && \
        (defined(CONFIG_CRYPTO_SHA1) || (defined(CONFIG_CRYPTO_MANAGER) && !defined(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS))) && \
        (defined(LINUXKM_LKCAPI_REGISTER_ALL) || defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) || defined(CONFIG_CRYPTO_ECDSA))
        #undef WC_FIPS_186_5_PLUS
        #ifdef WC_FIPS_186_5
            #undef WC_FIPS_186_5
        #else
            #error Unknown and incompatible FIPS 186 is enabled.
        #endif
        #define WC_FIPS_186_4
    #endif

    #ifdef HAVE_CONFIG_H
        #ifndef PACKAGE_NAME
            #error wc_port.h included before config.h
        #endif
        /* config.h is autogenerated without gating, and is subject to repeat
         * inclusions, so gate it out here to keep autodetection masking
         * intact:
         */
        #undef HAVE_CONFIG_H
    #endif

    /* suppress inclusion of stdint-gcc.h to avoid conflicts with Linux native
     * include/linux/types.h:
     */
    #define _GCC_STDINT_H
    #define WC_PTR_TYPE uintptr_t

    /* needed to suppress inclusion of stdio.h in wolfssl/wolfcrypt/types.h */
    #define XSNPRINTF snprintf

    /* the rigmarole around kstrtoll() here is to accommodate its
     * warn-unused-result attribute.
     *
     * also needed to suppress inclusion of stdlib.h in
     * wolfssl/wolfcrypt/types.h.
     */
    #define XATOI(s) ({                                 \
          long long _xatoi_res = 0;                     \
          int _xatoi_ret = kstrtoll(s, 10, &_xatoi_res); \
          if (_xatoi_ret != 0) {                        \
            _xatoi_res = 0;                             \
          }                                             \
          (int)_xatoi_res;                              \
        })

    /* Kbuild+gcc on x86 doesn't consistently honor the default ALIGN16 on stack
     * objects, but gives adequate alignment with "32".
     */
    #if defined(CONFIG_X86) && !defined(ALIGN16)
        #define ALIGN16 __attribute__ ( (aligned (32)))
    #endif

    /* kvmalloc()/kvfree() and friends added in linux commit a7c3e901, merged for 4.12.
     * kvrealloc() added in de2860f463, merged for 5.15, backported to 5.10.137.
     * moved to ultimate home (slab.h) in 8587ca6f34, merged for 5.16.
     *
     * however, until 6.12 (commit 590b9d576c), it took an extra argument,
     * oldsize, that makes it incompatible with traditional libc usage patterns,
     * so we don't try to use it.
     */
    #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0) && \
        !defined(DONT_HAVE_KVMALLOC) && !defined(HAVE_KVMALLOC)
        #define HAVE_KVMALLOC
    #endif
    #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0) && \
        !defined(DONT_HAVE_KVREALLOC) && !defined(HAVE_KVREALLOC)
        #define HAVE_KVREALLOC
    #endif

    #ifdef WOLFCRYPT_ONLY
        #if defined(HAVE_KVMALLOC) && \
            !defined(DONT_USE_KVMALLOC) && !defined(USE_KVMALLOC)
            #define USE_KVMALLOC
        #endif
        #if defined(HAVE_KVREALLOC) && \
            !defined(DONT_USE_KVREALLOC) && !defined(USE_KVREALLOC)
            #define USE_KVREALLOC
        #endif
    #else
        /* functioning realloc() is needed for the TLS stack. */
        #if defined(HAVE_KVMALLOC) && defined(HAVE_KVREALLOC) && \
            !defined(DONT_USE_KVMALLOC) && !defined(DONT_USE_KVREALLOC)
            #ifndef USE_KVMALLOC
                #define USE_KVMALLOC
            #endif
            #ifndef USE_KVREALLOC
                #define USE_KVREALLOC
            #endif
        #endif
    #endif

    #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)
        /* added by 6bab69c650 */
        #define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr)
        #define __static_assert(expr, msg, ...) _Static_assert(expr, msg)
    #endif

    /* kernel printf doesn't implement fp. */
    #ifndef WOLFSSL_NO_FLOAT_FMT
        #define WOLFSSL_NO_FLOAT_FMT
    #endif

    #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
        (defined(RHEL_MAJOR) && \
         ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
        #define WOLFSSL_DEBUG_PRINTF_FN _printk
    #else
        #define WOLFSSL_DEBUG_PRINTF_FN printk
    #endif

#ifndef WOLFSSL_LINUXKM_USE_MUTEXES
    struct wolfSSL_Mutex;
    extern int wc_lkm_LockMutex(struct wolfSSL_Mutex* m);
#endif

    #ifndef WC_LINUXKM_INTR_SIGNALS
        #define WC_LINUXKM_INTR_SIGNALS { SIGKILL, SIGABRT, SIGHUP, SIGINT }
    #endif
    extern int wc_linuxkm_sig_ignore_begin(void);
    extern int wc_linuxkm_sig_ignore_end(void);
    extern int wc_linuxkm_check_for_intr_signals(void);
    #ifndef WC_LINUXKM_MAX_NS_WITHOUT_YIELD
        #define WC_LINUXKM_MAX_NS_WITHOUT_YIELD 1000000000
    #endif
    extern void wc_linuxkm_relax_long_loop(void);

    #ifndef WC_SIG_IGNORE_BEGIN
        #define WC_SIG_IGNORE_BEGIN() wc_linuxkm_sig_ignore_begin()
    #endif
    #ifndef WC_SIG_IGNORE_END
        #define WC_SIG_IGNORE_END() wc_linuxkm_sig_ignore_end()
    #endif
    #ifndef WC_CHECK_FOR_INTR_SIGNALS
        #define WC_CHECK_FOR_INTR_SIGNALS() wc_linuxkm_check_for_intr_signals()
    #endif
    #ifndef WC_RELAX_LONG_LOOP
        #define WC_RELAX_LONG_LOOP() wc_linuxkm_relax_long_loop()
    #endif

    enum wc_svr_flags {
        WC_SVR_FLAG_NONE = 0,
        WC_SVR_FLAG_INHIBIT = 1,
    };

    #if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP) || \
        defined(WOLFSSL_SP_X86_64_ASM)
        #ifndef CONFIG_X86
            #error X86 SIMD extensions requested, but CONFIG_X86 is not set.
        #endif
        #define WOLFSSL_LINUXKM_SIMD
        #define WOLFSSL_LINUXKM_SIMD_X86
        #ifndef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
            #define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
        #endif
    #elif defined(WOLFSSL_ARMASM) || defined(WOLFSSL_SP_ARM32_ASM) || \
          defined(WOLFSSL_SP_ARM64_ASM) || defined(WOLFSSL_SP_ARM_THUMB_ASM) ||\
          defined(WOLFSSL_SP_ARM_CORTEX_M_ASM)
        #if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
            #error ARM SIMD extensions requested, but CONFIG_ARM* is not set.
        #endif
        #define WOLFSSL_LINUXKM_SIMD
        #define WOLFSSL_LINUXKM_SIMD_ARM
        #ifndef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
            #define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
        #endif
    #endif

    #if defined(HAVE_HASHDRBG) && defined(HAVE_FIPS) && \
            defined(HAVE_ENTROPY_MEMUSE) && \
            !defined(WC_LINUXKM_WOLFENTROPY_IN_GLUE_LAYER)
        #define WC_LINUXKM_WOLFENTROPY_IN_GLUE_LAYER
    #elif defined(HAVE_HASHDRBG) && defined(HAVE_FIPS) && \
            (defined(HAVE_INTEL_RDSEED) || defined(HAVE_AMD_RDSEED)) && \
            !defined(HAVE_ENTROPY_MEMUSE) && \
            !defined(WC_LINUXKM_RDSEED_IN_GLUE_LAYER)
        #define WC_LINUXKM_RDSEED_IN_GLUE_LAYER
    #endif
    #if defined(WC_LINUXKM_WOLFENTROPY_IN_GLUE_LAYER)
        struct OS_Seed;
        extern int wc_linuxkm_GenerateSeed_wolfEntropy(struct OS_Seed* os, unsigned char* output, unsigned int sz);
        #define WC_GENERATE_SEED_DEFAULT wc_linuxkm_GenerateSeed_wolfEntropy
    #elif defined(WC_LINUXKM_RDSEED_IN_GLUE_LAYER)
        struct OS_Seed;
        extern int wc_linuxkm_GenerateSeed_IntelRD(struct OS_Seed* os, unsigned char* output, unsigned int sz);
        #define WC_GENERATE_SEED_DEFAULT wc_linuxkm_GenerateSeed_IntelRD
    #endif

    /* setup for LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT needs to be here
     * to assure that calls to get_random_bytes() in random.c are gated out
     * (they would recurse, potentially infinitely).
     */
    #if defined(LINUXKM_LKCAPI_REGISTER_ALL) && \
        !defined(LINUXKM_LKCAPI_DONT_REGISTER_HASH_DRBG) && \
        !defined(LINUXKM_LKCAPI_DONT_REGISTER_HASH_DRBG_DEFAULT) && \
        !defined(NO_LINUXKM_DRBG_GET_RANDOM_BYTES) && \
        !defined(LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT) && \
        defined(HAVE_HASHDRBG)
        #define LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
    #endif

    #ifdef BUILDING_WOLFSSL

    #if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)) || \
         (defined(RHEL_MAJOR) &&                                    \
          ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))) && \
        defined(CONFIG_X86)
        /* linux/slab.h recursively brings in linux/page-flags.h, bringing in
         * non-inline implementations of functions folio_flags() and
         * const_folio_flags().  but we can retrofit the attribute.
         */
        struct folio;
        static __always_inline unsigned long *folio_flags(
            struct folio *folio, unsigned n);
        #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 9, 0)
            static __always_inline const unsigned long *const_folio_flags(
                const struct folio *folio, unsigned n);
        #endif
    #endif

    #if defined(CONFIG_MIPS) && defined(WC_SYM_RELOC_TABLES)
        /* __ZBOOT__ disables some unhelpful macros around the mem*() funcs in
         * legacy arch/mips/include/asm/string.h
         */
        #define __ZBOOT__
        #define memcmp __builtin_memcmp
        #define __ARCH_MEMCMP_NO_REDIRECT
        #define __ARCH_MEMCPY_NO_REDIRECT
        #define __builtin_memcpy memcpy
        extern void *memcpy(void *dest, const void *src, unsigned int n);
        #define __ARCH_MEMCPY_NO_REDIRECT
        #define __builtin_memset memset
        extern void *memset(void *dest, int c, unsigned int n);
    #endif

    _Pragma("GCC diagnostic push");

    /* we include all the needed kernel headers with these masked out. else
     * there are profuse warnings, especially on older kernels.
     */
    _Pragma("GCC diagnostic ignored \"-Wunused-parameter\"");
    _Pragma("GCC diagnostic ignored \"-Wpointer-arith\"");
    _Pragma("GCC diagnostic ignored \"-Wshadow\"");
    _Pragma("GCC diagnostic ignored \"-Wnested-externs\"");
    _Pragma("GCC diagnostic ignored \"-Wredundant-decls\"");
    _Pragma("GCC diagnostic ignored \"-Wsign-compare\"");
    _Pragma("GCC diagnostic ignored \"-Wpointer-sign\"");
    _Pragma("GCC diagnostic ignored \"-Wbad-function-cast\"");
#ifndef __clang__
    _Pragma("GCC diagnostic ignored \"-Wdiscarded-qualifiers\"");
#endif
    _Pragma("GCC diagnostic ignored \"-Wtype-limits\"");
    _Pragma("GCC diagnostic ignored \"-Wswitch-enum\"");
    _Pragma("GCC diagnostic ignored \"-Wcast-function-type\""); /* needed for kernel 4.14.336 */
    _Pragma("GCC diagnostic ignored \"-Wformat-nonliteral\""); /* needed for kernel 4.9.282 */
    _Pragma("GCC diagnostic ignored \"-Wattributes\"");

    #ifdef CONFIG_KASAN
        #ifndef WC_SANITIZE_DISABLE
            #define WC_SANITIZE_DISABLE() kasan_disable_current()
        #endif
        #ifndef WC_SANITIZE_ENABLE
            #define WC_SANITIZE_ENABLE() kasan_enable_current()
        #endif
    #endif

    #if defined(CONFIG_FORTIFY_SOURCE) && \
        !defined(WC_FORCE_LINUXKM_FORTIFY_SOURCE) && \
        (defined(WC_SYM_RELOC_TABLES) || \
         (LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0)))
        /* fortify-source causes all sorts of awkward problems for the PIE
         * build, up to and including stubborn external references and multiple
         * definitions of string functions.
         *
         * fortify-source before kernel 5.18 has similar issues regardless of
         * PIE, around our macro-shimming of the string functions.
         */
        #undef CONFIG_FORTIFY_SOURCE
    #endif

    #if defined(WC_FORCE_LINUXKM_FORTIFY_SOURCE) && \
        !defined(CONFIG_FORTIFY_SOURCE)
        #error WC_FORCE_LINUXKM_FORTIFY_SOURCE without CONFIG_FORTIFY_SOURCE.
    #endif

    #if defined(WC_CONTAINERIZE_THIS) && defined(CONFIG_ARM64)
        /* alt_cb_patch_nops and queued_spin_lock_slowpath are defined early
         * to allow shimming in system headers.
         */
        /* alt_cb_patch_nops added by d926079f17, release 6.1 */
        #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)
            #define alt_cb_patch_nops my__alt_cb_patch_nops
        #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) */
        #define queued_spin_lock_slowpath my__queued_spin_lock_slowpath
    #endif

    /*
     * Disable ARM64 LSE atomics for out-of-tree modules.
     *
     * When CONFIG_ARM64_LSE_ATOMICS is enabled, the kernel uses static keys
     * (jump labels) in system_uses_lse_atomics() to choose between LSE and
     * LL/SC atomic implementations at runtime. These static keys generate
     * asm goto statements that reference .jump_table section symbols which
     * cannot be resolved in out-of-tree modules, causing:
     *   "error: impossible constraint in 'asm'"
     *
     * By undefining CONFIG_ARM64_LSE_ATOMICS here (before any kernel headers
     * that use atomics are included), we force use of the LL/SC fallback path
     * which works correctly in out-of-tree modules.
     */
    #undef CONFIG_ARM64_LSE_ATOMICS

    #include <linux/kernel.h>
    #include <linux/ctype.h>

    #if defined(CONFIG_FORTIFY_SOURCE) || defined(DEBUG_LINUXKM_FORTIFY_OVERLAY)
        #ifdef WC_CONTAINERIZE_THIS
            /* the inline definitions in fortify-string.h use non-inline
             * fortify_panic().
             */
            extern void __my_fortify_panic(const char *name) __noreturn __cold;
            #if LINUX_VERSION_CODE >= KERNEL_VERSION(6,9,0)
                /* see linux 3d965b33e40d9 */
                #define fortify_panic(func, write, avail, size, retfail) \
                        __my_fortify_panic(#func)
            #else
                #define fortify_panic __my_fortify_panic
            #endif
        #endif

        /* the _FORTIFY_SOURCE macros and implementations for several string
         * functions are incompatible with libwolfssl, so just reimplement with
         * inlines and remap with macros.
         */

        #define __ARCH_STRLEN_NO_REDIRECT
        #define __ARCH_MEMCPY_NO_REDIRECT
        #define __ARCH_MEMSET_NO_REDIRECT
        #define __ARCH_MEMMOVE_NO_REDIRECT

        /* the inline definitions in fortify-string.h use non-inline
         * strlen().
         */
        static inline size_t strlen(const char *s) {
            const char *s_start = s;
            while (*s)
                ++s;
            return (size_t)((uintptr_t)s - (uintptr_t)s_start);
        }

        #include <linux/string.h>

        #undef strlen
        #define strlen(s) \
            ((__builtin_constant_p(s) && __builtin_constant_p(*(s))) ? \
             (sizeof(s) - 1) : strlen(s))

        static inline void *my_memcpy(void *dest, const void *src, size_t n) {
            if (! (((uintptr_t)dest | (uintptr_t)src | (uintptr_t)n)
                   & (uintptr_t)(sizeof(uintptr_t) - 1)))
            {
                uintptr_t *src_longs = (uintptr_t *)src,
                    *dest_longs = (uintptr_t *)dest,
                    *endp = (uintptr_t *)((u8 *)src + n);
                while (src_longs < endp)
                    *dest_longs++ = *src_longs++;
            } else {
                u8 *src_bytes = (u8 *)src,
                    *dest_bytes = (u8 *)dest,
                    *endp = src_bytes + n;
                while (src_bytes < endp)
                    *dest_bytes++ = *src_bytes++;
            }
            return dest;
        }
        #undef memcpy
        #define memcpy my_memcpy

        static inline void *my_memset(void *dest, int c, size_t n) {
            if (! (((uintptr_t)dest | (uintptr_t)n)
                   & (uintptr_t)(sizeof(uintptr_t) - 1)))
            {
                uintptr_t c_long = __builtin_choose_expr(
                    sizeof(uintptr_t) == 8,
                    (uintptr_t)(u8)c * 0x0101010101010101UL,
                    (uintptr_t)(u8)c * 0x01010101U
                    );
                uintptr_t *dest_longs = (uintptr_t *)dest,
                    *endp = (uintptr_t *)((u8 *)dest_longs + n);
                while (dest_longs < endp)
                    *dest_longs++ = c_long;
            } else {
                u8 *dest_bytes = (u8 *)dest, *endp = dest_bytes + n;
                while (dest_bytes < endp)
                    *dest_bytes++ = (u8)c;
            }
            return dest;
        }
        #undef memset
        #define memset my_memset

        static inline void *my_memmove(void *dest, const void *src, size_t n) {
            if (! (((uintptr_t)dest | (uintptr_t)src | (uintptr_t)n)
                   & (uintptr_t)(sizeof(uintptr_t) - 1)))
            {
                uintptr_t *src_longs = (uintptr_t *)src,
                    *dest_longs = (uintptr_t *)dest;
                n >>= __builtin_choose_expr(
                    sizeof(uintptr_t) == 8,
                    3U,
                    2U);
                if (src_longs < dest_longs) {
                    uintptr_t *startp = src_longs;
                    src_longs += n - 1;
                    dest_longs += n - 1;
                    while (src_longs >= startp)
                        *dest_longs-- = *src_longs--;
                } else if (src_longs > dest_longs) {
                    uintptr_t *endp = src_longs + n;
                    while (src_longs < endp)
                        *dest_longs++ = *src_longs++;
                }
            } else {
                u8 *src_bytes = (u8 *)src, *dest_bytes = (u8 *)dest;
                if (src_bytes < dest_bytes) {
                    u8 *startp = src_bytes;
                    src_bytes += n - 1;
                    dest_bytes += n - 1;
                    while (src_bytes >= startp)
                        *dest_bytes-- = *src_bytes--;
                } else if (src_bytes > dest_bytes) {
                    u8 *endp = src_bytes + n;
                    while (src_bytes < endp)
                        *dest_bytes++ = *src_bytes++;
                }
            }
            return dest;
        }
        #undef memmove
        #define memmove my_memmove

    #else /* !CONFIG_FORTIFY_SOURCE */

        #include <linux/string.h>

    #endif /* !CONFIG_FORTIFY_SOURCE */

    #ifndef WC_CONTAINERIZE_THIS
        #include <linux/init.h>
        #include <linux/module.h>
        #include <linux/moduleparam.h>
        #include <linux/delay.h>
    #endif

    #if defined(HAVE_KVMALLOC) && \
        (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)) && \
        !(defined(RHEL_MAJOR) && ((RHEL_MAJOR > 9) || \
                                  ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
        /* before 5.16, the kvmalloc_node() and kvfree() prototypes were in
         * mm.h.  however, mm.h brings in static, but not inline, pmd_to_page(),
         * with direct references to global vmem variables.
         */
        #ifdef WC_CONTAINERIZE_THIS
            #include <linux/mm_types.h>
            #if USE_SPLIT_PMD_PTLOCKS
            static __always_inline struct page *pmd_to_page(pmd_t *pmd);
            #endif
        #endif
        #include <linux/mm.h>
    #endif

#ifndef WC_CONTAINERIZE_THIS
    #include <linux/kthread.h>
    #include <linux/net.h>
    #ifndef WOLFCRYPT_ONLY
        #include <linux/inet.h>
        static inline int wc_linuxkm_inet_pton(int af, const char *src, void *dst)
        {
            int ret;

            if (!src || !dst)
                return -EFAULT;

            switch (af) {
            case AF_INET:
                ret = in4_pton(src, -1, (u8 *)dst, '\0', NULL);
                return ret == 1 ? 1 : 0;

            case AF_INET6:
                ret = in6_pton(src, -1, (u8 *)dst, '\0', NULL);
                return ret == 1 ? 1 : 0;

            default:
                return -EAFNOSUPPORT;
            }
        }
        #define XINET_PTON(af, src, dst) wc_linuxkm_inet_pton(af, src, dst)
    #endif /* !WOLFCRYPT_ONLY */
#endif /* !WC_CONTAINERIZE_THIS */

    #if defined(WC_SYM_RELOC_TABLES) && defined(DEBUG_LINUXKM_PIE_SUPPORT) && \
        !defined(WC_LINUXKM_SUPPORT_DUMP_TO_FILE)
        #define WC_LINUXKM_SUPPORT_DUMP_TO_FILE
    #endif

    #ifdef WC_LINUXKM_SUPPORT_DUMP_TO_FILE
        #include <linux/fs.h>
        #include <linux/uaccess.h>
    #endif
    #include <linux/slab.h>
    #include <linux/sched.h>
    #if __has_include(<linux/sched/task_stack.h>)
        /* for task_stack_page() */
        #include <linux/sched/task_stack.h>
    #endif
    #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
        /* for signal_pending() */
        #include <linux/sched/signal.h>
        /* for local_clock() */
        #include <linux/sched/clock.h>
    #endif
    #include <linux/random.h>

    #if !defined(WC_CONTAINERIZE_THIS) && defined(CONFIG_HAVE_KPROBES)
        #include <linux/kprobes.h>
    #endif

    #ifdef LINUXKM_LKCAPI_REGISTER
        /* the LKCAPI assumes that expanded encrypt and decrypt keys will stay
         * loaded simultaneously, and the Linux in-tree implementations have two
         * AES key structs in each context, one for each direction.  in
         * linuxkm/lkcapi_aes_glue.c, we do the same
         * thing with "struct km_AesCtx".  however, wolfCrypt struct AesXts
         * already has two AES expanded keys, the main and tweak, and the tweak
         * is always used in the encrypt direction regardless of the main
         * direction.  to avoid allocating and computing a duplicate second
         * tweak encrypt key, we set
         * WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS, which adds a second
         * Aes slot to wolfCrypt's struct AesXts, and activates support for
         * AES_ENCRYPTION_AND_DECRYPTION on AES-XTS.
         */
        #ifndef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
            #define WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
        #endif

        #ifndef WC_CONTAINERIZE_THIS
            #include <linux/crypto.h>
            #include <linux/scatterlist.h>
            #include <crypto/scatterwalk.h>
            #include <crypto/internal/aead.h>
            #include <crypto/internal/hash.h>
            #include <crypto/internal/rng.h>
            #include <crypto/internal/skcipher.h>
            #include <crypto/internal/akcipher.h>
            #include <crypto/internal/kpp.h>
            #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 13, 0)
                #include <crypto/internal/sig.h>
            #endif /* linux ver >= 6.13 */

            #if defined(_LINUX_REFCOUNT_H) || defined(_LINUX_REFCOUNT_TYPES_H)
                static inline int wc_lkm_refcount_to_int(refcount_t *refcount) {
                    _Pragma("GCC diagnostic push");
                    _Pragma("GCC diagnostic ignored \"-Wnested-externs\"");
                    return atomic_read(&(refcount->refs));
                    _Pragma("GCC diagnostic pop");
                }
            #else
                static inline int wc_lkm_refcount_to_int(atomic_t *refcount) {
                    _Pragma("GCC diagnostic push");
                    _Pragma("GCC diagnostic ignored \"-Wnested-externs\"");
                    return atomic_read(&refcount);
                    _Pragma("GCC diagnostic pop");
                }
            #endif
            #define WC_LKM_REFCOUNT_TO_INT(refcount) wc_lkm_refcount_to_int(&(refcount))
        #endif /* !WC_CONTAINERIZE_THIS */

    #endif /* LINUXKM_LKCAPI_REGISTER */

    /* benchmarks.c uses floating point math, so needs a working
     * SAVE_VECTOR_REGISTERS().
     */
    #if defined(WOLFSSL_LINUXKM_BENCHMARKS) && \
        !defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
        #define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
    #endif

    #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \
        defined(CONFIG_X86)

        extern __must_check int allocate_wolfcrypt_linuxkm_fpu_states(void);
        extern void free_wolfcrypt_linuxkm_fpu_states(void);
        WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void);
        WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags);
        WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags);

        #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
            #include <asm/i387.h>
            #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
                /* added by a62b01cd6c */
                #include <asm-generic/simd.h>
            #endif
        #else
            #include <asm/simd.h>
            #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
                /* added by 266d051601 */
                #include <crypto/internal/simd.h>
            #endif
        #endif
        #ifndef CAN_SAVE_VECTOR_REGISTERS
            #ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING
                #define CAN_SAVE_VECTOR_REGISTERS() (wc_can_save_vector_registers_x86() && (SAVE_VECTOR_REGISTERS2_fuzzer() == 0))
            #else
                #define CAN_SAVE_VECTOR_REGISTERS() wc_can_save_vector_registers_x86()
            #endif
        #endif
        #ifndef SAVE_VECTOR_REGISTERS
            #define SAVE_VECTOR_REGISTERS(fail_clause) {     \
                int _svr_ret = wc_save_vector_registers_x86(WC_SVR_FLAG_NONE); \
                if (_svr_ret != 0) {                         \
                    fail_clause                              \
                }                                            \
            }
        #endif
        #ifndef SAVE_VECTOR_REGISTERS2
            #ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING
                #define SAVE_VECTOR_REGISTERS2() ({                    \
                    int _fuzzer_ret = SAVE_VECTOR_REGISTERS2_fuzzer(); \
                    (_fuzzer_ret == 0) ?                               \
                     wc_save_vector_registers_x86(WC_SVR_FLAG_NONE) :  \
                     _fuzzer_ret;                                      \
                })
            #else
                #define SAVE_VECTOR_REGISTERS2() wc_save_vector_registers_x86(WC_SVR_FLAG_NONE)
            #endif
        #endif
        #ifndef RESTORE_VECTOR_REGISTERS
            #define RESTORE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_NONE)
        #endif

        #ifndef DISABLE_VECTOR_REGISTERS
            #define DISABLE_VECTOR_REGISTERS() wc_save_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
        #endif
        #ifndef REENABLE_VECTOR_REGISTERS
            #define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
        #endif

    #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && (defined(CONFIG_ARM) || defined(CONFIG_ARM64))

        #error kernel module ARM SIMD is not yet tested or usable.

        #include <asm/fpsimd.h>

        static WARN_UNUSED_RESULT inline int save_vector_registers_arm(void)
        {
            preempt_disable();
            if (! may_use_simd()) {
                preempt_enable();
                return BAD_STATE_E;
            } else {
                fpsimd_preserve_current_state();
                return 0;
            }
        }
        static inline void restore_vector_registers_arm(void)
        {
            fpsimd_restore_current_state();
            preempt_enable();
        }

        #ifndef SAVE_VECTOR_REGISTERS
            #define SAVE_VECTOR_REGISTERS(fail_clause) { int _svr_ret = save_vector_registers_arm(); if (_svr_ret != 0) { fail_clause } }
        #endif
        #ifndef SAVE_VECTOR_REGISTERS2
            #define SAVE_VECTOR_REGISTERS2() save_vector_registers_arm()
        #endif
        #ifndef CAN_SAVE_VECTOR_REGISTERS
            #define CAN_SAVE_VECTOR_REGISTERS() can_save_vector_registers_arm()
        #endif
        #ifndef RESTORE_VECTOR_REGISTERS
            #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_arm()
        #endif

    #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
        #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.
    #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */

    _Pragma("GCC diagnostic pop");

    /* avoid -Wpointer-arith, encountered when -DCONFIG_FORTIFY_SOURCE */
    #undef __is_constexpr
    #define __is_constexpr(x) __builtin_constant_p(x)

    /* the kernel uses -std=c89, but not -pedantic, and makes full use of anon
     * structs/unions, so we should too.
     */
    #define HAVE_ANONYMOUS_INLINE_AGGREGATES 1

    #define NO_THREAD_LS
    #define NO_ATTRIBUTE_CONSTRUCTOR

    #ifdef HAVE_FIPS
        extern int wolfCrypt_FIPS_first(void);
        extern int wolfCrypt_FIPS_last(void);
        extern const unsigned int wolfCrypt_FIPS_ro_start[];
        extern const unsigned int wolfCrypt_FIPS_ro_end[];
        #if FIPS_VERSION3_GE(6,0,0)
#ifndef NO_AES
            extern int wolfCrypt_FIPS_AES_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_aes_ro_sanity[2];
#if defined(WOLFSSL_CMAC) && defined(WOLFSSL_AES_DIRECT)
            extern int wolfCrypt_FIPS_CMAC_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_cmac_ro_sanity[2];
#endif
#endif
#ifndef NO_DH
            extern int wolfCrypt_FIPS_DH_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_dh_ro_sanity[2];
#endif
#ifdef HAVE_ECC
            extern int wolfCrypt_FIPS_ECC_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_ecc_ro_sanity[2];
#endif
#ifdef HAVE_ED25519
            extern int wolfCrypt_FIPS_ED25519_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_ed25519_ro_sanity[2];
#endif
#ifdef HAVE_ED448
            extern int wolfCrypt_FIPS_ED448_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_ed448_ro_sanity[2];
#endif
            extern int wolfCrypt_FIPS_HMAC_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_hmac_ro_sanity[2];
#ifndef NO_KDF
            extern int wolfCrypt_FIPS_KDF_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_kdf_ro_sanity[2];
#endif
#ifdef HAVE_PBKDF2
            extern int wolfCrypt_FIPS_PBKDF_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_pbkdf_ro_sanity[2];
#endif
#ifdef HAVE_HASHDRBG
            extern int wolfCrypt_FIPS_DRBG_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_drbg_ro_sanity[2];
#endif
#ifndef NO_RSA
            extern int wolfCrypt_FIPS_RSA_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_rsa_ro_sanity[2];
#endif
#ifndef NO_SHA
            extern int wolfCrypt_FIPS_SHA_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_sha_ro_sanity[2];
#endif
#ifndef NO_SHA256
            extern int wolfCrypt_FIPS_SHA256_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_sha256_ro_sanity[2];
#endif
#ifdef WOLFSSL_SHA512
            extern int wolfCrypt_FIPS_SHA512_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_sha512_ro_sanity[2];
#endif
#ifdef WOLFSSL_SHA3
            extern int wolfCrypt_FIPS_SHA3_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_sha3_ro_sanity[2];
#endif
            extern int wolfCrypt_FIPS_FT_sanity(void);
            extern const unsigned int wolfCrypt_FIPS_ft_ro_sanity[2];
            extern const unsigned int wolfCrypt_FIPS_f_ro_sanity[2];
            extern int wc_RunAllCast_fips(void);
        #endif
    #endif

    #if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS)
        /* work around backward dependency of asn.c on ssl.c. */
        struct Signer;
        struct Signer *GetCA(void *signers, unsigned char *hash);
        #ifndef NO_SKID
            struct Signer *GetCAByName(void* signers, unsigned char *hash);
            #ifdef HAVE_OCSP
                struct Signer* GetCAByKeyHash(void* vp, const unsigned char* keyHash);
            #endif /* HAVE_OCSP */
            #ifdef WOLFSSL_AKID_NAME
                #ifdef WOLFSSL_API_PREFIX_MAP
                    #define GetCAByAKID wolfSSL_GetCAByAKID
                #endif
                struct Signer* GetCAByAKID(void* vp, const unsigned char* issuer,
                                           unsigned int issuerSz,
                                           const unsigned char* serial,
                                           unsigned int serialSz);
            #endif
        #endif /* NO_SKID */

        #if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)
            struct WOLFSSL_X509_NAME;
            extern int wolfSSL_X509_NAME_add_entry_by_NID(struct WOLFSSL_X509_NAME *name, int nid,
                                               int type, const unsigned char *bytes,
                                               int len, int loc, int set);
            extern void wolfSSL_X509_NAME_free(struct WOLFSSL_X509_NAME* name);
            extern struct WOLFSSL_X509_NAME* wolfSSL_X509_NAME_new_ex(void *heap);
        #endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */

    #endif /* !WOLFCRYPT_ONLY && !NO_CERTS */

    #if defined(WC_CONTAINERIZE_THIS) && !defined(WC_SYM_RELOC_TABLES)
        #error "compiling -DWC_CONTAINERIZE_THIS requires relocation tables."
    #endif

    #ifdef WC_SYM_RELOC_TABLES

    extern __attribute__((error("uncallable fencepost"))) int __wc_text_start(void);
    extern __attribute__((error("uncallable fencepost"))) int __wc_text_end(void);
    #ifdef HAVE_FIPS
        static_assert(__builtin_types_compatible_p(typeof(__wc_text_start), typeof(wolfCrypt_FIPS_first)));
        static_assert(__builtin_types_compatible_p(typeof(__wc_text_end), typeof(wolfCrypt_FIPS_last)));
    #endif
    extern const unsigned int __wc_rodata_start[], __wc_rodata_end[];
    #ifdef HAVE_FIPS
        static_assert(__builtin_types_compatible_p(typeof(__wc_rodata_start), typeof(wolfCrypt_FIPS_ro_start)));
        static_assert(__builtin_types_compatible_p(typeof(__wc_rodata_end), typeof(wolfCrypt_FIPS_ro_end)));
    #endif

    extern const u8
        __wc_rwdata_start[],
        __wc_rwdata_end[],
        __wc_bss_start[],
        __wc_bss_end[];

    extern ssize_t wc_linuxkm_normalize_relocations(
        const u8 *text_in,
        size_t text_in_len,
        u8 *text_out,
        ssize_t *cur_index_p);

    #ifndef WOLFSSL_TEXT_SEGMENT_CANONICALIZER
        #define WOLFSSL_TEXT_SEGMENT_CANONICALIZER(text_in, text_in_len, text_out, cur_index_p) \
            wc_linuxkm_normalize_relocations(text_in, text_in_len, text_out, cur_index_p)
        #define WOLFSSL_TEXT_SEGMENT_CANONICALIZER_BUFSIZ 8192
    #endif

#ifdef CONFIG_MIPS
    #undef __ARCH_MEMCMP_NO_REDIRECT
    #undef memcmp
    extern int memcmp(const void *s1, const void *s2, size_t n);
#endif

    struct wolfssl_linuxkm_pie_redirect_table {
    #ifdef HAVE_FIPS
        typeof(wc_linuxkm_normalize_relocations) *wc_linuxkm_normalize_relocations;
    #endif

    #ifndef __ARCH_MEMCMP_NO_REDIRECT
        typeof(memcmp) *memcmp;
    #endif
    #ifndef __ARCH_MEMCPY_NO_REDIRECT
        typeof(memcpy) *memcpy;
    #endif
    #ifndef __ARCH_MEMSET_NO_REDIRECT
        typeof(memset) *memset;
    #endif
    #ifndef __ARCH_MEMMOVE_NO_REDIRECT
        typeof(memmove) *memmove;
    #endif
    #ifndef __ARCH_STRCMP_NO_REDIRECT
        typeof(strcmp) *strcmp;
    #endif
    #ifndef __ARCH_STRNCMP_NO_REDIRECT
        typeof(strncmp) *strncmp;
    #endif
    #ifndef __ARCH_STRCASECMP_NO_REDIRECT
        typeof(strcasecmp) *strcasecmp;
    #endif
    #ifndef __ARCH_STRNCASECMP_NO_REDIRECT
        typeof(strncasecmp) *strncasecmp;
    #endif
    #ifndef __ARCH_STRLEN_NO_REDIRECT
        typeof(strlen) *strlen;
    #endif
    #ifndef __ARCH_STRSTR_NO_REDIRECT
        typeof(strstr) *strstr;
    #endif
    #ifndef __ARCH_STRNCPY_NO_REDIRECT
        typeof(strncpy) *strncpy;
    #endif
    #ifndef __ARCH_STRNCAT_NO_REDIRECT
        typeof(strncat) *strncat;
    #endif
        typeof(kstrtoll) *kstrtoll;

        #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
            (defined(RHEL_MAJOR) && \
             ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
            typeof(_printk) *_printk;
        #else
            typeof(printk) *printk;
        #endif

#ifdef CONFIG_FORTIFY_SOURCE
        typeof(__warn_printk) *__warn_printk;
#endif

        typeof(snprintf) *snprintf;

        const unsigned char *_ctype;

#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0)
        typeof(kmalloc_noprof) *kmalloc_noprof;
        typeof(krealloc_node_align_noprof) *krealloc_node_align_noprof;
        typeof(kzalloc_noprof) *kzalloc_noprof;
        typeof(__kvmalloc_node_noprof) *__kvmalloc_node_noprof;
        typeof(__kmalloc_cache_noprof) *__kmalloc_cache_noprof;
        #ifdef HAVE_KVREALLOC
            typeof(kvrealloc_node_align_noprof) *kvrealloc_node_align_noprof;
        #endif
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 11, 0)
        typeof(kmalloc_noprof) *kmalloc_noprof;
        typeof(krealloc_noprof) *krealloc_noprof;
        typeof(kzalloc_noprof) *kzalloc_noprof;
        typeof(__kvmalloc_node_noprof) *__kvmalloc_node_noprof;
        typeof(__kmalloc_cache_noprof) *__kmalloc_cache_noprof;
        #ifdef HAVE_KVREALLOC
            typeof(kvrealloc_noprof) *kvrealloc_noprof;
        #endif
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 10, 0)
        typeof(kmalloc_noprof) *kmalloc_noprof;
        typeof(krealloc_noprof) *krealloc_noprof;
        typeof(kzalloc_noprof) *kzalloc_noprof;
        typeof(kvmalloc_node_noprof) *kvmalloc_node_noprof;
        typeof(kmalloc_trace_noprof) *kmalloc_trace_noprof;
        #ifdef HAVE_KVREALLOC
            typeof(kvrealloc_noprof) *kvrealloc_noprof;
        #endif
#else /* <6.10.0 */
        typeof(kmalloc) *kmalloc;
        typeof(krealloc) *krealloc;
        #ifdef HAVE_KVMALLOC
            typeof(kvmalloc_node) *kvmalloc_node;
        #endif
        #ifdef HAVE_KVREALLOC
            typeof(kvrealloc) *kvrealloc;
        #endif
        #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) || \
            (defined(RHEL_MAJOR) &&                                    \
             ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
            typeof(kmalloc_trace) *kmalloc_trace;
        #else
            typeof(kmem_cache_alloc_trace) *kmem_cache_alloc_trace;
            typeof(kmalloc_order_trace) *kmalloc_order_trace;
        #endif
#endif /* <6.10.0 */
        #ifdef HAVE_KVMALLOC
        typeof(kvfree) *kvfree;
        #endif
        typeof(kfree) *kfree;
        typeof(ksize) *ksize;

#ifndef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
        typeof(get_random_bytes) *get_random_bytes;
#endif
        #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
            typeof(getnstimeofday) *getnstimeofday;
        #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
            typeof(current_kernel_time64) *current_kernel_time64;
        #else
            typeof(ktime_get_coarse_real_ts64) *ktime_get_coarse_real_ts64;
        #endif

        struct task_struct *(*get_current)(void);

        #ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS

            #ifdef CONFIG_X86
                typeof(allocate_wolfcrypt_linuxkm_fpu_states) *allocate_wolfcrypt_linuxkm_fpu_states;
                typeof(wc_can_save_vector_registers_x86) *wc_can_save_vector_registers_x86;
                typeof(free_wolfcrypt_linuxkm_fpu_states) *free_wolfcrypt_linuxkm_fpu_states;
                typeof(wc_restore_vector_registers_x86) *wc_restore_vector_registers_x86;
                typeof(wc_save_vector_registers_x86) *wc_save_vector_registers_x86;
            #else /* !CONFIG_X86 */
                #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.
            #endif /* arch */

        #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */

        #ifndef CONFIG_PREEMPT_RT
            typeof(__mutex_init) *__mutex_init;
        #else
            typeof(__rt_mutex_init) *__rt_mutex_init;
            typeof(rt_mutex_base_init) *rt_mutex_base_init;
            typeof(rt_spin_lock) *rt_spin_lock;
            typeof(rt_spin_unlock) *rt_spin_unlock;
        #endif

        #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
            typeof(mutex_lock_nested) *mutex_lock_nested;
        #else
            typeof(mutex_lock) *mutex_lock;
        #endif
        typeof(mutex_unlock) *mutex_unlock;
        #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
            typeof(mutex_destroy) *mutex_destroy;
        #endif

        #ifdef HAVE_FIPS
        typeof(wolfCrypt_FIPS_first) *wolfCrypt_FIPS_first;
        typeof(wolfCrypt_FIPS_last) *wolfCrypt_FIPS_last;
        typeof(wolfCrypt_FIPS_ro_start) *wolfCrypt_FIPS_ro_start;
        typeof(wolfCrypt_FIPS_ro_end) *wolfCrypt_FIPS_ro_end;
        #if FIPS_VERSION3_GE(6,0,0)
#ifndef NO_AES
            typeof(wolfCrypt_FIPS_AES_sanity) *wolfCrypt_FIPS_AES_sanity;
            typeof(wolfCrypt_FIPS_aes_ro_sanity) *wolfCrypt_FIPS_aes_ro_sanity;
#if defined(WOLFSSL_CMAC) && defined(WOLFSSL_AES_DIRECT)
            typeof(wolfCrypt_FIPS_CMAC_sanity) *wolfCrypt_FIPS_CMAC_sanity;
            typeof(wolfCrypt_FIPS_cmac_ro_sanity) *wolfCrypt_FIPS_cmac_ro_sanity;
#endif
#endif
#ifndef NO_DH
            typeof(wolfCrypt_FIPS_DH_sanity) *wolfCrypt_FIPS_DH_sanity;
            typeof(wolfCrypt_FIPS_dh_ro_sanity) *wolfCrypt_FIPS_dh_ro_sanity;
#endif
#ifdef HAVE_ECC
            typeof(wolfCrypt_FIPS_ECC_sanity) *wolfCrypt_FIPS_ECC_sanity;
            typeof(wolfCrypt_FIPS_ecc_ro_sanity) *wolfCrypt_FIPS_ecc_ro_sanity;
#endif
#ifdef HAVE_ED25519
            typeof(wolfCrypt_FIPS_ED25519_sanity) *wolfCrypt_FIPS_ED25519_sanity;
            typeof(wolfCrypt_FIPS_ed25519_ro_sanity) *wolfCrypt_FIPS_ed25519_ro_sanity;
#endif
#ifdef HAVE_ED448
            typeof(wolfCrypt_FIPS_ED448_sanity) *wolfCrypt_FIPS_ED448_sanity;
            typeof(wolfCrypt_FIPS_ed448_ro_sanity) *wolfCrypt_FIPS_ed448_ro_sanity;
#endif
            typeof(wolfCrypt_FIPS_HMAC_sanity) *wolfCrypt_FIPS_HMAC_sanity;
            typeof(wolfCrypt_FIPS_hmac_ro_sanity) *wolfCrypt_FIPS_hmac_ro_sanity;
#ifndef NO_KDF
            typeof(wolfCrypt_FIPS_KDF_sanity) *wolfCrypt_FIPS_KDF_sanity;
            typeof(wolfCrypt_FIPS_kdf_ro_sanity) *wolfCrypt_FIPS_kdf_ro_sanity;
#endif
#ifdef HAVE_PBKDF2
            typeof(wolfCrypt_FIPS_PBKDF_sanity) *wolfCrypt_FIPS_PBKDF_sanity;
            typeof(wolfCrypt_FIPS_pbkdf_ro_sanity) *wolfCrypt_FIPS_pbkdf_ro_sanity;
#endif
#ifdef HAVE_HASHDRBG
            typeof(wolfCrypt_FIPS_DRBG_sanity) *wolfCrypt_FIPS_DRBG_sanity;
            typeof(wolfCrypt_FIPS_drbg_ro_sanity) *wolfCrypt_FIPS_drbg_ro_sanity;
#endif
#ifndef NO_RSA
            typeof(wolfCrypt_FIPS_RSA_sanity) *wolfCrypt_FIPS_RSA_sanity;
            typeof(wolfCrypt_FIPS_rsa_ro_sanity) *wolfCrypt_FIPS_rsa_ro_sanity;
#endif
#ifndef NO_SHA
            typeof(wolfCrypt_FIPS_SHA_sanity) *wolfCrypt_FIPS_SHA_sanity;
            typeof(wolfCrypt_FIPS_sha_ro_sanity) *wolfCrypt_FIPS_sha_ro_sanity;
#endif
#ifndef NO_SHA256
            typeof(wolfCrypt_FIPS_SHA256_sanity) *wolfCrypt_FIPS_SHA256_sanity;
            typeof(wolfCrypt_FIPS_sha256_ro_sanity) *wolfCrypt_FIPS_sha256_ro_sanity;
#endif
#ifdef WOLFSSL_SHA512
            typeof(wolfCrypt_FIPS_SHA512_sanity) *wolfCrypt_FIPS_SHA512_sanity;
            typeof(wolfCrypt_FIPS_sha512_ro_sanity) *wolfCrypt_FIPS_sha512_ro_sanity;
#endif
#ifdef WOLFSSL_SHA3
            typeof(wolfCrypt_FIPS_SHA3_sanity) *wolfCrypt_FIPS_SHA3_sanity;
            typeof(wolfCrypt_FIPS_sha3_ro_sanity) *wolfCrypt_FIPS_sha3_ro_sanity;
#endif
            typeof(wolfCrypt_FIPS_FT_sanity) *wolfCrypt_FIPS_FT_sanity;
            typeof(wolfCrypt_FIPS_ft_ro_sanity) *wolfCrypt_FIPS_ft_ro_sanity;
            typeof(wolfCrypt_FIPS_f_ro_sanity) *wolfCrypt_FIPS_f_ro_sanity;
            typeof(wc_RunAllCast_fips) *wc_RunAllCast_fips;
        #endif
        #endif

        #if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS)
        typeof(GetCA) *GetCA;
        #ifndef NO_SKID
        typeof(GetCAByName) *GetCAByName;
        #ifdef HAVE_OCSP
        typeof(GetCAByKeyHash) *GetCAByKeyHash;
        #endif /* HAVE_OCSP */
        #endif /* NO_SKID */
        #ifdef WOLFSSL_AKID_NAME
        typeof(GetCAByAKID) *GetCAByAKID;
        #endif /* WOLFSSL_AKID_NAME */

        #if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)
        typeof(wolfSSL_X509_NAME_add_entry_by_NID) *wolfSSL_X509_NAME_add_entry_by_NID;
        typeof(wolfSSL_X509_NAME_free) *wolfSSL_X509_NAME_free;
        typeof(wolfSSL_X509_NAME_new_ex) *wolfSSL_X509_NAME_new_ex;
        #endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */

        #endif /* !WOLFCRYPT_ONLY && !NO_CERTS */

        typeof(dump_stack) *dump_stack;

        #ifdef CONFIG_ARM64
        #ifndef CONFIG_ARCH_TEGRA
        #ifdef WC_CONTAINERIZE_THIS
            /* alt_cb_patch_nops and queued_spin_lock_slowpath are defined early
             * to allow shimming in system headers, but now we need the native
             * ones.
             */
            #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)
            #undef alt_cb_patch_nops
            typeof(my__alt_cb_patch_nops) *alt_cb_patch_nops;
            #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) */
            #undef queued_spin_lock_slowpath
            typeof(my__queued_spin_lock_slowpath) *queued_spin_lock_slowpath;
        #else
            #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)
            typeof(alt_cb_patch_nops) *alt_cb_patch_nops;
            #endif
            typeof(queued_spin_lock_slowpath) *queued_spin_lock_slowpath;
        #endif
        #endif
        #endif

        typeof(preempt_count) *preempt_count;
        #ifndef _raw_spin_lock_irqsave
            typeof(_raw_spin_lock_irqsave) *_raw_spin_lock_irqsave;
        #endif
        #ifndef _raw_spin_trylock
            typeof(_raw_spin_trylock) *_raw_spin_trylock;
        #endif
        #ifndef _raw_spin_unlock_irqrestore
            typeof(_raw_spin_unlock_irqrestore) *_raw_spin_unlock_irqrestore;
        #endif
        typeof(_cond_resched) *_cond_resched;
        #ifndef WOLFSSL_LINUXKM_USE_MUTEXES
        typeof(wc_lkm_LockMutex) *wc_lkm_LockMutex;
        #endif

        typeof(wc_linuxkm_sig_ignore_begin) *wc_linuxkm_sig_ignore_begin;
        typeof(wc_linuxkm_sig_ignore_end) *wc_linuxkm_sig_ignore_end;
        typeof(wc_linuxkm_check_for_intr_signals) *wc_linuxkm_check_for_intr_signals;
        typeof(wc_linuxkm_relax_long_loop) *wc_linuxkm_relax_long_loop;

        #ifdef CONFIG_KASAN
            typeof(kasan_disable_current) *kasan_disable_current;
            typeof(kasan_enable_current) *kasan_enable_current;
        #endif

        const void *_last_slot;
    };

    extern const struct wolfssl_linuxkm_pie_redirect_table *wolfssl_linuxkm_get_pie_redirect_table(void);
    extern struct wolfssl_linuxkm_pie_redirect_table wolfssl_linuxkm_pie_redirect_table;


    #if defined(WC_PIE_INDIRECT_SYM)
        /* keep user-supplied override definition. */
    #elif defined(WC_PIE_INDIRECT_SYM_BY_FUNC_ONLY) || \
        defined(WC_PIE_INDIRECT_SYM_BY_DIRECT_TABLE_READ)
        /* keep user-supplied override method. */
    #elif defined(CONFIG_X86)
        #define WC_PIE_INDIRECT_SYM_BY_DIRECT_TABLE_READ
    #elif defined(CONFIG_ARM64)
        /* direct access to wolfssl_linuxkm_pie_redirect_table.x on aarch64
         * produces GOT relocations, e.g. R_AARCH64_LD64_GOT_LO12_NC.
         */
        #define WC_PIE_INDIRECT_SYM_BY_FUNC_ONLY
    #else
        /* for other archs, by default use the safe way. */
        #define WC_PIE_INDIRECT_SYM_BY_FUNC_ONLY
    #endif

    #if defined(WC_PIE_INDIRECT_SYM)
        /* keep user-supplied override definition. */
    #elif defined(WC_PIE_INDIRECT_SYM_BY_FUNC_ONLY)
        static const struct wolfssl_linuxkm_pie_redirect_table *wolfssl_linuxkm_pie_redirect_table_local_ptr = NULL;
        static inline const struct wolfssl_linuxkm_pie_redirect_table *wolfssl_linuxkm_get_pie_redirect_table_local(void) {
            if (wolfssl_linuxkm_pie_redirect_table_local_ptr == NULL)
                wolfssl_linuxkm_pie_redirect_table_local_ptr = wolfssl_linuxkm_get_pie_redirect_table();
            return wolfssl_linuxkm_pie_redirect_table_local_ptr;
        }
        #define WC_PIE_INDIRECT_SYM(x) (wolfssl_linuxkm_get_pie_redirect_table_local()->x)
    #elif defined(WC_PIE_INDIRECT_SYM_BY_DIRECT_TABLE_READ)
        #define WC_PIE_INDIRECT_SYM(x) (wolfssl_linuxkm_pie_redirect_table.x)
    #else
        #error no WC_PIE_INDIRECT_SYM method defined.
    #endif

    #ifdef WC_CONTAINERIZE_THIS

    #ifdef HAVE_FIPS
        #define wc_linuxkm_normalize_relocations \
            WC_PIE_INDIRECT_SYM(wc_linuxkm_normalize_relocations)
    #endif

    #ifndef __ARCH_MEMCMP_NO_REDIRECT
        #define memcmp WC_PIE_INDIRECT_SYM(memcmp)
    #endif
    #ifndef __ARCH_MEMCPY_NO_REDIRECT
        #define memcpy WC_PIE_INDIRECT_SYM(memcpy)
    #endif
    #ifndef __ARCH_MEMSET_NO_REDIRECT
        #define memset WC_PIE_INDIRECT_SYM(memset)
    #endif
    #ifndef __ARCH_MEMMOVE_NO_REDIRECT
        #define memmove WC_PIE_INDIRECT_SYM(memmove)
    #endif
    #ifndef __ARCH_STRCMP_NO_REDIRECT
        #define strcmp WC_PIE_INDIRECT_SYM(strcmp)
    #endif
    #ifndef __ARCH_STRNCMP_NO_REDIRECT
        #define strncmp WC_PIE_INDIRECT_SYM(strncmp)
    #endif
    #ifndef __ARCH_STRCASECMP_NO_REDIRECT
        #define strcasecmp WC_PIE_INDIRECT_SYM(strcasecmp)
    #endif
    #ifndef __ARCH_STRNCASECMP_NO_REDIRECT
        #define strncasecmp WC_PIE_INDIRECT_SYM(strncasecmp)
    #endif
    #ifndef __ARCH_STRLEN_NO_REDIRECT
        #define strlen WC_PIE_INDIRECT_SYM(strlen)
    #endif
    #ifndef __ARCH_STRSTR_NO_REDIRECT
        #define strstr WC_PIE_INDIRECT_SYM(strstr)
    #endif
    #ifndef __ARCH_STRNCPY_NO_REDIRECT
        #define strncpy WC_PIE_INDIRECT_SYM(strncpy)
    #endif
    #ifndef __ARCH_STRNCAT_NO_REDIRECT
        #define strncat WC_PIE_INDIRECT_SYM(strncat)
    #endif
    #define kstrtoll WC_PIE_INDIRECT_SYM(kstrtoll)

    #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
        (defined(RHEL_MAJOR) && \
         ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
        #define _printk WC_PIE_INDIRECT_SYM(_printk)
    #else
        #define printk WC_PIE_INDIRECT_SYM(printk)
    #endif

    #ifdef CONFIG_FORTIFY_SOURCE
        #define __warn_printk WC_PIE_INDIRECT_SYM(__warn_printk)
    #endif

    #define snprintf WC_PIE_INDIRECT_SYM(snprintf)

    #define _ctype WC_PIE_INDIRECT_SYM(_ctype)

#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0)
    /* see include/linux/alloc_tag.h and include/linux/slab.h */
    #define kmalloc_noprof WC_PIE_INDIRECT_SYM(kmalloc_noprof)
    #define krealloc_node_align_noprof WC_PIE_INDIRECT_SYM(krealloc_node_align_noprof)
    #define kzalloc_noprof WC_PIE_INDIRECT_SYM(kzalloc_noprof)
    #define __kvmalloc_node_noprof WC_PIE_INDIRECT_SYM(__kvmalloc_node_noprof)
    #define __kmalloc_cache_noprof WC_PIE_INDIRECT_SYM(__kmalloc_cache_noprof)
    #ifdef HAVE_KVREALLOC
        #define kvrealloc_node_align_noprof WC_PIE_INDIRECT_SYM(kvrealloc_node_align_noprof)
    #endif
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 11, 0)
    /* see include/linux/alloc_tag.h and include/linux/slab.h */
    #define kmalloc_noprof WC_PIE_INDIRECT_SYM(kmalloc_noprof)
    #define krealloc_noprof WC_PIE_INDIRECT_SYM(krealloc_noprof)
    #define kzalloc_noprof WC_PIE_INDIRECT_SYM(kzalloc_noprof)
    #define __kvmalloc_node_noprof WC_PIE_INDIRECT_SYM(__kvmalloc_node_noprof)
    #define __kmalloc_cache_noprof WC_PIE_INDIRECT_SYM(__kmalloc_cache_noprof)
    #ifdef HAVE_KVREALLOC
        #define kvrealloc_noprof WC_PIE_INDIRECT_SYM(kvrealloc_noprof)
    #endif
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 10, 0)
    /* see include/linux/alloc_tag.h and include/linux/slab.h */
    #define kmalloc_noprof WC_PIE_INDIRECT_SYM(kmalloc_noprof)
    #define krealloc_noprof WC_PIE_INDIRECT_SYM(krealloc_noprof)
    #define kzalloc_noprof WC_PIE_INDIRECT_SYM(kzalloc_noprof)
    #define kvmalloc_node_noprof WC_PIE_INDIRECT_SYM(kvmalloc_node_noprof)
    #define kmalloc_trace_noprof WC_PIE_INDIRECT_SYM(kmalloc_trace_noprof)
    #ifdef HAVE_KVREALLOC
        #define kvrealloc_noprof WC_PIE_INDIRECT_SYM(kvrealloc_noprof)
    #endif
#else /* <6.10.0 */
    #define kmalloc WC_PIE_INDIRECT_SYM(kmalloc)
    #define krealloc WC_PIE_INDIRECT_SYM(krealloc)
    #define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO)
    #ifdef HAVE_KVMALLOC
        #define kvmalloc_node WC_PIE_INDIRECT_SYM(kvmalloc_node)
    #endif
    #ifdef HAVE_KVREALLOC
        #define kvrealloc WC_PIE_INDIRECT_SYM(kvrealloc)
    #endif
    #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) || \
        (defined(RHEL_MAJOR) &&                                    \
         ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
        #define kmalloc_trace WC_PIE_INDIRECT_SYM(kmalloc_trace)
    #else
        #define kmem_cache_alloc_trace WC_PIE_INDIRECT_SYM(kmem_cache_alloc_trace)
        #define kmalloc_order_trace WC_PIE_INDIRECT_SYM(kmalloc_order_trace)
    #endif
#endif /* <6.10.0 */

    #define kfree WC_PIE_INDIRECT_SYM(kfree)
    #ifdef HAVE_KVMALLOC
        #define kvfree WC_PIE_INDIRECT_SYM(kvfree)
    #endif
    #define ksize WC_PIE_INDIRECT_SYM(ksize)

#ifndef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
    #define get_random_bytes WC_PIE_INDIRECT_SYM(get_random_bytes)
#endif
    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
        #define getnstimeofday WC_PIE_INDIRECT_SYM(getnstimeofday)
    #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
        #define current_kernel_time64 WC_PIE_INDIRECT_SYM(current_kernel_time64)
    #else
        #define ktime_get_coarse_real_ts64 WC_PIE_INDIRECT_SYM(ktime_get_coarse_real_ts64)
    #endif

    #undef get_current
    #define get_current WC_PIE_INDIRECT_SYM(get_current)

    #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
        #define allocate_wolfcrypt_linuxkm_fpu_states WC_PIE_INDIRECT_SYM(allocate_wolfcrypt_linuxkm_fpu_states)
        #define wc_can_save_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_can_save_vector_registers_x86)
        #define free_wolfcrypt_linuxkm_fpu_states WC_PIE_INDIRECT_SYM(free_wolfcrypt_linuxkm_fpu_states)
        #define wc_restore_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_restore_vector_registers_x86)
        #define wc_save_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_save_vector_registers_x86)
    #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
        #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.
    #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */

    #ifndef CONFIG_PREEMPT_RT
        #define __mutex_init WC_PIE_INDIRECT_SYM(__mutex_init)
    #else
        /* On RT kernels, __mutex_init is a macro pointing to __rt_mutex_init */
        #undef __mutex_init
        #define __rt_mutex_init WC_PIE_INDIRECT_SYM(__rt_mutex_init)
        #define __mutex_init(mutex, name, key) __rt_mutex_init(mutex, name, key)
        #define rt_mutex_base_init WC_PIE_INDIRECT_SYM(rt_mutex_base_init)
        #define rt_spin_lock WC_PIE_INDIRECT_SYM(rt_spin_lock)
        #define rt_spin_unlock WC_PIE_INDIRECT_SYM(rt_spin_unlock)
    #endif
    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
        #define mutex_lock_nested WC_PIE_INDIRECT_SYM(mutex_lock_nested)
    #else
        #define mutex_lock WC_PIE_INDIRECT_SYM(mutex_lock)
    #endif
    #define mutex_unlock WC_PIE_INDIRECT_SYM(mutex_unlock)
    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
        #define mutex_destroy WC_PIE_INDIRECT_SYM(mutex_destroy)
    #endif

    /* per linux/ctype.h, tolower() and toupper() are macros bound to static inlines
     * that use macros that bring in the _ctype global.  for WC_CONTAINERIZE_THIS,
     * this needs to be masked out.
     */
    #undef tolower
    #undef toupper
    #define tolower(c) (islower(c) ? (c) : ((c) + ('a'-'A')))
    #define toupper(c) (isupper(c) ? (c) : ((c) - ('a'-'A')))

    #if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS)
        #define GetCA WC_PIE_INDIRECT_SYM(GetCA)
        #ifndef NO_SKID
            #define GetCAByName WC_PIE_INDIRECT_SYM(GetCAByName)
            #ifdef HAVE_OCSP
                #define GetCAByKeyHash WC_PIE_INDIRECT_SYM(GetCAByKeyHash)
            #endif /* HAVE_OCSP */
        #endif /* NO_SKID */
        #ifdef WOLFSSL_AKID_NAME
            #ifdef WOLFSSL_API_PREFIX_MAP
                #define wolfSSL_GetCAByAKID WC_PIE_INDIRECT_SYM(wolfSSL_GetCAByAKID)
            #else
                #define GetCAByAKID WC_PIE_INDIRECT_SYM(GetCAByAKID)
            #endif
        #endif

        #if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)
            #define wolfSSL_X509_NAME_add_entry_by_NID WC_PIE_INDIRECT_SYM(wolfSSL_X509_NAME_add_entry_by_NID)
            #define wolfSSL_X509_NAME_free WC_PIE_INDIRECT_SYM(wolfSSL_X509_NAME_free)
            #define wolfSSL_X509_NAME_new_ex WC_PIE_INDIRECT_SYM(wolfSSL_X509_NAME_new_ex)
        #endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */

    #endif /* !WOLFCRYPT_ONLY && !NO_CERTS */

        #define dump_stack WC_PIE_INDIRECT_SYM(dump_stack)

    #undef preempt_count /* just in case -- not a macro on x86. */
    #define preempt_count WC_PIE_INDIRECT_SYM(preempt_count)

    #ifndef WOLFSSL_LINUXKM_USE_MUTEXES
        #ifndef _raw_spin_lock_irqsave
            #define _raw_spin_lock_irqsave WC_PIE_INDIRECT_SYM(_raw_spin_lock_irqsave)
        #endif
        #ifndef _raw_spin_trylock
            #define _raw_spin_trylock WC_PIE_INDIRECT_SYM(_raw_spin_trylock)
        #endif
        #ifndef _raw_spin_unlock_irqrestore
            #define _raw_spin_unlock_irqrestore WC_PIE_INDIRECT_SYM(_raw_spin_unlock_irqrestore)
        #endif
    #endif

    #define _cond_resched WC_PIE_INDIRECT_SYM(_cond_resched)

    /* this is defined in linux/spinlock.h as an inline that calls the unshimmed
     * raw_spin_unlock_irqrestore().  use a macro here to supersede it.
     * Note: On PREEMPT_RT kernels, spinlock_t doesn't have rlock member,
     * so we skip this redefinition and use the kernel's native implementation.
     */
    #ifndef CONFIG_PREEMPT_RT
        #define spin_unlock_irqrestore(lock, flags) raw_spin_unlock_irqrestore(&((lock)->rlock), flags)
    #else
        /* Undo internal wolfSSL PIE macro rewriting */
        #ifdef rt_spin_unlock
        #undef rt_spin_unlock
        #endif
        #ifdef rt_spin_lock
        #undef rt_spin_lock
        #endif
        static inline int wolfssl_spin_unlock_irqrestore_rt(spinlock_t *lock,
                                                             unsigned long flags)
        {
            (void)flags; /* rt_spin_unlock ignores flags */
            WC_PIE_INDIRECT_SYM(rt_spin_unlock)(lock);
            return 0;
        }

        #undef spin_unlock_irqrestore
        #define spin_unlock_irqrestore(lock, flags) \
        wolfssl_spin_unlock_irqrestore_rt((lock), (flags))
    #endif

    #define wc_linuxkm_sig_ignore_begin WC_PIE_INDIRECT_SYM(wc_linuxkm_sig_ignore_begin);
    #define wc_linuxkm_sig_ignore_end WC_PIE_INDIRECT_SYM(wc_linuxkm_sig_ignore_end);
    #define wc_linuxkm_check_for_intr_signals WC_PIE_INDIRECT_SYM(wc_linuxkm_check_for_intr_signals)
    #define wc_linuxkm_relax_long_loop WC_PIE_INDIRECT_SYM(wc_linuxkm_relax_long_loop)

    #ifdef CONFIG_KASAN
        #define kasan_disable_current WC_PIE_INDIRECT_SYM(kasan_disable_current)
        #define kasan_enable_current WC_PIE_INDIRECT_SYM(kasan_enable_current)
    #endif

    #endif /* WC_CONTAINERIZE_THIS */

    #endif /* WC_SYM_RELOC_TABLES */

#if defined(WOLFSSL_KERNEL_STACK_DEBUG) || defined(WC_LINUXKM_STACK_DEBUG)

    #ifndef CONFIG_THREAD_INFO_IN_TASK
        #error WC_LINUXKM_STACK_DEBUG requires CONFIG_THREAD_INFO_IN_TASK
    #endif
    #ifdef CONFIG_STACK_GROWSUP
        #error WC_LINUXKM_STACK_DEBUG requires !CONFIG_STACK_GROWSUP
    #endif

    static __always_inline unsigned long wc_linuxkm_stack_bottom(void) {
        void *ret = task_stack_page(get_current());
        return (unsigned long)(uintptr_t)ret;
    }

    static __always_inline unsigned long wc_linuxkm_stack_top(void) {
        return wc_linuxkm_stack_bottom() + THREAD_SIZE;
    }

    #if defined(CONFIG_X86)

    static __always_inline unsigned long wc_linuxkm_stack_current(void) {
        unsigned long rsp;

        asm volatile("mov %%rsp, %0" : "=r" (rsp));
        return wc_linuxkm_stack_top() - rsp;
    }

    static __always_inline unsigned long wc_linuxkm_stack_left(void) {
        unsigned long rsp;
        asm volatile("mov %%rsp, %0" : "=r" (rsp));
        return rsp - wc_linuxkm_stack_bottom();
    }

    #define WC_LINUXKM_HAVE_STACK_DEBUG

    #elif defined(CONFIG_ARM64)

    static __always_inline unsigned long wc_linuxkm_stack_current(void) {
        unsigned long sp;
        asm volatile("mov %0, sp" : "=r" (sp));
        return wc_linuxkm_stack_top() - sp;
    }

    static __always_inline unsigned long wc_linuxkm_stack_left(void) {
        unsigned long sp;
        asm volatile("mov %0, sp" : "=r" (sp));
        return sp - wc_linuxkm_stack_bottom();
    }

    #define WC_LINUXKM_HAVE_STACK_DEBUG

    #elif defined(CONFIG_ARM)

    static __always_inline unsigned long wc_linuxkm_stack_current(void) {
        unsigned long sp;
        asm volatile("mov %0, sp" : "=r" (sp));
        return wc_linuxkm_stack_top() - sp;
    }

    static __always_inline unsigned long wc_linuxkm_stack_left(void) {
        unsigned long sp;
        asm volatile("mov %0, sp" : "=r" (sp));
        return sp - wc_linuxkm_stack_bottom();
    }

    #define WC_LINUXKM_HAVE_STACK_DEBUG

    #endif /* CONFIG_ARM */

    #ifndef WC_LINUXKM_HAVE_STACK_DEBUG
        #error WC_LINUXKM_STACK_DEBUG implementation missing for target.
    #endif

    /* An unsigned long STACK_END_MAGIC is stored at the bottom of the stack.
     * Additionally, though the kernel stack doesn't have a red zone, it
     * nonetheless uses some bytes below the current stack pointer and mayhem
     * ensues immediately if it's overwritten.
     */
    #ifndef WC_KERNEL_STACK_MARGIN_BOTTOM
        #define WC_KERNEL_STACK_MARGIN_BOTTOM sizeof(unsigned long)
    #endif
    #ifndef WC_KERNEL_STACK_MARGIN_TOP
        #define WC_KERNEL_STACK_MARGIN_TOP 8
    #endif

    static __always_inline void wc_linuxkm_stack_hwm_prepare(unsigned char sentinel) {
        unsigned long s = wc_linuxkm_stack_bottom();
        unsigned long z;
        unsigned long flags;

        if (*(unsigned long *)s != STACK_END_MAGIC)
            pr_err("ERROR: bottom of stack is not STACK_END_MAGIC.\n");

        local_irq_save(flags);
        kasan_disable_current();
        z = wc_linuxkm_stack_left();
        if (z > WC_KERNEL_STACK_MARGIN_BOTTOM + WC_KERNEL_STACK_MARGIN_TOP)
            memset((void *)(s + WC_KERNEL_STACK_MARGIN_BOTTOM), sentinel,
                   z - (WC_KERNEL_STACK_MARGIN_BOTTOM + WC_KERNEL_STACK_MARGIN_TOP));
        kasan_enable_current();
        local_irq_restore(flags);
        if (z <= WC_KERNEL_STACK_MARGIN_BOTTOM + WC_KERNEL_STACK_MARGIN_TOP)
            pr_err("ERROR: wc_linuxkm_stack_hwm_prepare() called with only %lu bytes of stack left, "
                   "versus margin %zu.\n", z, WC_KERNEL_STACK_MARGIN_BOTTOM + WC_KERNEL_STACK_MARGIN_TOP);
    }
    static __always_inline unsigned long wc_linuxkm_stack_hwm_measure_rel(unsigned char sentinel) {
        unsigned long s = wc_linuxkm_stack_bottom();
        unsigned long z = wc_linuxkm_stack_left();
        unsigned char *i;
        if (z <= WC_KERNEL_STACK_MARGIN_BOTTOM + WC_KERNEL_STACK_MARGIN_TOP)
            return (unsigned long)-1;
        kasan_disable_current();
        for (i = (unsigned char *)s + WC_KERNEL_STACK_MARGIN_BOTTOM;
             i < ((unsigned char *)s + z) && (*i == sentinel);
             ++i);
        kasan_enable_current();
        return z - ((unsigned long)i - s);
    }
    static __always_inline unsigned long wc_linuxkm_stack_hwm_measure_total(unsigned char sentinel) {
        unsigned long rel = wc_linuxkm_stack_hwm_measure_rel(sentinel);
        if (rel == (unsigned long)-1)
            return rel;
        else
            return rel + wc_linuxkm_stack_current();
    }

#endif /* WC_LINUXKM_STACK_DEBUG */

    /* remove this multifariously conflicting macro, picked up from
     * Linux arch/<arch>/include/asm/current.h.
     */
    #ifndef WOLFSSL_LINUXKM_NEED_LINUX_CURRENT
        #undef current
    #endif

    /* min() and max() in linux/kernel.h over-aggressively type-check, producing
     * myriad spurious -Werrors throughout the codebase.
     */
    #undef min
    #undef max

    /* work around namespace conflict between wolfssl/internal.h (enum HandShakeType)
     * and linux/key.h (extern int()).
     */
    #define key_update wc_key_update

    #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
        (defined(RHEL_MAJOR) && \
         ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
        #define wc_km_printf(format, args...) _printk(KERN_INFO "wolfssl: %s(): " format, __func__, ## args)
    #else
        #define wc_km_printf(format, args...) printk(KERN_INFO "wolfssl: %s(): " format, __func__, ## args)
    #endif
    #define printf(...) wc_km_printf(__VA_ARGS__)

    #ifdef HAVE_FIPS
        extern void fipsEntry(void);
    #endif

    /* suppress false-positive "writing 1 byte into a region of size 0" warnings
     * building old kernels with new gcc:
     */
    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
    _Pragma("GCC diagnostic ignored \"-Wstringop-overflow\"");
    #endif

    /* includes are all above, with incompatible warnings masked out. */
    #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0)
    typedef __kernel_time_t time_t;
    #else
    typedef __kernel_time64_t time_t;
    #endif
    extern time_t time(time_t * timer);
    #define XTIME time
    #define WOLFSSL_GMTIME
    #define XGMTIME(c, t) gmtime(c)
    #define NO_TIMEVAL 1

    #endif /* BUILDING_WOLFSSL */

    #if !defined(BUILDING_WOLFSSL)
        /* some caller code needs these. */
        #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
            #if defined(CONFIG_X86)
                WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void);
                WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags);
                WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags);
                #ifndef DISABLE_VECTOR_REGISTERS
                    #define DISABLE_VECTOR_REGISTERS() wc_save_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
                #endif
                #ifndef REENABLE_VECTOR_REGISTERS
                    #define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
                #endif
            #else /* !CONFIG_X86 */
                #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.
            #endif /* !CONFIG_X86 */
        #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */
    #endif /* !BUILDING_WOLFSSL */

    /* Copied from wc_port.h */
    #if defined(HAVE_FIPS) && !defined(WOLFSSL_API_PREFIX_MAP)
        /* For FIPS keep the function names the same */
        #define wc_InitMutex   InitMutex
        #define wc_FreeMutex   FreeMutex
        #define wc_LockMutex   LockMutex
        #define wc_UnLockMutex UnLockMutex
    #endif /* HAVE_FIPS */

    #ifdef WOLFSSL_LINUXKM_USE_MUTEXES
        #define WC_MUTEX_OPS_INLINE

        #ifdef LINUXKM_LKCAPI_REGISTER
            /* must use spin locks when registering implementations with the
             * kernel, because mutexes are forbidden when calling with nonzero
             * irq_count().
             */
            #error WOLFSSL_LINUXKM_USE_MUTEXES is incompatible with LINUXKM_LKCAPI_REGISTER.
        #endif

        /* if BUILDING_WOLFSSL, mutex.h will have already been included
         * recursively above, with the bevy of warnings suppressed, and the
         * below include will be a redundant no-op.
         */
        #include <linux/mutex.h>
        typedef struct mutex wolfSSL_Mutex;
        #define WOLFSSL_MUTEX_INITIALIZER(lockname) __MUTEX_INITIALIZER(lockname)

        /* Linux kernel mutex routines are voids, alas. */

        static inline int wc_InitMutex(wolfSSL_Mutex* m)
        {
            mutex_init(m);
            return 0;
        }

        static inline int wc_FreeMutex(wolfSSL_Mutex* m)
        {
            mutex_destroy(m);
            return 0;
        }

        static __must_check inline int wc_LockMutex(wolfSSL_Mutex* m)
        {
            if (in_nmi() || hardirq_count() || in_softirq())
                return -1;
            mutex_lock(m);
            return 0;
        }

        static inline int wc_UnLockMutex(wolfSSL_Mutex* m)
        {
            mutex_unlock(m);
            return 0;
        }
    #else
        #define WC_MUTEX_OPS_INLINE

        /* if BUILDING_WOLFSSL, spinlock.h will have already been included
         * recursively above, with the bevy of warnings suppressed, and the
         * below include will be a redundant no-op.
         */
        #include <linux/spinlock.h>

        typedef struct wolfSSL_Mutex {
            spinlock_t lock;
            unsigned long irq_flags;
        } wolfSSL_Mutex;

        #define WOLFSSL_MUTEX_INITIALIZER(lockname) { .lock =__SPIN_LOCK_UNLOCKED(lockname), .irq_flags = 0 }

        static __always_inline int wc_InitMutex(wolfSSL_Mutex* m)
        {
        /* Tegra vendor kernels do not support assignment of __SPIN_LOCK_UNLOCKED() */
        # ifndef CONFIG_ARCH_TEGRA
            m->lock = __SPIN_LOCK_UNLOCKED(m);
        # else
            spin_lock_init(&m->lock);
        #endif
            m->irq_flags = 0;

            return 0;
        }

        static __always_inline int wc_FreeMutex(wolfSSL_Mutex* m)
        {
            (void)m;
            return 0;
        }

        #ifdef WC_CONTAINERIZE_THIS
        /* wc_lkm_LockMutex() can't be used inline in WC_CONTAINERIZE_THIS objects, due to
         * direct access to pv_ops.
         */
        static __must_check __always_inline int wc_LockMutex(wolfSSL_Mutex *m)
        {
            return WC_PIE_INDIRECT_SYM(wc_lkm_LockMutex)(m);
        }

        #else /* !WC_CONTAINERIZE_THIS */

        static __must_check __always_inline int wc_LockMutex(wolfSSL_Mutex *m)
        {
            return wc_lkm_LockMutex(m);
        }

        #endif /* !WC_CONTAINERIZE_THIS */

        static __always_inline int wc_UnLockMutex(wolfSSL_Mutex* m)
        {
            spin_unlock_irqrestore(&m->lock, m->irq_flags);
            return 0;
        }

    #endif

    #ifdef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
        struct crypto_rng;
        WOLFSSL_API int wc_linux_kernel_rng_is_wolfcrypt(struct crypto_rng *rng);
    #endif

    /* Undo copied defines from wc_port.h, to avoid redefinition warnings. */
    #ifdef HAVE_FIPS
    #undef wc_InitMutex
    #undef wc_FreeMutex
    #undef wc_LockMutex
    #undef wc_UnLockMutex
    #endif /* HAVE_FIPS */

    /* prevent gcc's mm_malloc.h from being included, since it unconditionally
     * includes stdlib.h, which is kernel-incompatible.
     */
    #define _MM_MALLOC_H_INCLUDED

    #ifndef BUILDING_WOLFSSL
        #include <linux/slab.h>
        #if defined(USE_KVMALLOC) && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0))
            #include <linux/mm.h>
        #endif
    #endif

    /* fun fact: since linux commit 59bb47985c, kmalloc with power-of-2 size is
     * aligned to the size.
     */
    #define WC_LINUXKM_ROUND_UP_P_OF_2(x) (                                \
    {                                                                      \
        size_t _alloc_sz = (x);                                            \
        if ((_alloc_sz < 8192) && (_alloc_sz != 0))                        \
          _alloc_sz = 1UL <<                                               \
              ((sizeof(_alloc_sz) * 8UL) - __builtin_clzl(_alloc_sz - 1)); \
        _alloc_sz;                                                         \
    })
    #ifdef USE_KVMALLOC
        #define malloc(size) kvmalloc_node(WC_LINUXKM_ROUND_UP_P_OF_2(size), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC), NUMA_NO_NODE)
        #define free(ptr) kvfree(ptr)
        #ifdef USE_KVREALLOC
            #define realloc(ptr, newsize) kvrealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC))
        #else
            #define realloc(ptr, newsize) ((void)(ptr), (void)(newsize), NULL)
        #endif
    #else
        #define malloc(size) kmalloc(WC_LINUXKM_ROUND_UP_P_OF_2(size), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC))
        #define free(ptr) kfree(ptr)
        #define realloc(ptr, newsize) krealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC))
    #endif

    #ifndef static_assert
        #define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr)
        #define __static_assert(expr, msg, ...) _Static_assert(expr, msg)
    #endif

#ifdef WOLFSSL_TRACK_MEMORY
    #define XMALLOC(s, h, t)     ({(void)(h); (void)(t); wolfSSL_Malloc(s);})
    #ifdef WOLFSSL_XFREE_NO_NULLNESS_CHECK
        #define XFREE(p, h, t)       ({(void)(h); (void)(t); wolfSSL_Free(p);})
    #else
        #define XFREE(p, h, t)       ({void* _xp; (void)(h); _xp = (p); if(_xp) wolfSSL_Free(_xp);})
    #endif
    #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); wolfSSL_Realloc(p, n);})
#else
    #if !defined(XMALLOC_USER) && !defined(XMALLOC_OVERRIDE)
        #define XMALLOC(s, h, t)     ({(void)(h); (void)(t); malloc(s);})
        #ifdef WOLFSSL_XFREE_NO_NULLNESS_CHECK
            #define XFREE(p, h, t)       ({(void)(h); (void)(t); free(p);})
        #else
            #define XFREE(p, h, t)       ({void* _xp; (void)(h); (void)(t); _xp = (p); if(_xp) free(_xp);})
        #endif
        #if defined(USE_KVREALLOC) || !defined(USE_KVMALLOC)
            #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); realloc(p, n);})
        #endif
    #endif /* !XMALLOC_USER && !XMALLOC_OVERRIDE */
#endif

    #include <linux/limits.h>

    #ifndef INT32_MAX
        #define INT32_MAX INT_MAX
    #endif
    #ifndef UINT32_MAX
        #define UINT32_MAX UINT_MAX
    #endif

    /* Linux headers define these using C expressions, but we need
     * them to be evaluable by the preprocessor, for use in sp_int.h.
     */
    #if BITS_PER_LONG == 64
        /* NOLINTBEGIN(bugprone-sizeof-expression) */
        static_assert(sizeof(ULONG_MAX) == 8,
                       "BITS_PER_LONG is 64, but ULONG_MAX is not.");
        /* NOLINTEND(bugprone-sizeof-expression) */

        #undef UCHAR_MAX
        #define UCHAR_MAX 255
        #undef USHRT_MAX
        #define USHRT_MAX 65535
        #undef UINT_MAX
        #define UINT_MAX 4294967295U
        #undef ULONG_MAX
        #define ULONG_MAX 18446744073709551615UL
        #undef ULLONG_MAX
        #define ULLONG_MAX ULONG_MAX
        #undef INT_MAX
        #define INT_MAX 2147483647
        #undef LONG_MAX
        #define LONG_MAX 9223372036854775807L
        #undef LLONG_MAX
        #define LLONG_MAX LONG_MAX

    #elif BITS_PER_LONG == 32

        static_assert(sizeof(ULONG_MAX) == 4,
                       "BITS_PER_LONG is 32, but ULONG_MAX is not.");

        #undef UCHAR_MAX
        #define UCHAR_MAX 255
        #undef USHRT_MAX
        #define USHRT_MAX 65535
        #undef UINT_MAX
        #define UINT_MAX 4294967295U
        #undef ULONG_MAX
        #define ULONG_MAX 4294967295UL
        #undef INT_MAX
        #define INT_MAX 2147483647
        #undef LONG_MAX
        #define LONG_MAX 2147483647L

        #undef ULLONG_MAX
        #undef LLONG_MAX
        #if BITS_PER_LONG_LONG == 64
            #define ULLONG_MAX 18446744073709551615UL
            #define LLONG_MAX 9223372036854775807L
        #else
            #undef NO_64BIT
            #define NO_64BIT
            #define ULLONG_MAX ULONG_MAX
            #define LLONG_MAX LONG_MAX
        #endif

#else
        #error unexpected BITS_PER_LONG value.
#endif

/* WC_DUMP_BACKTRACE_NONDEBUG is intended to dump a backtrace only if it hasn't
 * already been dumped by the called function.
 */
#if defined(WOLFSSL_DEBUG_TRACE_ERROR_CODES) && defined(WOLFSSL_DEBUG_BACKTRACE_ERROR_CODES)
    #define WC_DUMP_BACKTRACE_NONDEBUG WC_DO_NOTHING
#else
    #define WC_DUMP_BACKTRACE_NONDEBUG dump_stack()
#endif

#endif /* LINUXKM_WC_PORT_H */