mirror of https://github.com/torvalds/linux.git
lib/crypto: riscv/sha512: Migrate optimized SHA-512 code to library
Instead of exposing the riscv-optimized SHA-512 code via riscv-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be riscv-optimized, and it fixes the longstanding issue where the riscv-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha512_blocks(), change the type of the nblocks parameter of the assembly function from int to size_t. The assembly function actually already treated it as size_t. Note: to see the diff from arch/riscv/crypto/sha512-riscv64-glue.c to lib/crypto/riscv/sha512.h, view this commit with 'git show -M10'. Acked-by: Ard Biesheuvel <ardb@kernel.org> Link: https://lore.kernel.org/r/20250630160320.2888-12-ebiggers@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>
This commit is contained in:
Eric Biggers
parent
7117739ad2
commit
b59059a22c
|
|
@ -28,18 +28,6 @@ config CRYPTO_GHASH_RISCV64
|
|||
Architecture: riscv64 using:
|
||||
- Zvkg vector crypto extension
|
||||
|
||||
config CRYPTO_SHA512_RISCV64
|
||||
tristate "Hash functions: SHA-384 and SHA-512"
|
||||
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
|
||||
select CRYPTO_LIB_SHA512
|
||||
select CRYPTO_SHA512
|
||||
help
|
||||
SHA-384 and SHA-512 secure hash algorithm (FIPS 180)
|
||||
|
||||
Architecture: riscv64 using:
|
||||
- Zvknhb vector crypto extension
|
||||
- Zvkb vector crypto extension
|
||||
|
||||
config CRYPTO_SM3_RISCV64
|
||||
tristate "Hash functions: SM3 (ShangMi 3)"
|
||||
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
|
||||
|
|
|
|||
|
|
@ -7,9 +7,6 @@ aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \
|
|||
obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o
|
||||
ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_SHA512_RISCV64) += sha512-riscv64.o
|
||||
sha512-riscv64-y := sha512-riscv64-glue.o sha512-riscv64-zvknhb-zvkb.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o
|
||||
sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o
|
||||
|
||||
|
|
|
|||
|
|
@ -1,130 +0,0 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
/*
|
||||
* SHA-512 and SHA-384 using the RISC-V vector crypto extensions
|
||||
*
|
||||
* Copyright (C) 2023 VRULL GmbH
|
||||
* Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
|
||||
*
|
||||
* Copyright (C) 2023 SiFive, Inc.
|
||||
* Author: Jerry Shih <jerry.shih@sifive.com>
|
||||
*/
|
||||
|
||||
#include <asm/simd.h>
|
||||
#include <asm/vector.h>
|
||||
#include <crypto/internal/hash.h>
|
||||
#include <crypto/internal/simd.h>
|
||||
#include <crypto/sha512_base.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
|
||||
/*
|
||||
* Note: the asm function only uses the 'state' field of struct sha512_state.
|
||||
* It is assumed to be the first field.
|
||||
*/
|
||||
asmlinkage void sha512_transform_zvknhb_zvkb(
|
||||
struct sha512_state *state, const u8 *data, int num_blocks);
|
||||
|
||||
static void sha512_block(struct sha512_state *state, const u8 *data,
|
||||
int num_blocks)
|
||||
{
|
||||
/*
|
||||
* Ensure struct sha512_state begins directly with the SHA-512
|
||||
* 512-bit internal state, as this is what the asm function expects.
|
||||
*/
|
||||
BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);
|
||||
|
||||
if (crypto_simd_usable()) {
|
||||
kernel_vector_begin();
|
||||
sha512_transform_zvknhb_zvkb(state, data, num_blocks);
|
||||
kernel_vector_end();
|
||||
} else {
|
||||
struct __sha512_ctx ctx = {};
|
||||
|
||||
static_assert(sizeof(ctx.state) == sizeof(state->state));
|
||||
memcpy(&ctx.state, state->state, sizeof(ctx.state));
|
||||
__sha512_update(&ctx, data,
|
||||
(size_t)num_blocks * SHA512_BLOCK_SIZE);
|
||||
memcpy(state->state, &ctx.state, sizeof(state->state));
|
||||
}
|
||||
}
|
||||
|
||||
static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len)
|
||||
{
|
||||
return sha512_base_do_update_blocks(desc, data, len, sha512_block);
|
||||
}
|
||||
|
||||
static int riscv64_sha512_finup(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len, u8 *out)
|
||||
{
|
||||
sha512_base_do_finup(desc, data, len, sha512_block);
|
||||
return sha512_base_finish(desc, out);
|
||||
}
|
||||
|
||||
static int riscv64_sha512_digest(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len, u8 *out)
|
||||
{
|
||||
return sha512_base_init(desc) ?:
|
||||
riscv64_sha512_finup(desc, data, len, out);
|
||||
}
|
||||
|
||||
static struct shash_alg riscv64_sha512_algs[] = {
|
||||
{
|
||||
.init = sha512_base_init,
|
||||
.update = riscv64_sha512_update,
|
||||
.finup = riscv64_sha512_finup,
|
||||
.digest = riscv64_sha512_digest,
|
||||
.descsize = SHA512_STATE_SIZE,
|
||||
.digestsize = SHA512_DIGEST_SIZE,
|
||||
.base = {
|
||||
.cra_blocksize = SHA512_BLOCK_SIZE,
|
||||
.cra_priority = 300,
|
||||
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
|
||||
CRYPTO_AHASH_ALG_FINUP_MAX,
|
||||
.cra_name = "sha512",
|
||||
.cra_driver_name = "sha512-riscv64-zvknhb-zvkb",
|
||||
.cra_module = THIS_MODULE,
|
||||
},
|
||||
}, {
|
||||
.init = sha384_base_init,
|
||||
.update = riscv64_sha512_update,
|
||||
.finup = riscv64_sha512_finup,
|
||||
.descsize = SHA512_STATE_SIZE,
|
||||
.digestsize = SHA384_DIGEST_SIZE,
|
||||
.base = {
|
||||
.cra_blocksize = SHA384_BLOCK_SIZE,
|
||||
.cra_priority = 300,
|
||||
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
|
||||
CRYPTO_AHASH_ALG_FINUP_MAX,
|
||||
.cra_name = "sha384",
|
||||
.cra_driver_name = "sha384-riscv64-zvknhb-zvkb",
|
||||
.cra_module = THIS_MODULE,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
static int __init riscv64_sha512_mod_init(void)
|
||||
{
|
||||
if (riscv_isa_extension_available(NULL, ZVKNHB) &&
|
||||
riscv_isa_extension_available(NULL, ZVKB) &&
|
||||
riscv_vector_vlen() >= 128)
|
||||
return crypto_register_shashes(riscv64_sha512_algs,
|
||||
ARRAY_SIZE(riscv64_sha512_algs));
|
||||
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
static void __exit riscv64_sha512_mod_exit(void)
|
||||
{
|
||||
crypto_unregister_shashes(riscv64_sha512_algs,
|
||||
ARRAY_SIZE(riscv64_sha512_algs));
|
||||
}
|
||||
|
||||
module_init(riscv64_sha512_mod_init);
|
||||
module_exit(riscv64_sha512_mod_exit);
|
||||
|
||||
MODULE_DESCRIPTION("SHA-512 (RISC-V accelerated)");
|
||||
MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_ALIAS_CRYPTO("sha512");
|
||||
MODULE_ALIAS_CRYPTO("sha384");
|
||||
|
|
@ -180,6 +180,7 @@ config CRYPTO_LIB_SHA512_ARCH
|
|||
default y if ARM && !CPU_V7M
|
||||
default y if ARM64
|
||||
default y if MIPS && CPU_CAVIUM_OCTEON
|
||||
default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
|
||||
|
||||
config CRYPTO_LIB_SM3
|
||||
tristate
|
||||
|
|
|
|||
|
|
@ -92,6 +92,8 @@ $(obj)/arm64/sha512-core.S: $(src)/../../arch/arm64/lib/crypto/sha2-armv8.pl
|
|||
clean-files += arm64/sha512-core.S
|
||||
libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o
|
||||
endif
|
||||
|
||||
libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o
|
||||
endif # CONFIG_CRYPTO_LIB_SHA512_ARCH
|
||||
|
||||
obj-$(CONFIG_MPILIB) += mpi/
|
||||
|
|
|
|||
|
|
@ -93,8 +93,8 @@
|
|||
sha512_4rounds \last, W3, W0, W1, W2
|
||||
.endm
|
||||
|
||||
// void sha512_transform_zvknhb_zvkb(u64 state[8], const u8 *data,
|
||||
// int num_blocks);
|
||||
// void sha512_transform_zvknhb_zvkb(struct sha512_block_state *state,
|
||||
// const u8 *data, size_t nblocks);
|
||||
SYM_FUNC_START(sha512_transform_zvknhb_zvkb)
|
||||
|
||||
// Setup mask for the vmerge to replace the first word (idx==0) in
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
||||
/*
|
||||
* SHA-512 and SHA-384 using the RISC-V vector crypto extensions
|
||||
*
|
||||
* Copyright (C) 2023 VRULL GmbH
|
||||
* Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
|
||||
*
|
||||
* Copyright (C) 2023 SiFive, Inc.
|
||||
* Author: Jerry Shih <jerry.shih@sifive.com>
|
||||
*/
|
||||
|
||||
#include <asm/simd.h>
|
||||
#include <asm/vector.h>
|
||||
#include <crypto/internal/simd.h>
|
||||
|
||||
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions);
|
||||
|
||||
asmlinkage void sha512_transform_zvknhb_zvkb(struct sha512_block_state *state,
|
||||
const u8 *data, size_t nblocks);
|
||||
|
||||
static void sha512_blocks(struct sha512_block_state *state,
|
||||
const u8 *data, size_t nblocks)
|
||||
{
|
||||
if (static_branch_likely(&have_extensions) &&
|
||||
likely(crypto_simd_usable())) {
|
||||
kernel_vector_begin();
|
||||
sha512_transform_zvknhb_zvkb(state, data, nblocks);
|
||||
kernel_vector_end();
|
||||
} else {
|
||||
sha512_blocks_generic(state, data, nblocks);
|
||||
}
|
||||
}
|
||||
|
||||
#define sha512_mod_init_arch sha512_mod_init_arch
|
||||
static inline void sha512_mod_init_arch(void)
|
||||
{
|
||||
if (riscv_isa_extension_available(NULL, ZVKNHB) &&
|
||||
riscv_isa_extension_available(NULL, ZVKB) &&
|
||||
riscv_vector_vlen() >= 128)
|
||||
static_branch_enable(&have_extensions);
|
||||
}
|
||||
Loading…
Reference in New Issue