Gh0st
/
linux
mirror of https://github.com/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

lib/crypto: Switch ARM and arm64 to 'ksimd' scoped guard API 

Before modifying the prototypes of kernel_neon_begin() and
kernel_neon_end() to accommodate kernel mode FP/SIMD state buffers
allocated on the stack, move arm64 to the new 'ksimd' scoped guard API,
which encapsulates the calls to those functions.

For symmetry, do the same for 32-bit ARM too.

Reviewed-by: Eric Biggers <ebiggers@kernel.org>
Reviewed-by: Jonathan Cameron <jonathan.cameron@huawei.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
This commit is contained in:
Ard Biesheuvel 2025-10-01 13:29:23 +02:00
parent 814f5415d3
commit f53d18a4e6
11 changed files with 43 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
lib/crypto/arm/chacha.h
View File

@ -12,7 +12,6 @@
#include <asm/cputype.h> #include <asm/cputype.h>
#include <asm/hwcap.h> #include <asm/hwcap.h>
#include <asm/neon.h>
#include <asm/simd.h> #include <asm/simd.h>
asmlinkage void chacha_block_xor_neon(const struct chacha_state *state, asmlinkage void chacha_block_xor_neon(const struct chacha_state *state,
@ -68,9 +67,8 @@ static void hchacha_block_arch(const struct chacha_state *state,
if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) { if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {
hchacha_block_arm(state, out, nrounds); hchacha_block_arm(state, out, nrounds);
} else { } else {
kernel_neon_begin(); scoped_ksimd()
hchacha_block_neon(state, out, nrounds); hchacha_block_neon(state, out, nrounds);
kernel_neon_end();
} }
} }
@ -87,9 +85,8 @@ static void chacha_crypt_arch(struct chacha_state *state, u8 *dst,
do { do {
unsigned int todo = min_t(unsigned int, bytes, SZ_4K); unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
kernel_neon_begin(); scoped_ksimd()
chacha_doneon(state, dst, src, todo, nrounds); chacha_doneon(state, dst, src, todo, nrounds);
kernel_neon_end();
bytes -= todo; bytes -= todo;
src += todo; src += todo;

5
lib/crypto/arm/curve25519.h
View File

@ -25,9 +25,8 @@ static void curve25519_arch(u8 out[CURVE25519_KEY_SIZE],
const u8 point[CURVE25519_KEY_SIZE]) const u8 point[CURVE25519_KEY_SIZE])
{ {
if (static_branch_likely(&have_neon) && crypto_simd_usable()) { if (static_branch_likely(&have_neon) && crypto_simd_usable()) {
kernel_neon_begin(); scoped_ksimd()
curve25519_neon(out, scalar, point); curve25519_neon(out, scalar, point);
kernel_neon_end();
} else { } else {
curve25519_generic(out, scalar, point); curve25519_generic(out, scalar, point);
} }

6
lib/crypto/arm/poly1305.h
View File

@ -6,7 +6,6 @@
*/ */
#include <asm/hwcap.h> #include <asm/hwcap.h>
#include <asm/neon.h>
#include <asm/simd.h> #include <asm/simd.h>
#include <linux/cpufeature.h> #include <linux/cpufeature.h>
#include <linux/jump_label.h> #include <linux/jump_label.h>
@ -32,9 +31,8 @@ static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src,
do { do {
unsigned int todo = min_t(unsigned int, len, SZ_4K); unsigned int todo = min_t(unsigned int, len, SZ_4K);
kernel_neon_begin(); scoped_ksimd()
poly1305_blocks_neon(state, src, todo, padbit); poly1305_blocks_neon(state, src, todo, padbit);
kernel_neon_end();
len -= todo; len -= todo;
src += todo; src += todo;

13
lib/crypto/arm/sha1.h
View File

@ -4,7 +4,6 @@
* *
* Copyright 2025 Google LLC * Copyright 2025 Google LLC
*/ */
#include <asm/neon.h>
#include <asm/simd.h> #include <asm/simd.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
@ -22,12 +21,12 @@ static void sha1_blocks(struct sha1_block_state *state,
{ {
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
static_branch_likely(&have_neon) && likely(may_use_simd())) { static_branch_likely(&have_neon) && likely(may_use_simd())) {
kernel_neon_begin(); scoped_ksimd() {
if (static_branch_likely(&have_ce)) if (static_branch_likely(&have_ce))
sha1_ce_transform(state, data, nblocks); sha1_ce_transform(state, data, nblocks);
else else
sha1_transform_neon(state, data, nblocks); sha1_transform_neon(state, data, nblocks);
kernel_neon_end(); }
} else { } else {
sha1_block_data_order(state, data, nblocks); sha1_block_data_order(state, data, nblocks);
} }

12
lib/crypto/arm/sha256.h
View File

@ -22,12 +22,12 @@ static void sha256_blocks(struct sha256_block_state *state,
{ {
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
static_branch_likely(&have_neon) && likely(may_use_simd())) { static_branch_likely(&have_neon) && likely(may_use_simd())) {
kernel_neon_begin(); scoped_ksimd() {
if (static_branch_likely(&have_ce)) if (static_branch_likely(&have_ce))
sha256_ce_transform(state, data, nblocks); sha256_ce_transform(state, data, nblocks);
else else
sha256_block_data_order_neon(state, data, nblocks); sha256_block_data_order_neon(state, data, nblocks);
kernel_neon_end(); }
} else { } else {
sha256_block_data_order(state, data, nblocks); sha256_block_data_order(state, data, nblocks);
} }

5
lib/crypto/arm/sha512.h
View File

@ -19,9 +19,8 @@ static void sha512_blocks(struct sha512_block_state *state,
{ {
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
static_branch_likely(&have_neon) && likely(may_use_simd())) { static_branch_likely(&have_neon) && likely(may_use_simd())) {
kernel_neon_begin(); scoped_ksimd()
sha512_block_data_order_neon(state, data, nblocks); sha512_block_data_order_neon(state, data, nblocks);
kernel_neon_end();
} else { } else {
sha512_block_data_order(state, data, nblocks); sha512_block_data_order(state, data, nblocks);
} }

11
lib/crypto/arm64/chacha.h
View File

@ -23,7 +23,6 @@
#include <linux/kernel.h> #include <linux/kernel.h>
#include <asm/hwcap.h> #include <asm/hwcap.h>
#include <asm/neon.h>
#include <asm/simd.h> #include <asm/simd.h>
asmlinkage void chacha_block_xor_neon(const struct chacha_state *state, asmlinkage void chacha_block_xor_neon(const struct chacha_state *state,
@ -65,9 +64,8 @@ static void hchacha_block_arch(const struct chacha_state *state,
if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) { if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) {
hchacha_block_generic(state, out, nrounds); hchacha_block_generic(state, out, nrounds);
} else { } else {
kernel_neon_begin(); scoped_ksimd()
hchacha_block_neon(state, out, nrounds); hchacha_block_neon(state, out, nrounds);
kernel_neon_end();
} }
} }
@ -81,9 +79,8 @@ static void chacha_crypt_arch(struct chacha_state *state, u8 *dst,
do { do {
unsigned int todo = min_t(unsigned int, bytes, SZ_4K); unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
kernel_neon_begin(); scoped_ksimd()
chacha_doneon(state, dst, src, todo, nrounds); chacha_doneon(state, dst, src, todo, nrounds);
kernel_neon_end();
bytes -= todo; bytes -= todo;
src += todo; src += todo;

6
lib/crypto/arm64/poly1305.h
View File

@ -6,7 +6,6 @@
*/ */
#include <asm/hwcap.h> #include <asm/hwcap.h>
#include <asm/neon.h>
#include <asm/simd.h> #include <asm/simd.h>
#include <linux/cpufeature.h> #include <linux/cpufeature.h>
#include <linux/jump_label.h> #include <linux/jump_label.h>
@ -31,9 +30,8 @@ static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src,
do { do {
unsigned int todo = min_t(unsigned int, len, SZ_4K); unsigned int todo = min_t(unsigned int, len, SZ_4K);
kernel_neon_begin(); scoped_ksimd()
poly1305_blocks_neon(state, src, todo, padbit); poly1305_blocks_neon(state, src, todo, padbit);
kernel_neon_end();
len -= todo; len -= todo;
src += todo; src += todo;

7
lib/crypto/arm64/sha1.h
View File

@ -4,7 +4,6 @@
* *
* Copyright 2025 Google LLC * Copyright 2025 Google LLC
*/ */
#include <asm/neon.h>
#include <asm/simd.h> #include <asm/simd.h>
#include <linux/cpufeature.h> #include <linux/cpufeature.h>
@ -20,9 +19,9 @@ static void sha1_blocks(struct sha1_block_state *state,
do { do {
size_t rem; size_t rem;
kernel_neon_begin(); scoped_ksimd()
rem = __sha1_ce_transform(state, data, nblocks); rem = __sha1_ce_transform(state, data, nblocks);
kernel_neon_end();
data += (nblocks - rem) * SHA1_BLOCK_SIZE; data += (nblocks - rem) * SHA1_BLOCK_SIZE;
nblocks = rem; nblocks = rem;
} while (nblocks); } while (nblocks);

19
lib/crypto/arm64/sha256.h
View File

@ -4,7 +4,6 @@
* *
* Copyright 2025 Google LLC * Copyright 2025 Google LLC
*/ */
#include <asm/neon.h>
#include <asm/simd.h> #include <asm/simd.h>
#include <linux/cpufeature.h> #include <linux/cpufeature.h>
@ -27,17 +26,16 @@ static void sha256_blocks(struct sha256_block_state *state,
do { do {
size_t rem; size_t rem;
kernel_neon_begin(); scoped_ksimd()
rem = __sha256_ce_transform(state, rem = __sha256_ce_transform(state, data,
data, nblocks); nblocks);
kernel_neon_end();
data += (nblocks - rem) * SHA256_BLOCK_SIZE; data += (nblocks - rem) * SHA256_BLOCK_SIZE;
nblocks = rem; nblocks = rem;
} while (nblocks); } while (nblocks);
} else { } else {
kernel_neon_begin(); scoped_ksimd()
sha256_block_neon(state, data, nblocks); sha256_block_neon(state, data, nblocks);
kernel_neon_end();
} }
} else { } else {
sha256_block_data_order(state, data, nblocks); sha256_block_data_order(state, data, nblocks);
@ -66,9 +64,8 @@ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx,
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE && static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE &&
len <= 65536 && likely(may_use_simd())) { len <= 65536 && likely(may_use_simd())) {
kernel_neon_begin(); scoped_ksimd()
sha256_ce_finup2x(ctx, data1, data2, len, out1, out2); sha256_ce_finup2x(ctx, data1, data2, len, out1, out2);
kernel_neon_end();
kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE); kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE);
kmsan_unpoison_memory(out2, SHA256_DIGEST_SIZE); kmsan_unpoison_memory(out2, SHA256_DIGEST_SIZE);
return true; return true;

8
lib/crypto/arm64/sha512.h
View File

@ -4,7 +4,7 @@
* *
* Copyright 2025 Google LLC * Copyright 2025 Google LLC
*/ */
#include <asm/neon.h>
#include <asm/simd.h> #include <asm/simd.h>
#include <linux/cpufeature.h> #include <linux/cpufeature.h>
@ -24,9 +24,9 @@ static void sha512_blocks(struct sha512_block_state *state,
do { do {
size_t rem; size_t rem;
kernel_neon_begin(); scoped_ksimd()
rem = __sha512_ce_transform(state, data, nblocks); rem = __sha512_ce_transform(state, data, nblocks);
kernel_neon_end();
data += (nblocks - rem) * SHA512_BLOCK_SIZE; data += (nblocks - rem) * SHA512_BLOCK_SIZE;
nblocks = rem; nblocks = rem;
} while (nblocks); } while (nblocks);