Proxmox-Port/linux-loongson
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson synced 2025-09-01 06:39:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

lib/crypto: sha256: Consolidate into single module

Browse Source 
Consolidate the CPU-based SHA-256 code into a single module, following
what I did with SHA-512:

- Each arch now provides a header file lib/crypto/$(SRCARCH)/sha256.h,
  replacing lib/crypto/$(SRCARCH)/sha256.c.  The header defines
  sha256_blocks() and optionally sha256_mod_init_arch().  It is included
  by lib/crypto/sha256.c, and thus the code gets built into the single
  libsha256 module, with proper inlining and dead code elimination.

- sha256_blocks_generic() is moved from lib/crypto/sha256-generic.c into
  lib/crypto/sha256.c.  It's now a static function marked with
  __maybe_unused, so the compiler automatically eliminates it in any
  cases where it's not used.

- Whether arch-optimized SHA-256 is buildable is now controlled
  centrally by lib/crypto/Kconfig instead of by
  lib/crypto/$(SRCARCH)/Kconfig.  The conditions for enabling it remain
  the same as before, and it remains enabled by default.

- Any additional arch-specific translation units for the optimized
  SHA-256 code (such as assembly files) are now compiled by
  lib/crypto/Makefile instead of lib/crypto/$(SRCARCH)/Makefile.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20250630160645.3198-13-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
This commit is contained in:
Eric Biggers 2025-06-30 09:06:43 -07:00
parent 9f9846a72e
commit e96cb9507f
29 changed files with 231 additions and 465 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
arch/mips/cavium-octeon/Kconfig
View File

@ -23,12 +23,6 @@ config CAVIUM_OCTEON_CVMSEG_SIZE
legally range is from zero to 54 cache blocks (i.e. CVMSEG LM is legally range is from zero to 54 cache blocks (i.e. CVMSEG LM is
between zero and 6192 bytes). between zero and 6192 bytes).
config CRYPTO_SHA256_OCTEON
tristate
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256
select CRYPTO_LIB_SHA256_GENERIC
endif # CPU_CAVIUM_OCTEON endif # CPU_CAVIUM_OCTEON
if CAVIUM_OCTEON_SOC if CAVIUM_OCTEON_SOC

1
arch/mips/cavium-octeon/crypto/Makefile
View File

@ -7,4 +7,3 @@ obj-y += octeon-crypto.o
obj-$(CONFIG_CRYPTO_MD5_OCTEON) += octeon-md5.o obj-$(CONFIG_CRYPTO_MD5_OCTEON) += octeon-md5.o
obj-$(CONFIG_CRYPTO_SHA1_OCTEON) += octeon-sha1.o obj-$(CONFIG_CRYPTO_SHA1_OCTEON) += octeon-sha1.o
obj-$(CONFIG_CRYPTO_SHA256_OCTEON) += octeon-sha256.o

52
include/crypto/internal/sha2.h
View File

@ -1,52 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef _CRYPTO_INTERNAL_SHA2_H
#define _CRYPTO_INTERNAL_SHA2_H
#include <crypto/sha2.h>
#include <linux/compiler_attributes.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/unaligned.h>
void sha256_blocks_generic(struct sha256_block_state *state,
const u8 *data, size_t nblocks);
void sha256_blocks_arch(struct sha256_block_state *state,
const u8 *data, size_t nblocks);
static __always_inline void sha256_choose_blocks(
u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks,
bool force_generic, bool force_simd)
{
if (!IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) || force_generic)
sha256_blocks_generic((struct sha256_block_state *)state, data, nblocks);
else
sha256_blocks_arch((struct sha256_block_state *)state, data, nblocks);
}
static __always_inline void sha256_finup(
struct crypto_sha256_state *sctx, u8 buf[SHA256_BLOCK_SIZE],
size_t len, u8 out[SHA256_DIGEST_SIZE], size_t digest_size,
bool force_generic, bool force_simd)
{
const size_t bit_offset = SHA256_BLOCK_SIZE - 8;
__be64 *bits = (__be64 *)&buf[bit_offset];
int i;
buf[len++] = 0x80;
if (len > bit_offset) {
memset(&buf[len], 0, SHA256_BLOCK_SIZE - len);
sha256_choose_blocks(sctx->state, buf, 1, force_generic,
force_simd);
len = 0;
}
memset(&buf[len], 0, bit_offset - len);
*bits = cpu_to_be64(sctx->count << 3);
sha256_choose_blocks(sctx->state, buf, 1, force_generic, force_simd);
for (i = 0; i < digest_size; i += 4)
put_unaligned_be32(sctx->state[i / 4], out + i);
}
#endif /* _CRYPTO_INTERNAL_SHA2_H */

26
lib/crypto/Kconfig
View File

@ -144,20 +144,17 @@ config CRYPTO_LIB_SHA256
by either the generic implementation or an arch-specific one, if one by either the generic implementation or an arch-specific one, if one
is available and enabled. is available and enabled.
config CRYPTO_ARCH_HAVE_LIB_SHA256 config CRYPTO_LIB_SHA256_ARCH
bool bool
help depends on CRYPTO_LIB_SHA256 && !UML
Declares whether the architecture provides an arch-specific default y if ARM && !CPU_V7M
accelerated implementation of the SHA-256 library interface. default y if ARM64
default y if MIPS && CPU_CAVIUM_OCTEON
config CRYPTO_LIB_SHA256_GENERIC default y if PPC && SPE
tristate default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
default CRYPTO_LIB_SHA256 if !CRYPTO_ARCH_HAVE_LIB_SHA256 default y if S390
help default y if SPARC64
This symbol can be selected by arch implementations of the SHA-256 default y if X86_64
library interface that require the generic code as a fallback, e.g.,
for SIMD implementations. If no arch specific implementation is
enabled, this implementation serves the users of CRYPTO_LIB_SHA256.
config CRYPTO_LIB_SHA512 config CRYPTO_LIB_SHA512
tristate tristate
@ -199,9 +196,6 @@ endif
if S390 if S390
source "lib/crypto/s390/Kconfig" source "lib/crypto/s390/Kconfig"
endif endif
if SPARC
source "lib/crypto/sparc/Kconfig"
endif
if X86 if X86
source "lib/crypto/x86/Kconfig" source "lib/crypto/x86/Kconfig"
endif endif

39
lib/crypto/Makefile
View File

@ -66,11 +66,39 @@ libpoly1305-generic-y += poly1305-generic.o
obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o
libsha1-y := sha1.o libsha1-y := sha1.o
obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o ################################################################################
libsha256-y := sha256.o
obj-$(CONFIG_CRYPTO_LIB_SHA256_GENERIC) += libsha256-generic.o obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o
libsha256-generic-y := sha256-generic.o libsha256-y := sha256.o
ifeq ($(CONFIG_CRYPTO_LIB_SHA256_ARCH),y)
CFLAGS_sha256.o += -I$(src)/$(SRCARCH)
ifeq ($(CONFIG_ARM),y)
libsha256-y += arm/sha256-ce.o arm/sha256-core.o
$(obj)/arm/sha256-core.S: $(src)/arm/sha256-armv4.pl
$(call cmd,perlasm)
clean-files += arm/sha256-core.S
AFLAGS_arm/sha256-core.o += $(aflags-thumb2-y)
endif
ifeq ($(CONFIG_ARM64),y)
libsha256-y += arm64/sha256-core.o
$(obj)/arm64/sha256-core.S: $(src)/arm64/sha2-armv8.pl
$(call cmd,perlasm_with_args)
clean-files += arm64/sha256-core.S
libsha256-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha256-ce.o
endif
libsha256-$(CONFIG_PPC) += powerpc/sha256-spe-asm.o
libsha256-$(CONFIG_RISCV) += riscv/sha256-riscv64-zvknha_or_zvknhb-zvkb.o
libsha256-$(CONFIG_SPARC) += sparc/sha256_asm.o
libsha256-$(CONFIG_X86) += x86/sha256-ssse3-asm.o \
x86/sha256-avx-asm.o \
x86/sha256-avx2-asm.o \
x86/sha256-ni-asm.o
endif # CONFIG_CRYPTO_LIB_SHA256_ARCH
################################################################################
obj-$(CONFIG_CRYPTO_LIB_SHA512) += libsha512.o obj-$(CONFIG_CRYPTO_LIB_SHA512) += libsha512.o
libsha512-y := sha512.o libsha512-y := sha512.o
@ -100,6 +128,8 @@ libsha512-$(CONFIG_X86) += x86/sha512-ssse3-asm.o \
x86/sha512-avx2-asm.o x86/sha512-avx2-asm.o
endif # CONFIG_CRYPTO_LIB_SHA512_ARCH endif # CONFIG_CRYPTO_LIB_SHA512_ARCH
################################################################################
obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_MPILIB) += mpi/
obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o
@ -113,5 +143,4 @@ obj-$(CONFIG_MIPS) += mips/
obj-$(CONFIG_PPC) += powerpc/ obj-$(CONFIG_PPC) += powerpc/
obj-$(CONFIG_RISCV) += riscv/ obj-$(CONFIG_RISCV) += riscv/
obj-$(CONFIG_S390) += s390/ obj-$(CONFIG_S390) += s390/
obj-$(CONFIG_SPARC) += sparc/
obj-$(CONFIG_X86) += x86/ obj-$(CONFIG_X86) += x86/

6
lib/crypto/arm/Kconfig
View File

@ -22,9 +22,3 @@ config CRYPTO_POLY1305_ARM
tristate tristate
default CRYPTO_LIB_POLY1305 default CRYPTO_LIB_POLY1305
select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305
config CRYPTO_SHA256_ARM
tristate
depends on !CPU_V7M
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256

8
lib/crypto/arm/Makefile
View File

@ -10,17 +10,13 @@ chacha-neon-$(CONFIG_KERNEL_MODE_NEON) += chacha-neon-core.o
obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o
poly1305-arm-y := poly1305-core.o poly1305-glue.o poly1305-arm-y := poly1305-core.o poly1305-glue.o
obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o
sha256-arm-y := sha256.o sha256-core.o
sha256-arm-$(CONFIG_KERNEL_MODE_NEON) += sha256-ce.o
quiet_cmd_perl = PERL $@ quiet_cmd_perl = PERL $@
cmd_perl = $(PERL) $(<) > $(@) cmd_perl = $(PERL) $(<) > $(@)
$(obj)/%-core.S: $(src)/%-armv4.pl $(obj)/%-core.S: $(src)/%-armv4.pl
$(call cmd,perl) $(call cmd,perl)
clean-files += poly1305-core.S sha256-core.S clean-files += poly1305-core.S
aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1 aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1
@ -28,5 +24,3 @@ aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1
poly1305-aflags-$(CONFIG_CPU_V7) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=5 poly1305-aflags-$(CONFIG_CPU_V7) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=5
poly1305-aflags-$(CONFIG_KERNEL_MODE_NEON) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=7 poly1305-aflags-$(CONFIG_KERNEL_MODE_NEON) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=7
AFLAGS_poly1305-core.o += $(poly1305-aflags-y) $(aflags-thumb2-y) AFLAGS_poly1305-core.o += $(poly1305-aflags-y) $(aflags-thumb2-y)
AFLAGS_sha256-core.o += $(aflags-thumb2-y)

27
lib/crypto/arm/sha256.c → lib/crypto/arm/sha256.h
View File

@ -1,14 +1,11 @@
// SPDX-License-Identifier: GPL-2.0-or-later /* SPDX-License-Identifier: GPL-2.0-or-later */
/* /*
* SHA-256 optimized for ARM * SHA-256 optimized for ARM
* *
* Copyright 2025 Google LLC * Copyright 2025 Google LLC
*/ */
#include <asm/neon.h> #include <asm/neon.h>
#include <crypto/internal/sha2.h>
#include <crypto/internal/simd.h> #include <crypto/internal/simd.h>
#include <linux/kernel.h>
#include <linux/module.h>
asmlinkage void sha256_block_data_order(struct sha256_block_state *state, asmlinkage void sha256_block_data_order(struct sha256_block_state *state,
const u8 *data, size_t nblocks); const u8 *data, size_t nblocks);
@ -20,8 +17,8 @@ asmlinkage void sha256_ce_transform(struct sha256_block_state *state,
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
void sha256_blocks_arch(struct sha256_block_state *state, static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks) const u8 *data, size_t nblocks)
{ {
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
static_branch_likely(&have_neon) && crypto_simd_usable()) { static_branch_likely(&have_neon) && crypto_simd_usable()) {
@ -35,23 +32,15 @@ void sha256_blocks_arch(struct sha256_block_state *state,
sha256_block_data_order(state, data, nblocks); sha256_block_data_order(state, data, nblocks);
} }
} }
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
static int __init sha256_arm_mod_init(void) #ifdef CONFIG_KERNEL_MODE_NEON
#define sha256_mod_init_arch sha256_mod_init_arch
static inline void sha256_mod_init_arch(void)
{ {
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) { if (elf_hwcap & HWCAP_NEON) {
static_branch_enable(&have_neon); static_branch_enable(&have_neon);
if (elf_hwcap2 & HWCAP2_SHA2) if (elf_hwcap2 & HWCAP2_SHA2)
static_branch_enable(&have_ce); static_branch_enable(&have_ce);
} }
return 0;
} }
subsys_initcall(sha256_arm_mod_init); #endif /* CONFIG_KERNEL_MODE_NEON */
static void __exit sha256_arm_mod_exit(void)
{
}
module_exit(sha256_arm_mod_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA-256 optimized for ARM");

5
lib/crypto/arm64/Kconfig
View File

@ -12,8 +12,3 @@ config CRYPTO_POLY1305_NEON
depends on KERNEL_MODE_NEON depends on KERNEL_MODE_NEON
default CRYPTO_LIB_POLY1305 default CRYPTO_LIB_POLY1305
select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305
config CRYPTO_SHA256_ARM64
tristate
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256

9
lib/crypto/arm64/Makefile
View File

@ -8,17 +8,10 @@ poly1305-neon-y := poly1305-core.o poly1305-glue.o
AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_block_init_arch AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_block_init_arch
AFLAGS_poly1305-core.o += -Dpoly1305_emit=poly1305_emit_arch AFLAGS_poly1305-core.o += -Dpoly1305_emit=poly1305_emit_arch
obj-$(CONFIG_CRYPTO_SHA256_ARM64) += sha256-arm64.o
sha256-arm64-y := sha256.o sha256-core.o
sha256-arm64-$(CONFIG_KERNEL_MODE_NEON) += sha256-ce.o
quiet_cmd_perlasm = PERLASM $@ quiet_cmd_perlasm = PERLASM $@
cmd_perlasm = $(PERL) $(<) void $(@) cmd_perlasm = $(PERL) $(<) void $(@)
$(obj)/%-core.S: $(src)/%-armv8.pl $(obj)/%-core.S: $(src)/%-armv8.pl
$(call cmd,perlasm) $(call cmd,perlasm)
$(obj)/sha256-core.S: $(src)/sha2-armv8.pl clean-files += poly1305-core.S
$(call cmd,perlasm)
clean-files += poly1305-core.S sha256-core.S

29
lib/crypto/arm64/sha256.c → lib/crypto/arm64/sha256.h
View File

@ -1,14 +1,12 @@
// SPDX-License-Identifier: GPL-2.0-or-later /* SPDX-License-Identifier: GPL-2.0-or-later */
/* /*
* SHA-256 optimized for ARM64 * SHA-256 optimized for ARM64
* *
* Copyright 2025 Google LLC * Copyright 2025 Google LLC
*/ */
#include <asm/neon.h> #include <asm/neon.h>
#include <crypto/internal/sha2.h>
#include <crypto/internal/simd.h> #include <crypto/internal/simd.h>
#include <linux/kernel.h> #include <linux/cpufeature.h>
#include <linux/module.h>
asmlinkage void sha256_block_data_order(struct sha256_block_state *state, asmlinkage void sha256_block_data_order(struct sha256_block_state *state,
const u8 *data, size_t nblocks); const u8 *data, size_t nblocks);
@ -20,8 +18,8 @@ asmlinkage size_t __sha256_ce_transform(struct sha256_block_state *state,
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
void sha256_blocks_arch(struct sha256_block_state *state, static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks) const u8 *data, size_t nblocks)
{ {
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
static_branch_likely(&have_neon) && crypto_simd_usable()) { static_branch_likely(&have_neon) && crypto_simd_usable()) {
@ -45,24 +43,15 @@ void sha256_blocks_arch(struct sha256_block_state *state,
sha256_block_data_order(state, data, nblocks); sha256_block_data_order(state, data, nblocks);
} }
} }
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
static int __init sha256_arm64_mod_init(void) #ifdef CONFIG_KERNEL_MODE_NEON
#define sha256_mod_init_arch sha256_mod_init_arch
static inline void sha256_mod_init_arch(void)
{ {
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && if (cpu_have_named_feature(ASIMD)) {
cpu_have_named_feature(ASIMD)) {
static_branch_enable(&have_neon); static_branch_enable(&have_neon);
if (cpu_have_named_feature(SHA2)) if (cpu_have_named_feature(SHA2))
static_branch_enable(&have_ce); static_branch_enable(&have_ce);
} }
return 0;
} }
subsys_initcall(sha256_arm64_mod_init); #endif /* CONFIG_KERNEL_MODE_NEON */
static void __exit sha256_arm64_mod_exit(void)
{
}
module_exit(sha256_arm64_mod_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA-256 optimized for ARM64");

14
arch/mips/cavium-octeon/crypto/octeon-sha256.c → lib/crypto/mips/sha256.h
View File

@ -1,4 +1,4 @@
// SPDX-License-Identifier: GPL-2.0-or-later /* SPDX-License-Identifier: GPL-2.0-or-later */
/* /*
* SHA-256 Secure Hash Algorithm. * SHA-256 Secure Hash Algorithm.
* *
@ -14,16 +14,13 @@
#include <asm/octeon/crypto.h> #include <asm/octeon/crypto.h>
#include <asm/octeon/octeon.h> #include <asm/octeon/octeon.h>
#include <crypto/internal/sha2.h>
#include <linux/kernel.h>
#include <linux/module.h>
/* /*
* We pass everything as 64-bit. OCTEON can handle misaligned data. * We pass everything as 64-bit. OCTEON can handle misaligned data.
*/ */
void sha256_blocks_arch(struct sha256_block_state *state, static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks) const u8 *data, size_t nblocks)
{ {
struct octeon_cop2_state cop2_state; struct octeon_cop2_state cop2_state;
u64 *state64 = (u64 *)state; u64 *state64 = (u64 *)state;
@ -59,8 +56,3 @@ void sha256_blocks_arch(struct sha256_block_state *state,
state64[3] = read_octeon_64bit_hash_dword(3); state64[3] = read_octeon_64bit_hash_dword(3);
octeon_crypto_disable(&cop2_state, flags); octeon_crypto_disable(&cop2_state, flags);
} }
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm (OCTEON)");
MODULE_AUTHOR("Aaro Koskinen <aaro.koskinen@iki.fi>");

6
lib/crypto/powerpc/Kconfig
View File

@ -14,9 +14,3 @@ config CRYPTO_POLY1305_P10
default CRYPTO_LIB_POLY1305 default CRYPTO_LIB_POLY1305
select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305
select CRYPTO_LIB_POLY1305_GENERIC select CRYPTO_LIB_POLY1305_GENERIC
config CRYPTO_SHA256_PPC_SPE
tristate
depends on SPE
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256

3
lib/crypto/powerpc/Makefile
View File

@ -5,6 +5,3 @@ chacha-p10-crypto-y := chacha-p10-glue.o chacha-p10le-8x.o
obj-$(CONFIG_CRYPTO_POLY1305_P10) += poly1305-p10-crypto.o obj-$(CONFIG_CRYPTO_POLY1305_P10) += poly1305-p10-crypto.o
poly1305-p10-crypto-y := poly1305-p10-glue.o poly1305-p10le_64.o poly1305-p10-crypto-y := poly1305-p10-glue.o poly1305-p10le_64.o
obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o
sha256-ppc-spe-y := sha256.o sha256-spe-asm.o

13
lib/crypto/powerpc/sha256.c → lib/crypto/powerpc/sha256.h
View File

@ -1,4 +1,4 @@
// SPDX-License-Identifier: GPL-2.0-or-later /* SPDX-License-Identifier: GPL-2.0-or-later */
/* /*
* SHA-256 Secure Hash Algorithm, SPE optimized * SHA-256 Secure Hash Algorithm, SPE optimized
* *
@ -9,9 +9,6 @@
*/ */
#include <asm/switch_to.h> #include <asm/switch_to.h>
#include <crypto/internal/sha2.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/preempt.h> #include <linux/preempt.h>
/* /*
@ -43,8 +40,8 @@ static void spe_end(void)
preempt_enable(); preempt_enable();
} }
void sha256_blocks_arch(struct sha256_block_state *state, static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks) const u8 *data, size_t nblocks)
{ {
do { do {
/* cut input data into smaller blocks */ /* cut input data into smaller blocks */
@ -59,7 +56,3 @@ void sha256_blocks_arch(struct sha256_block_state *state,
nblocks -= unit; nblocks -= unit;
} while (nblocks); } while (nblocks);
} }
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm, SPE optimized");

7
lib/crypto/riscv/Kconfig
View File

@ -6,10 +6,3 @@ config CRYPTO_CHACHA_RISCV64
default CRYPTO_LIB_CHACHA default CRYPTO_LIB_CHACHA
select CRYPTO_ARCH_HAVE_LIB_CHACHA select CRYPTO_ARCH_HAVE_LIB_CHACHA
select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_LIB_CHACHA_GENERIC
config CRYPTO_SHA256_RISCV64
tristate
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256
select CRYPTO_LIB_SHA256_GENERIC

3
lib/crypto/riscv/Makefile
View File

@ -2,6 +2,3 @@
obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o
chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o
obj-$(CONFIG_CRYPTO_SHA256_RISCV64) += sha256-riscv64.o
sha256-riscv64-y := sha256.o sha256-riscv64-zvknha_or_zvknhb-zvkb.o

24
lib/crypto/riscv/sha256.c → lib/crypto/riscv/sha256.h
View File

@ -1,4 +1,4 @@
// SPDX-License-Identifier: GPL-2.0-or-later /* SPDX-License-Identifier: GPL-2.0-or-later */
/* /*
* SHA-256 (RISC-V accelerated) * SHA-256 (RISC-V accelerated)
* *
@ -10,10 +10,7 @@
*/ */
#include <asm/vector.h> #include <asm/vector.h>
#include <crypto/internal/sha2.h>
#include <crypto/internal/simd.h> #include <crypto/internal/simd.h>
#include <linux/kernel.h>
#include <linux/module.h>
asmlinkage void asmlinkage void
sha256_transform_zvknha_or_zvknhb_zvkb(struct sha256_block_state *state, sha256_transform_zvknha_or_zvknhb_zvkb(struct sha256_block_state *state,
@ -21,8 +18,8 @@ sha256_transform_zvknha_or_zvknhb_zvkb(struct sha256_block_state *state,
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions);
void sha256_blocks_arch(struct sha256_block_state *state, static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks) const u8 *data, size_t nblocks)
{ {
if (static_branch_likely(&have_extensions) && crypto_simd_usable()) { if (static_branch_likely(&have_extensions) && crypto_simd_usable()) {
kernel_vector_begin(); kernel_vector_begin();
@ -32,9 +29,9 @@ void sha256_blocks_arch(struct sha256_block_state *state,
sha256_blocks_generic(state, data, nblocks); sha256_blocks_generic(state, data, nblocks);
} }
} }
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
static int __init riscv64_sha256_mod_init(void) #define sha256_mod_init_arch sha256_mod_init_arch
static inline void sha256_mod_init_arch(void)
{ {
/* Both zvknha and zvknhb provide the SHA-256 instructions. */ /* Both zvknha and zvknhb provide the SHA-256 instructions. */
if ((riscv_isa_extension_available(NULL, ZVKNHA) || if ((riscv_isa_extension_available(NULL, ZVKNHA) ||
@ -42,15 +39,4 @@ static int __init riscv64_sha256_mod_init(void)
riscv_isa_extension_available(NULL, ZVKB) && riscv_isa_extension_available(NULL, ZVKB) &&
riscv_vector_vlen() >= 128) riscv_vector_vlen() >= 128)
static_branch_enable(&have_extensions); static_branch_enable(&have_extensions);
return 0;
} }
subsys_initcall(riscv64_sha256_mod_init);
static void __exit riscv64_sha256_mod_exit(void)
{
}
module_exit(riscv64_sha256_mod_exit);
MODULE_DESCRIPTION("SHA-256 (RISC-V accelerated)");
MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
MODULE_LICENSE("GPL");

6
lib/crypto/s390/Kconfig
View File

@ -5,9 +5,3 @@ config CRYPTO_CHACHA_S390
default CRYPTO_LIB_CHACHA default CRYPTO_LIB_CHACHA
select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_LIB_CHACHA_GENERIC
select CRYPTO_ARCH_HAVE_LIB_CHACHA select CRYPTO_ARCH_HAVE_LIB_CHACHA
config CRYPTO_SHA256_S390
tristate
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256
select CRYPTO_LIB_SHA256_GENERIC

3
lib/crypto/s390/Makefile
View File

@ -2,6 +2,3 @@
obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o
chacha_s390-y := chacha-glue.o chacha-s390.o chacha_s390-y := chacha-glue.o chacha-s390.o
obj-$(CONFIG_CRYPTO_SHA256_S390) += sha256-s390.o
sha256-s390-y := sha256.o

23
lib/crypto/s390/sha256.c → lib/crypto/s390/sha256.h
View File

@ -1,19 +1,16 @@
// SPDX-License-Identifier: GPL-2.0-or-later /* SPDX-License-Identifier: GPL-2.0-or-later */
/* /*
* SHA-256 optimized using the CP Assist for Cryptographic Functions (CPACF) * SHA-256 optimized using the CP Assist for Cryptographic Functions (CPACF)
* *
* Copyright 2025 Google LLC * Copyright 2025 Google LLC
*/ */
#include <asm/cpacf.h> #include <asm/cpacf.h>
#include <crypto/internal/sha2.h>
#include <linux/cpufeature.h> #include <linux/cpufeature.h>
#include <linux/kernel.h>
#include <linux/module.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_sha256); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_sha256);
void sha256_blocks_arch(struct sha256_block_state *state, static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks) const u8 *data, size_t nblocks)
{ {
if (static_branch_likely(&have_cpacf_sha256)) if (static_branch_likely(&have_cpacf_sha256))
cpacf_kimd(CPACF_KIMD_SHA_256, state, data, cpacf_kimd(CPACF_KIMD_SHA_256, state, data,
@ -21,21 +18,11 @@ void sha256_blocks_arch(struct sha256_block_state *state,
else else
sha256_blocks_generic(state, data, nblocks); sha256_blocks_generic(state, data, nblocks);
} }
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
static int __init sha256_s390_mod_init(void) #define sha256_mod_init_arch sha256_mod_init_arch
static inline void sha256_mod_init_arch(void)
{ {
if (cpu_have_feature(S390_CPU_FEATURE_MSA) && if (cpu_have_feature(S390_CPU_FEATURE_MSA) &&
cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_256)) cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_256))
static_branch_enable(&have_cpacf_sha256); static_branch_enable(&have_cpacf_sha256);
return 0;
} }
subsys_initcall(sha256_s390_mod_init);
static void __exit sha256_s390_mod_exit(void)
{
}
module_exit(sha256_s390_mod_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA-256 using the CP Assist for Cryptographic Functions (CPACF)");

150
lib/crypto/sha256-generic.c
View File

@ -1,150 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* SHA-256, as specified in
* http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf
*
* SHA-256 code by Jean-Luc Cooke <jlcooke@certainkey.com>.
*
* Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com>
* Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
* Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
* Copyright (c) 2014 Red Hat Inc.
*/
#include <crypto/internal/sha2.h>
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/unaligned.h>
static const u32 SHA256_K[] = {
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
};
static inline u32 Ch(u32 x, u32 y, u32 z)
{
return z ^ (x & (y ^ z));
}
static inline u32 Maj(u32 x, u32 y, u32 z)
{
return (x & y) | (z & (x | y));
}
#define e0(x) (ror32(x, 2) ^ ror32(x, 13) ^ ror32(x, 22))
#define e1(x) (ror32(x, 6) ^ ror32(x, 11) ^ ror32(x, 25))
#define s0(x) (ror32(x, 7) ^ ror32(x, 18) ^ (x >> 3))
#define s1(x) (ror32(x, 17) ^ ror32(x, 19) ^ (x >> 10))
static inline void LOAD_OP(int I, u32 *W, const u8 *input)
{
W[I] = get_unaligned_be32((__u32 *)input + I);
}
static inline void BLEND_OP(int I, u32 *W)
{
W[I] = s1(W[I-2]) + W[I-7] + s0(W[I-15]) + W[I-16];
}
#define SHA256_ROUND(i, a, b, c, d, e, f, g, h) do { \
u32 t1, t2; \
t1 = h + e1(e) + Ch(e, f, g) + SHA256_K[i] + W[i]; \
t2 = e0(a) + Maj(a, b, c); \
d += t1; \
h = t1 + t2; \
} while (0)
static void sha256_block_generic(struct sha256_block_state *state,
const u8 *input, u32 W[64])
{
u32 a, b, c, d, e, f, g, h;
int i;
/* load the input */
for (i = 0; i < 16; i += 8) {
LOAD_OP(i + 0, W, input);
LOAD_OP(i + 1, W, input);
LOAD_OP(i + 2, W, input);
LOAD_OP(i + 3, W, input);
LOAD_OP(i + 4, W, input);
LOAD_OP(i + 5, W, input);
LOAD_OP(i + 6, W, input);
LOAD_OP(i + 7, W, input);
}
/* now blend */
for (i = 16; i < 64; i += 8) {
BLEND_OP(i + 0, W);
BLEND_OP(i + 1, W);
BLEND_OP(i + 2, W);
BLEND_OP(i + 3, W);
BLEND_OP(i + 4, W);
BLEND_OP(i + 5, W);
BLEND_OP(i + 6, W);
BLEND_OP(i + 7, W);
}
/* load the state into our registers */
a = state->h[0];
b = state->h[1];
c = state->h[2];
d = state->h[3];
e = state->h[4];
f = state->h[5];
g = state->h[6];
h = state->h[7];
/* now iterate */
for (i = 0; i < 64; i += 8) {
SHA256_ROUND(i + 0, a, b, c, d, e, f, g, h);
SHA256_ROUND(i + 1, h, a, b, c, d, e, f, g);
SHA256_ROUND(i + 2, g, h, a, b, c, d, e, f);
SHA256_ROUND(i + 3, f, g, h, a, b, c, d, e);
SHA256_ROUND(i + 4, e, f, g, h, a, b, c, d);
SHA256_ROUND(i + 5, d, e, f, g, h, a, b, c);
SHA256_ROUND(i + 6, c, d, e, f, g, h, a, b);
SHA256_ROUND(i + 7, b, c, d, e, f, g, h, a);
}
state->h[0] += a;
state->h[1] += b;
state->h[2] += c;
state->h[3] += d;
state->h[4] += e;
state->h[5] += f;
state->h[6] += g;
state->h[7] += h;
}
void sha256_blocks_generic(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
{
u32 W[64];
do {
sha256_block_generic(state, data, W);
data += SHA256_BLOCK_SIZE;
} while (--nblocks);
memzero_explicit(W, sizeof(W));
}
EXPORT_SYMBOL_GPL(sha256_blocks_generic);
MODULE_DESCRIPTION("SHA-256 Algorithm (generic implementation)");
MODULE_LICENSE("GPL");

156
lib/crypto/sha256.c
View File

@ -6,15 +6,17 @@
* Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
* Copyright (c) 2002 James Morris <jmorris@intercode.com.au> * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
* Copyright (c) 2014 Red Hat Inc. * Copyright (c) 2014 Red Hat Inc.
* Copyright 2025 Google LLC
*/ */
#include <crypto/hmac.h> #include <crypto/hmac.h>
#include <crypto/internal/blockhash.h> #include <crypto/internal/blockhash.h>
#include <crypto/internal/sha2.h> #include <crypto/sha2.h>
#include <linux/export.h> #include <linux/export.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/string.h> #include <linux/string.h>
#include <linux/unaligned.h>
#include <linux/wordpart.h> #include <linux/wordpart.h>
static const struct sha256_block_state sha224_iv = { static const struct sha256_block_state sha224_iv = {
@ -31,26 +33,128 @@ static const struct sha256_block_state sha256_iv = {
}, },
}; };
/* static const u32 sha256_K[64] = {
* If __DISABLE_EXPORTS is defined, then this file is being compiled for a 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1,
* pre-boot environment. In that case, ignore the kconfig options, pull the 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
* generic code into the same translation unit, and use that only. 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
*/ 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
#ifdef __DISABLE_EXPORTS 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147,
#include "sha256-generic.c" 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
};
#define Ch(x, y, z) ((z) ^ ((x) & ((y) ^ (z))))
#define Maj(x, y, z) (((x) & (y)) | ((z) & ((x) | (y))))
#define e0(x) (ror32((x), 2) ^ ror32((x), 13) ^ ror32((x), 22))
#define e1(x) (ror32((x), 6) ^ ror32((x), 11) ^ ror32((x), 25))
#define s0(x) (ror32((x), 7) ^ ror32((x), 18) ^ ((x) >> 3))
#define s1(x) (ror32((x), 17) ^ ror32((x), 19) ^ ((x) >> 10))
static inline void LOAD_OP(int I, u32 *W, const u8 *input)
{
W[I] = get_unaligned_be32((__u32 *)input + I);
}
static inline void BLEND_OP(int I, u32 *W)
{
W[I] = s1(W[I - 2]) + W[I - 7] + s0(W[I - 15]) + W[I - 16];
}
#define SHA256_ROUND(i, a, b, c, d, e, f, g, h) \
do { \
u32 t1, t2; \
t1 = h + e1(e) + Ch(e, f, g) + sha256_K[i] + W[i]; \
t2 = e0(a) + Maj(a, b, c); \
d += t1; \
h = t1 + t2; \
} while (0)
static void sha256_block_generic(struct sha256_block_state *state,
const u8 *input, u32 W[64])
{
u32 a, b, c, d, e, f, g, h;
int i;
/* load the input */
for (i = 0; i < 16; i += 8) {
LOAD_OP(i + 0, W, input);
LOAD_OP(i + 1, W, input);
LOAD_OP(i + 2, W, input);
LOAD_OP(i + 3, W, input);
LOAD_OP(i + 4, W, input);
LOAD_OP(i + 5, W, input);
LOAD_OP(i + 6, W, input);
LOAD_OP(i + 7, W, input);
}
/* now blend */
for (i = 16; i < 64; i += 8) {
BLEND_OP(i + 0, W);
BLEND_OP(i + 1, W);
BLEND_OP(i + 2, W);
BLEND_OP(i + 3, W);
BLEND_OP(i + 4, W);
BLEND_OP(i + 5, W);
BLEND_OP(i + 6, W);
BLEND_OP(i + 7, W);
}
/* load the state into our registers */
a = state->h[0];
b = state->h[1];
c = state->h[2];
d = state->h[3];
e = state->h[4];
f = state->h[5];
g = state->h[6];
h = state->h[7];
/* now iterate */
for (i = 0; i < 64; i += 8) {
SHA256_ROUND(i + 0, a, b, c, d, e, f, g, h);
SHA256_ROUND(i + 1, h, a, b, c, d, e, f, g);
SHA256_ROUND(i + 2, g, h, a, b, c, d, e, f);
SHA256_ROUND(i + 3, f, g, h, a, b, c, d, e);
SHA256_ROUND(i + 4, e, f, g, h, a, b, c, d);
SHA256_ROUND(i + 5, d, e, f, g, h, a, b, c);
SHA256_ROUND(i + 6, c, d, e, f, g, h, a, b);
SHA256_ROUND(i + 7, b, c, d, e, f, g, h, a);
}
state->h[0] += a;
state->h[1] += b;
state->h[2] += c;
state->h[3] += d;
state->h[4] += e;
state->h[5] += f;
state->h[6] += g;
state->h[7] += h;
}
static void __maybe_unused
sha256_blocks_generic(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
{
u32 W[64];
do {
sha256_block_generic(state, data, W);
data += SHA256_BLOCK_SIZE;
} while (--nblocks);
memzero_explicit(W, sizeof(W));
}
#if defined(CONFIG_CRYPTO_LIB_SHA256_ARCH) && !defined(__DISABLE_EXPORTS)
#include "sha256.h" /* $(SRCARCH)/sha256.h */
#else
#define sha256_blocks sha256_blocks_generic
#endif #endif
static inline bool sha256_purgatory(void)
{
return __is_defined(__DISABLE_EXPORTS);
}
static inline void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
{
sha256_choose_blocks(state->h, data, nblocks, sha256_purgatory(), false);
}
static void __sha256_init(struct __sha256_ctx *ctx, static void __sha256_init(struct __sha256_ctx *ctx,
const struct sha256_block_state *iv, const struct sha256_block_state *iv,
u64 initial_bytecount) u64 initial_bytecount)
@ -273,5 +377,19 @@ void hmac_sha256_usingrawkey(const u8 *raw_key, size_t raw_key_len,
EXPORT_SYMBOL_GPL(hmac_sha256_usingrawkey); EXPORT_SYMBOL_GPL(hmac_sha256_usingrawkey);
#endif /* !__DISABLE_EXPORTS */ #endif /* !__DISABLE_EXPORTS */
#ifdef sha256_mod_init_arch
static int __init sha256_mod_init(void)
{
sha256_mod_init_arch();
return 0;
}
subsys_initcall(sha256_mod_init);
static void __exit sha256_mod_exit(void)
{
}
module_exit(sha256_mod_exit);
#endif
MODULE_DESCRIPTION("SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256 library functions"); MODULE_DESCRIPTION("SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256 library functions");
MODULE_LICENSE("GPL"); MODULE_LICENSE("GPL");

8
lib/crypto/sparc/Kconfig
View File

@ -1,8 +0,0 @@
# SPDX-License-Identifier: GPL-2.0-only
config CRYPTO_SHA256_SPARC64
tristate
depends on SPARC64
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256
select CRYPTO_LIB_SHA256_GENERIC

4
lib/crypto/sparc/Makefile
View File

@ -1,4 +0,0 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_CRYPTO_SHA256_SPARC64) += sha256-sparc64.o
sha256-sparc64-y := sha256.o sha256_asm.o

29
lib/crypto/sparc/sha256.c → lib/crypto/sparc/sha256.h
View File

@ -1,4 +1,4 @@
// SPDX-License-Identifier: GPL-2.0-only /* SPDX-License-Identifier: GPL-2.0-only */
/* /*
* SHA-256 accelerated using the sparc64 sha256 opcodes * SHA-256 accelerated using the sparc64 sha256 opcodes
* *
@ -8,51 +8,36 @@
* SHA224 Support Copyright 2007 Intel Corporation <jonathan.lynch@intel.com> * SHA224 Support Copyright 2007 Intel Corporation <jonathan.lynch@intel.com>
*/ */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <asm/elf.h> #include <asm/elf.h>
#include <asm/opcodes.h> #include <asm/opcodes.h>
#include <asm/pstate.h> #include <asm/pstate.h>
#include <crypto/internal/sha2.h>
#include <linux/kernel.h>
#include <linux/module.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_opcodes); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_opcodes);
asmlinkage void sha256_sparc64_transform(struct sha256_block_state *state, asmlinkage void sha256_sparc64_transform(struct sha256_block_state *state,
const u8 *data, size_t nblocks); const u8 *data, size_t nblocks);
void sha256_blocks_arch(struct sha256_block_state *state, static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks) const u8 *data, size_t nblocks)
{ {
if (static_branch_likely(&have_sha256_opcodes)) if (static_branch_likely(&have_sha256_opcodes))
sha256_sparc64_transform(state, data, nblocks); sha256_sparc64_transform(state, data, nblocks);
else else
sha256_blocks_generic(state, data, nblocks); sha256_blocks_generic(state, data, nblocks);
} }
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
static int __init sha256_sparc64_mod_init(void) #define sha256_mod_init_arch sha256_mod_init_arch
static inline void sha256_mod_init_arch(void)
{ {
unsigned long cfr; unsigned long cfr;
if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
return 0; return;
__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
if (!(cfr & CFR_SHA256)) if (!(cfr & CFR_SHA256))
return 0; return;
static_branch_enable(&have_sha256_opcodes); static_branch_enable(&have_sha256_opcodes);
pr_info("Using sparc64 sha256 opcode optimized SHA-256/SHA-224 implementation\n"); pr_info("Using sparc64 sha256 opcode optimized SHA-256/SHA-224 implementation\n");
return 0;
} }
subsys_initcall(sha256_sparc64_mod_init);
static void __exit sha256_sparc64_mod_exit(void)
{
}
module_exit(sha256_sparc64_mod_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA-256 accelerated using the sparc64 sha256 opcodes");

7
lib/crypto/x86/Kconfig
View File

@ -24,10 +24,3 @@ config CRYPTO_POLY1305_X86_64
depends on 64BIT depends on 64BIT
default CRYPTO_LIB_POLY1305 default CRYPTO_LIB_POLY1305
select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305
config CRYPTO_SHA256_X86_64
tristate
depends on 64BIT
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256
select CRYPTO_LIB_SHA256_GENERIC

3
lib/crypto/x86/Makefile
View File

@ -10,9 +10,6 @@ obj-$(CONFIG_CRYPTO_POLY1305_X86_64) += poly1305-x86_64.o
poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o
targets += poly1305-x86_64-cryptogams.S targets += poly1305-x86_64-cryptogams.S
obj-$(CONFIG_CRYPTO_SHA256_X86_64) += sha256-x86_64.o
sha256-x86_64-y := sha256.o sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256-ni-asm.o
quiet_cmd_perlasm = PERLASM $@ quiet_cmd_perlasm = PERLASM $@
cmd_perlasm = $(PERL) $< > $@ cmd_perlasm = $(PERL) $< > $@

29
lib/crypto/x86/sha256.c → lib/crypto/x86/sha256.h
View File

@ -1,14 +1,11 @@
// SPDX-License-Identifier: GPL-2.0-or-later /* SPDX-License-Identifier: GPL-2.0-or-later */
/* /*
* SHA-256 optimized for x86_64 * SHA-256 optimized for x86_64
* *
* Copyright 2025 Google LLC * Copyright 2025 Google LLC
*/ */
#include <asm/fpu/api.h> #include <asm/fpu/api.h>
#include <crypto/internal/sha2.h>
#include <crypto/internal/simd.h> #include <crypto/internal/simd.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/static_call.h> #include <linux/static_call.h>
asmlinkage void sha256_transform_ssse3(struct sha256_block_state *state, asmlinkage void sha256_transform_ssse3(struct sha256_block_state *state,
@ -24,8 +21,8 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86);
DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3); DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3);
void sha256_blocks_arch(struct sha256_block_state *state, static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks) const u8 *data, size_t nblocks)
{ {
if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) { if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) {
kernel_fpu_begin(); kernel_fpu_begin();
@ -35,14 +32,14 @@ void sha256_blocks_arch(struct sha256_block_state *state,
sha256_blocks_generic(state, data, nblocks); sha256_blocks_generic(state, data, nblocks);
} }
} }
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
static int __init sha256_x86_mod_init(void) #define sha256_mod_init_arch sha256_mod_init_arch
static inline void sha256_mod_init_arch(void)
{ {
if (boot_cpu_has(X86_FEATURE_SHA_NI)) { if (boot_cpu_has(X86_FEATURE_SHA_NI)) {
static_call_update(sha256_blocks_x86, sha256_ni_transform); static_call_update(sha256_blocks_x86, sha256_ni_transform);
} else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
XFEATURE_MASK_YMM, NULL) && NULL) &&
boot_cpu_has(X86_FEATURE_AVX)) { boot_cpu_has(X86_FEATURE_AVX)) {
if (boot_cpu_has(X86_FEATURE_AVX2) && if (boot_cpu_has(X86_FEATURE_AVX2) &&
boot_cpu_has(X86_FEATURE_BMI2)) boot_cpu_has(X86_FEATURE_BMI2))
@ -52,17 +49,7 @@ static int __init sha256_x86_mod_init(void)
static_call_update(sha256_blocks_x86, static_call_update(sha256_blocks_x86,
sha256_transform_avx); sha256_transform_avx);
} else if (!boot_cpu_has(X86_FEATURE_SSSE3)) { } else if (!boot_cpu_has(X86_FEATURE_SSSE3)) {
return 0; return;
} }
static_branch_enable(&have_sha256_x86); static_branch_enable(&have_sha256_x86);
return 0;
} }
subsys_initcall(sha256_x86_mod_init);
static void __exit sha256_x86_mod_exit(void)
{
}
module_exit(sha256_x86_mod_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA-256 optimized for x86_64");