crypto: aesni - Add GCM_COMPLETE macro
Merge encode and decode tag calculations in GCM_COMPLETE macro. Scatter/gather routines will call this once at the end of encryption or decryption. Signed-off-by: Dave Watson <davejwatson@fb.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Dave Watson
Herbert Xu
parent
7af964c2fc
commit
adcadab369
|
|
@ -223,6 +223,67 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff
|
||||||
mov %r13, %r12
|
mov %r13, %r12
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
|
# GCM_COMPLETE Finishes update of tag of last partial block
|
||||||
|
# Output: Authorization Tag (AUTH_TAG)
|
||||||
|
# Clobbers rax, r10-r12, and xmm0, xmm1, xmm5-xmm15
|
||||||
|
.macro GCM_COMPLETE
|
||||||
|
mov arg8, %r12 # %r13 = aadLen (number of bytes)
|
||||||
|
shl $3, %r12 # convert into number of bits
|
||||||
|
movd %r12d, %xmm15 # len(A) in %xmm15
|
||||||
|
shl $3, %arg4 # len(C) in bits (*128)
|
||||||
|
MOVQ_R64_XMM %arg4, %xmm1
|
||||||
|
pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000
|
||||||
|
pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C)
|
||||||
|
pxor %xmm15, %xmm8
|
||||||
|
GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
|
||||||
|
# final GHASH computation
|
||||||
|
movdqa SHUF_MASK(%rip), %xmm10
|
||||||
|
PSHUFB_XMM %xmm10, %xmm8
|
||||||
|
|
||||||
|
mov %arg5, %rax # %rax = *Y0
|
||||||
|
movdqu (%rax), %xmm0 # %xmm0 = Y0
|
||||||
|
ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Y0)
|
||||||
|
pxor %xmm8, %xmm0
|
||||||
|
_return_T_\@:
|
||||||
|
mov arg9, %r10 # %r10 = authTag
|
||||||
|
mov arg10, %r11 # %r11 = auth_tag_len
|
||||||
|
cmp $16, %r11
|
||||||
|
je _T_16_\@
|
||||||
|
cmp $8, %r11
|
||||||
|
jl _T_4_\@
|
||||||
|
_T_8_\@:
|
||||||
|
MOVQ_R64_XMM %xmm0, %rax
|
||||||
|
mov %rax, (%r10)
|
||||||
|
add $8, %r10
|
||||||
|
sub $8, %r11
|
||||||
|
psrldq $8, %xmm0
|
||||||
|
cmp $0, %r11
|
||||||
|
je _return_T_done_\@
|
||||||
|
_T_4_\@:
|
||||||
|
movd %xmm0, %eax
|
||||||
|
mov %eax, (%r10)
|
||||||
|
add $4, %r10
|
||||||
|
sub $4, %r11
|
||||||
|
psrldq $4, %xmm0
|
||||||
|
cmp $0, %r11
|
||||||
|
je _return_T_done_\@
|
||||||
|
_T_123_\@:
|
||||||
|
movd %xmm0, %eax
|
||||||
|
cmp $2, %r11
|
||||||
|
jl _T_1_\@
|
||||||
|
mov %ax, (%r10)
|
||||||
|
cmp $2, %r11
|
||||||
|
je _return_T_done_\@
|
||||||
|
add $2, %r10
|
||||||
|
sar $16, %eax
|
||||||
|
_T_1_\@:
|
||||||
|
mov %al, (%r10)
|
||||||
|
jmp _return_T_done_\@
|
||||||
|
_T_16_\@:
|
||||||
|
movdqu %xmm0, (%r10)
|
||||||
|
_return_T_done_\@:
|
||||||
|
.endm
|
||||||
|
|
||||||
#ifdef __x86_64__
|
#ifdef __x86_64__
|
||||||
/* GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
|
/* GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
|
||||||
*
|
*
|
||||||
|
|
@ -1272,61 +1333,7 @@ _less_than_8_bytes_left_decrypt:
|
||||||
sub $1, %r13
|
sub $1, %r13
|
||||||
jne _less_than_8_bytes_left_decrypt
|
jne _less_than_8_bytes_left_decrypt
|
||||||
_multiple_of_16_bytes_decrypt:
|
_multiple_of_16_bytes_decrypt:
|
||||||
mov arg8, %r12 # %r13 = aadLen (number of bytes)
|
GCM_COMPLETE
|
||||||
shl $3, %r12 # convert into number of bits
|
|
||||||
movd %r12d, %xmm15 # len(A) in %xmm15
|
|
||||||
shl $3, %arg4 # len(C) in bits (*128)
|
|
||||||
MOVQ_R64_XMM %arg4, %xmm1
|
|
||||||
pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000
|
|
||||||
pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C)
|
|
||||||
pxor %xmm15, %xmm8
|
|
||||||
GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
|
|
||||||
# final GHASH computation
|
|
||||||
movdqa SHUF_MASK(%rip), %xmm10
|
|
||||||
PSHUFB_XMM %xmm10, %xmm8
|
|
||||||
|
|
||||||
mov %arg5, %rax # %rax = *Y0
|
|
||||||
movdqu (%rax), %xmm0 # %xmm0 = Y0
|
|
||||||
ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Y0)
|
|
||||||
pxor %xmm8, %xmm0
|
|
||||||
_return_T_decrypt:
|
|
||||||
mov arg9, %r10 # %r10 = authTag
|
|
||||||
mov arg10, %r11 # %r11 = auth_tag_len
|
|
||||||
cmp $16, %r11
|
|
||||||
je _T_16_decrypt
|
|
||||||
cmp $8, %r11
|
|
||||||
jl _T_4_decrypt
|
|
||||||
_T_8_decrypt:
|
|
||||||
MOVQ_R64_XMM %xmm0, %rax
|
|
||||||
mov %rax, (%r10)
|
|
||||||
add $8, %r10
|
|
||||||
sub $8, %r11
|
|
||||||
psrldq $8, %xmm0
|
|
||||||
cmp $0, %r11
|
|
||||||
je _return_T_done_decrypt
|
|
||||||
_T_4_decrypt:
|
|
||||||
movd %xmm0, %eax
|
|
||||||
mov %eax, (%r10)
|
|
||||||
add $4, %r10
|
|
||||||
sub $4, %r11
|
|
||||||
psrldq $4, %xmm0
|
|
||||||
cmp $0, %r11
|
|
||||||
je _return_T_done_decrypt
|
|
||||||
_T_123_decrypt:
|
|
||||||
movd %xmm0, %eax
|
|
||||||
cmp $2, %r11
|
|
||||||
jl _T_1_decrypt
|
|
||||||
mov %ax, (%r10)
|
|
||||||
cmp $2, %r11
|
|
||||||
je _return_T_done_decrypt
|
|
||||||
add $2, %r10
|
|
||||||
sar $16, %eax
|
|
||||||
_T_1_decrypt:
|
|
||||||
mov %al, (%r10)
|
|
||||||
jmp _return_T_done_decrypt
|
|
||||||
_T_16_decrypt:
|
|
||||||
movdqu %xmm0, (%r10)
|
|
||||||
_return_T_done_decrypt:
|
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
ret
|
||||||
ENDPROC(aesni_gcm_dec)
|
ENDPROC(aesni_gcm_dec)
|
||||||
|
|
@ -1502,61 +1509,8 @@ _less_than_8_bytes_left_encrypt:
|
||||||
sub $1, %r13
|
sub $1, %r13
|
||||||
jne _less_than_8_bytes_left_encrypt
|
jne _less_than_8_bytes_left_encrypt
|
||||||
_multiple_of_16_bytes_encrypt:
|
_multiple_of_16_bytes_encrypt:
|
||||||
mov arg8, %r12 # %r12 = addLen (number of bytes)
|
|
||||||
shl $3, %r12
|
|
||||||
movd %r12d, %xmm15 # len(A) in %xmm15
|
|
||||||
shl $3, %arg4 # len(C) in bits (*128)
|
|
||||||
MOVQ_R64_XMM %arg4, %xmm1
|
|
||||||
pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000
|
|
||||||
pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C)
|
|
||||||
pxor %xmm15, %xmm8
|
|
||||||
GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
|
|
||||||
# final GHASH computation
|
|
||||||
movdqa SHUF_MASK(%rip), %xmm10
|
|
||||||
PSHUFB_XMM %xmm10, %xmm8 # perform a 16 byte swap
|
|
||||||
|
|
||||||
mov %arg5, %rax # %rax = *Y0
|
|
||||||
movdqu (%rax), %xmm0 # %xmm0 = Y0
|
|
||||||
ENCRYPT_SINGLE_BLOCK %xmm0, %xmm15 # Encrypt(K, Y0)
|
|
||||||
pxor %xmm8, %xmm0
|
|
||||||
_return_T_encrypt:
|
_return_T_encrypt:
|
||||||
mov arg9, %r10 # %r10 = authTag
|
GCM_COMPLETE
|
||||||
mov arg10, %r11 # %r11 = auth_tag_len
|
|
||||||
cmp $16, %r11
|
|
||||||
je _T_16_encrypt
|
|
||||||
cmp $8, %r11
|
|
||||||
jl _T_4_encrypt
|
|
||||||
_T_8_encrypt:
|
|
||||||
MOVQ_R64_XMM %xmm0, %rax
|
|
||||||
mov %rax, (%r10)
|
|
||||||
add $8, %r10
|
|
||||||
sub $8, %r11
|
|
||||||
psrldq $8, %xmm0
|
|
||||||
cmp $0, %r11
|
|
||||||
je _return_T_done_encrypt
|
|
||||||
_T_4_encrypt:
|
|
||||||
movd %xmm0, %eax
|
|
||||||
mov %eax, (%r10)
|
|
||||||
add $4, %r10
|
|
||||||
sub $4, %r11
|
|
||||||
psrldq $4, %xmm0
|
|
||||||
cmp $0, %r11
|
|
||||||
je _return_T_done_encrypt
|
|
||||||
_T_123_encrypt:
|
|
||||||
movd %xmm0, %eax
|
|
||||||
cmp $2, %r11
|
|
||||||
jl _T_1_encrypt
|
|
||||||
mov %ax, (%r10)
|
|
||||||
cmp $2, %r11
|
|
||||||
je _return_T_done_encrypt
|
|
||||||
add $2, %r10
|
|
||||||
sar $16, %eax
|
|
||||||
_T_1_encrypt:
|
|
||||||
mov %al, (%r10)
|
|
||||||
jmp _return_T_done_encrypt
|
|
||||||
_T_16_encrypt:
|
|
||||||
movdqu %xmm0, (%r10)
|
|
||||||
_return_T_done_encrypt:
|
|
||||||
FUNC_RESTORE
|
FUNC_RESTORE
|
||||||
ret
|
ret
|
||||||
ENDPROC(aesni_gcm_enc)
|
ENDPROC(aesni_gcm_enc)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue