Skip to content
This repository has been archived by the owner on Dec 27, 2019. It is now read-only.

Commit

Permalink
chacha20poly1305: port to sgmitter for 5.5
Browse files Browse the repository at this point in the history 
I'm not totally comfortable with these changes yet, and it'll require
some more scrutiny. But it's a start.
  • Loading branch information
@zx2c4
zx2c4 committed Dec 5, 2019
1 parent 5eb87fb commit 12b4b22
Show file tree
Hide file tree
Showing 6 changed files with 151 additions and 122 deletions.
3 changes: 1 addition & 2 deletions src/Kconfig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@ config WIREGUARD
select NET_UDP_TUNNEL
select DST_CACHE
select CRYPTO
select CRYPTO_BLKCIPHER
select XOR_BLOCKS
select CRYPTO_ALGAPI
select VFP
select VFPv3 if CPU_V7
select NEON if CPU_V7
Expand Down
12 changes: 6 additions & 6 deletions src/crypto/include/zinc/chacha20poly1305.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,19 @@ void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE]);

bool __must_check chacha20poly1305_encrypt_sg(
struct scatterlist *dst, struct scatterlist *src, const size_t src_len,
const u8 *ad, const size_t ad_len, const u64 nonce,
bool __must_check chacha20poly1305_encrypt_sg_inplace(
struct scatterlist *src, const size_t src_len, const u8 *ad,
const size_t ad_len, const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE], simd_context_t *simd_context);

bool __must_check
chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 *ad, const size_t ad_len, const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE]);

bool __must_check chacha20poly1305_decrypt_sg(
struct scatterlist *dst, struct scatterlist *src, const size_t src_len,
const u8 *ad, const size_t ad_len, const u64 nonce,
bool __must_check chacha20poly1305_decrypt_sg_inplace(
struct scatterlist *src, size_t src_len, const u8 *ad,
const size_t ad_len, const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE], simd_context_t *simd_context);

void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
Expand Down
218 changes: 128 additions & 90 deletions src/crypto/zinc/chacha20poly1305.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,7 @@
#include <linux/init.h>
#include <crypto/scatterwalk.h> // For blkcipher_walk.

static const u8 pad0[16] = { 0 };

static struct blkcipher_desc desc = { .tfm = &(struct crypto_blkcipher){
.base = { .__crt_alg = &(struct crypto_alg){
.cra_blocksize = 1,
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
.cra_alignmask = sizeof(u32) - 1
#endif
} }
} };
static const u8 pad0[CHACHA20_BLOCK_SIZE] = { 0 };

static inline void
__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
Expand Down Expand Up @@ -82,22 +73,25 @@ void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
}
EXPORT_SYMBOL(chacha20poly1305_encrypt);

bool chacha20poly1305_encrypt_sg(struct scatterlist *dst,
struct scatterlist *src, const size_t src_len,
const u8 *ad, const size_t ad_len,
const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE],
simd_context_t *simd_context)
bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src,
const size_t src_len,
const u8 *ad, const size_t ad_len,
const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE],
simd_context_t *simd_context)
{
struct poly1305_ctx poly1305_state;
struct chacha20_ctx chacha20_state;
int ret = 0;
struct blkcipher_walk walk;
struct sg_mapping_iter miter;
size_t partial = 0;
ssize_t sl;
union {
u8 chacha20_stream[CHACHA20_BLOCK_SIZE];
u8 block0[POLY1305_KEY_SIZE];
u8 mac[POLY1305_MAC_SIZE];
__le64 lens[2];
} b = { { 0 } };
} b __aligned(16) = { { 0 } };


chacha20_init(&chacha20_state, key, nonce);
chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0),
Expand All @@ -108,32 +102,43 @@ bool chacha20poly1305_encrypt_sg(struct scatterlist *dst,
poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf,
simd_context);

if (likely(src_len)) {
blkcipher_walk_init(&walk, dst, src, src_len);
ret = blkcipher_walk_virt_block(&desc, &walk,
CHACHA20_BLOCK_SIZE);
while (walk.nbytes >= CHACHA20_BLOCK_SIZE) {
size_t chunk_len =
rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE);

chacha20(&chacha20_state, walk.dst.virt.addr,
walk.src.virt.addr, chunk_len, simd_context);
poly1305_update(&poly1305_state, walk.dst.virt.addr,
chunk_len, simd_context);
simd_relax(simd_context);
ret = blkcipher_walk_done(&desc, &walk,
walk.nbytes % CHACHA20_BLOCK_SIZE);
sg_miter_start(&miter, src, sg_nents(src), SG_MITER_TO_SG | SG_MITER_ATOMIC);
for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
u8 *addr = miter.addr;
size_t length = min_t(size_t, sl, miter.length);

if (unlikely(partial)) {
size_t l = min(length, CHACHA20_BLOCK_SIZE - partial);

crypto_xor(addr, b.chacha20_stream + partial, l);
partial = (partial + l) & (CHACHA20_BLOCK_SIZE - 1);

addr += l;
length -= l;
}
if (walk.nbytes) {
chacha20(&chacha20_state, walk.dst.virt.addr,
walk.src.virt.addr, walk.nbytes, simd_context);
poly1305_update(&poly1305_state, walk.dst.virt.addr,
walk.nbytes, simd_context);
ret = blkcipher_walk_done(&desc, &walk, 0);

if (likely(length >= CHACHA20_BLOCK_SIZE || length == sl)) {
size_t l = length;

if (unlikely(length < sl))
l &= ~(CHACHA20_BLOCK_SIZE - 1);
chacha20(&chacha20_state, addr, addr, l, simd_context);
addr += l;
length -= l;
}

if (unlikely(length > 0)) {
chacha20(&chacha20_state, b.chacha20_stream, pad0,
CHACHA20_BLOCK_SIZE, simd_context);
crypto_xor(addr, b.chacha20_stream, length);
partial = length;
}

poly1305_update(&poly1305_state, miter.addr,
min_t(size_t, sl, miter.length), simd_context);

simd_relax(simd_context);
}
if (unlikely(ret))
goto err;

poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf,
simd_context);
Expand All @@ -143,14 +148,22 @@ bool chacha20poly1305_encrypt_sg(struct scatterlist *dst,
poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens),
simd_context);

poly1305_final(&poly1305_state, b.mac, simd_context);
scatterwalk_map_and_copy(b.mac, dst, src_len, sizeof(b.mac), 1);
err:
if (likely(sl <= -POLY1305_MAC_SIZE))
poly1305_final(&poly1305_state, miter.addr + miter.length + sl,
simd_context);

sg_miter_stop(&miter);

if (unlikely(sl > -POLY1305_MAC_SIZE)) {
poly1305_final(&poly1305_state, b.mac, simd_context);
scatterwalk_map_and_copy(b.mac, src, src_len, sizeof(b.mac), 1);
}

memzero_explicit(&chacha20_state, sizeof(chacha20_state));
memzero_explicit(&b, sizeof(b));
return !ret;
return true;
}
EXPORT_SYMBOL(chacha20poly1305_encrypt_sg);
EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);

static inline bool
__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
Expand Down Expand Up @@ -217,29 +230,32 @@ bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
}
EXPORT_SYMBOL(chacha20poly1305_decrypt);

bool chacha20poly1305_decrypt_sg(struct scatterlist *dst,
struct scatterlist *src, const size_t src_len,
const u8 *ad, const size_t ad_len,
const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE],
simd_context_t *simd_context)
bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src,
size_t src_len,
const u8 *ad, const size_t ad_len,
const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE],
simd_context_t *simd_context)
{
struct poly1305_ctx poly1305_state;
struct chacha20_ctx chacha20_state;
struct blkcipher_walk walk;
int ret = 0;
size_t dst_len;
struct sg_mapping_iter miter;
size_t partial = 0;
ssize_t sl;
union {
u8 chacha20_stream[CHACHA20_BLOCK_SIZE];
u8 block0[POLY1305_KEY_SIZE];
struct {
u8 read_mac[POLY1305_MAC_SIZE];
u8 computed_mac[POLY1305_MAC_SIZE];
};
__le64 lens[2];
} b = { { 0 } };
} b __aligned(16) = { { 0 } };
bool ret = false;

if (unlikely(src_len < POLY1305_MAC_SIZE))
return false;
return ret;
src_len -= POLY1305_MAC_SIZE;

chacha20_init(&chacha20_state, key, nonce);
chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0),
Expand All @@ -250,52 +266,74 @@ bool chacha20poly1305_decrypt_sg(struct scatterlist *dst,
poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf,
simd_context);

dst_len = src_len - POLY1305_MAC_SIZE;
if (likely(dst_len)) {
blkcipher_walk_init(&walk, dst, src, dst_len);
ret = blkcipher_walk_virt_block(&desc, &walk,
CHACHA20_BLOCK_SIZE);
while (walk.nbytes >= CHACHA20_BLOCK_SIZE) {
size_t chunk_len =
rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE);

poly1305_update(&poly1305_state, walk.src.virt.addr,
chunk_len, simd_context);
chacha20(&chacha20_state, walk.dst.virt.addr,
walk.src.virt.addr, chunk_len, simd_context);
simd_relax(simd_context);
ret = blkcipher_walk_done(&desc, &walk,
walk.nbytes % CHACHA20_BLOCK_SIZE);
sg_miter_start(&miter, src, sg_nents(src), SG_MITER_TO_SG | SG_MITER_ATOMIC);
for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
u8 *addr = miter.addr;
size_t length = min_t(size_t, sl, miter.length);

poly1305_update(&poly1305_state, addr, length, simd_context);

if (unlikely(partial)) {
size_t l = min(length, CHACHA20_BLOCK_SIZE - partial);

crypto_xor(addr, b.chacha20_stream + partial, l);
partial = (partial + l) & (CHACHA20_BLOCK_SIZE - 1);

addr += l;
length -= l;
}
if (walk.nbytes) {
poly1305_update(&poly1305_state, walk.src.virt.addr,
walk.nbytes, simd_context);
chacha20(&chacha20_state, walk.dst.virt.addr,
walk.src.virt.addr, walk.nbytes, simd_context);
ret = blkcipher_walk_done(&desc, &walk, 0);

if (likely(length >= CHACHA20_BLOCK_SIZE || length == sl)) {
size_t l = length;

if (unlikely(length < sl))
l &= ~(CHACHA20_BLOCK_SIZE - 1);
chacha20(&chacha20_state, addr, addr, l, simd_context);
addr += l;
length -= l;
}

if (unlikely(length > 0)) {
chacha20(&chacha20_state, b.chacha20_stream, pad0,
CHACHA20_BLOCK_SIZE, simd_context);
crypto_xor(addr, b.chacha20_stream, length);
partial = length;
}

simd_relax(simd_context);
}
if (unlikely(ret))
goto err;

poly1305_update(&poly1305_state, pad0, (0x10 - dst_len) & 0xf,
poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf,
simd_context);

b.lens[0] = cpu_to_le64(ad_len);
b.lens[1] = cpu_to_le64(dst_len);
b.lens[1] = cpu_to_le64(src_len);
poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens),
simd_context);

poly1305_final(&poly1305_state, b.computed_mac, simd_context);
if (likely(sl <= -POLY1305_MAC_SIZE)) {
poly1305_final(&poly1305_state, b.computed_mac, simd_context);
ret = !crypto_memneq(b.computed_mac,
miter.addr + miter.length + sl,
POLY1305_MAC_SIZE);
}

sg_miter_stop(&miter);

if (unlikely(sl > -POLY1305_MAC_SIZE)) {
poly1305_final(&poly1305_state, b.computed_mac, simd_context);
scatterwalk_map_and_copy(b.read_mac, src, src_len,
sizeof(b.read_mac), 0);
ret = !crypto_memneq(b.read_mac, b.computed_mac,
POLY1305_MAC_SIZE);

}

scatterwalk_map_and_copy(b.read_mac, src, dst_len, POLY1305_MAC_SIZE, 0);
ret = crypto_memneq(b.read_mac, b.computed_mac, POLY1305_MAC_SIZE);
err:
memzero_explicit(&chacha20_state, sizeof(chacha20_state));
memzero_explicit(&b, sizeof(b));
return !ret;
return ret;
}
EXPORT_SYMBOL(chacha20poly1305_decrypt_sg);
EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);

void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 *ad, const size_t ad_len,
Expand Down
27 changes: 9 additions & 18 deletions src/crypto/zinc/selftest/chacha20poly1305.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8880,14 +8880,13 @@ static bool __init chacha20poly1305_selftest(void)
{
enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 };
size_t i;
u8 *computed_output = NULL, *heap_src = NULL;
u8 *computed_output = NULL;
bool success = true, ret;
simd_context_t simd_context;
struct scatterlist sg_src, sg_dst;
struct scatterlist sg_src;

heap_src = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL);
computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL);
if (!heap_src || !computed_output) {
if (!computed_output) {
pr_err("chacha20poly1305 self-test malloc: FAIL\n");
success = false;
goto out;
Expand Down Expand Up @@ -8916,15 +8915,12 @@ static bool __init chacha20poly1305_selftest(void)
for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) {
if (chacha20poly1305_enc_vectors[i].nlen != 8)
continue;
memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN);
memcpy(heap_src, chacha20poly1305_enc_vectors[i].input,
memcpy(computed_output, chacha20poly1305_enc_vectors[i].input,
chacha20poly1305_enc_vectors[i].ilen);
sg_init_one(&sg_src, heap_src,
chacha20poly1305_enc_vectors[i].ilen);
sg_init_one(&sg_dst, computed_output,
sg_init_one(&sg_src, computed_output,
chacha20poly1305_enc_vectors[i].ilen +
POLY1305_MAC_SIZE);
ret = chacha20poly1305_encrypt_sg(&sg_dst, &sg_src,
ret = chacha20poly1305_encrypt_sg_inplace(&sg_src,
chacha20poly1305_enc_vectors[i].ilen,
chacha20poly1305_enc_vectors[i].assoc,
chacha20poly1305_enc_vectors[i].alen,
Expand Down Expand Up @@ -8963,15 +8959,11 @@ static bool __init chacha20poly1305_selftest(void)
}
simd_get(&simd_context);
for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) {
memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN);
memcpy(heap_src, chacha20poly1305_dec_vectors[i].input,
memcpy(computed_output, chacha20poly1305_dec_vectors[i].input,
chacha20poly1305_dec_vectors[i].ilen);
sg_init_one(&sg_src, heap_src,
sg_init_one(&sg_src, computed_output,
chacha20poly1305_dec_vectors[i].ilen);
sg_init_one(&sg_dst, computed_output,
chacha20poly1305_dec_vectors[i].ilen -
POLY1305_MAC_SIZE);
ret = chacha20poly1305_decrypt_sg(&sg_dst, &sg_src,
ret = chacha20poly1305_decrypt_sg_inplace(&sg_src,
chacha20poly1305_dec_vectors[i].ilen,
chacha20poly1305_dec_vectors[i].assoc,
chacha20poly1305_dec_vectors[i].alen,
Expand Down Expand Up @@ -9028,7 +9020,6 @@ static bool __init chacha20poly1305_selftest(void)
}

out:
kfree(heap_src);
kfree(computed_output);
return success;
}
Loading

0 comments on commit 12b4b22

Please sign in to comment.