1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4  *
5  * This is an implementation of the ChaCha20Poly1305 AEAD construction.
6  *
7  * Information: https://tools.ietf.org/html/rfc8439
8  */
9 
10 #include <crypto/chacha20poly1305.h>
11 #include <crypto/chacha.h>
12 #include <crypto/poly1305.h>
13 #include <crypto/utils.h>
14 
15 #include <linux/unaligned.h>
16 #include <linux/kernel.h>
17 #include <linux/init.h>
18 #include <linux/mm.h>
19 #include <linux/module.h>
20 
21 static void chacha_load_key(u32 *k, const u8 *in)
22 {
23 	k[0] = get_unaligned_le32(in);
24 	k[1] = get_unaligned_le32(in + 4);
25 	k[2] = get_unaligned_le32(in + 8);
26 	k[3] = get_unaligned_le32(in + 12);
27 	k[4] = get_unaligned_le32(in + 16);
28 	k[5] = get_unaligned_le32(in + 20);
29 	k[6] = get_unaligned_le32(in + 24);
30 	k[7] = get_unaligned_le32(in + 28);
31 }
32 
33 static void xchacha_init(struct chacha_state *chacha_state,
34 			 const u8 *key, const u8 *nonce)
35 {
36 	u32 k[CHACHA_KEY_WORDS];
37 	u8 iv[CHACHA_IV_SIZE];
38 
39 	memset(iv, 0, 8);
40 	memcpy(iv + 8, nonce + 16, 8);
41 
42 	chacha_load_key(k, key);
43 
44 	/* Compute the subkey given the original key and first 128 nonce bits */
45 	chacha_init(chacha_state, k, nonce);
46 	hchacha_block(chacha_state, k, 20);
47 
48 	chacha_init(chacha_state, k, iv);
49 
50 	memzero_explicit(k, sizeof(k));
51 	memzero_explicit(iv, sizeof(iv));
52 }
53 
54 static void
55 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
56 			   const u8 *ad, const size_t ad_len,
57 			   struct chacha_state *chacha_state)
58 {
59 	const u8 *pad0 = page_address(ZERO_PAGE(0));
60 	struct poly1305_desc_ctx poly1305_state;
61 	union {
62 		u8 block0[POLY1305_KEY_SIZE];
63 		__le64 lens[2];
64 	} b;
65 
66 	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
67 	poly1305_init(&poly1305_state, b.block0);
68 
69 	poly1305_update(&poly1305_state, ad, ad_len);
70 	if (ad_len & 0xf)
71 		poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
72 
73 	chacha20_crypt(chacha_state, dst, src, src_len);
74 
75 	poly1305_update(&poly1305_state, dst, src_len);
76 	if (src_len & 0xf)
77 		poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
78 
79 	b.lens[0] = cpu_to_le64(ad_len);
80 	b.lens[1] = cpu_to_le64(src_len);
81 	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
82 
83 	poly1305_final(&poly1305_state, dst + src_len);
84 
85 	chacha_zeroize_state(chacha_state);
86 	memzero_explicit(&b, sizeof(b));
87 }
88 
89 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
90 			      const u8 *ad, const size_t ad_len,
91 			      const u64 nonce,
92 			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
93 {
94 	struct chacha_state chacha_state;
95 	u32 k[CHACHA_KEY_WORDS];
96 	__le64 iv[2];
97 
98 	chacha_load_key(k, key);
99 
100 	iv[0] = 0;
101 	iv[1] = cpu_to_le64(nonce);
102 
103 	chacha_init(&chacha_state, k, (u8 *)iv);
104 	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len,
105 				   &chacha_state);
106 
107 	memzero_explicit(iv, sizeof(iv));
108 	memzero_explicit(k, sizeof(k));
109 }
110 EXPORT_SYMBOL(chacha20poly1305_encrypt);
111 
112 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
113 			       const u8 *ad, const size_t ad_len,
114 			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
115 			       const u8 key[CHACHA20POLY1305_KEY_SIZE])
116 {
117 	struct chacha_state chacha_state;
118 
119 	xchacha_init(&chacha_state, key, nonce);
120 	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len,
121 				   &chacha_state);
122 }
123 EXPORT_SYMBOL(xchacha20poly1305_encrypt);
124 
125 static bool
126 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
127 			   const u8 *ad, const size_t ad_len,
128 			   struct chacha_state *chacha_state)
129 {
130 	const u8 *pad0 = page_address(ZERO_PAGE(0));
131 	struct poly1305_desc_ctx poly1305_state;
132 	size_t dst_len;
133 	int ret;
134 	union {
135 		u8 block0[POLY1305_KEY_SIZE];
136 		u8 mac[POLY1305_DIGEST_SIZE];
137 		__le64 lens[2];
138 	} b;
139 
140 	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
141 		return false;
142 
143 	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
144 	poly1305_init(&poly1305_state, b.block0);
145 
146 	poly1305_update(&poly1305_state, ad, ad_len);
147 	if (ad_len & 0xf)
148 		poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
149 
150 	dst_len = src_len - POLY1305_DIGEST_SIZE;
151 	poly1305_update(&poly1305_state, src, dst_len);
152 	if (dst_len & 0xf)
153 		poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
154 
155 	b.lens[0] = cpu_to_le64(ad_len);
156 	b.lens[1] = cpu_to_le64(dst_len);
157 	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
158 
159 	poly1305_final(&poly1305_state, b.mac);
160 
161 	ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
162 	if (likely(!ret))
163 		chacha20_crypt(chacha_state, dst, src, dst_len);
164 
165 	memzero_explicit(&b, sizeof(b));
166 
167 	return !ret;
168 }
169 
170 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
171 			      const u8 *ad, const size_t ad_len,
172 			      const u64 nonce,
173 			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
174 {
175 	struct chacha_state chacha_state;
176 	u32 k[CHACHA_KEY_WORDS];
177 	__le64 iv[2];
178 	bool ret;
179 
180 	chacha_load_key(k, key);
181 
182 	iv[0] = 0;
183 	iv[1] = cpu_to_le64(nonce);
184 
185 	chacha_init(&chacha_state, k, (u8 *)iv);
186 	ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
187 					 &chacha_state);
188 
189 	chacha_zeroize_state(&chacha_state);
190 	memzero_explicit(iv, sizeof(iv));
191 	memzero_explicit(k, sizeof(k));
192 	return ret;
193 }
194 EXPORT_SYMBOL(chacha20poly1305_decrypt);
195 
196 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
197 			       const u8 *ad, const size_t ad_len,
198 			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
199 			       const u8 key[CHACHA20POLY1305_KEY_SIZE])
200 {
201 	struct chacha_state chacha_state;
202 
203 	xchacha_init(&chacha_state, key, nonce);
204 	return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
205 					  &chacha_state);
206 }
207 EXPORT_SYMBOL(xchacha20poly1305_decrypt);
208 
209 static
210 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
211 				       const size_t src_len,
212 				       const u8 *ad, const size_t ad_len,
213 				       const u64 nonce,
214 				       const u8 key[CHACHA20POLY1305_KEY_SIZE],
215 				       int encrypt)
216 {
217 	const u8 *pad0 = page_address(ZERO_PAGE(0));
218 	struct poly1305_desc_ctx poly1305_state;
219 	struct chacha_state chacha_state;
220 	struct sg_mapping_iter miter;
221 	size_t partial = 0;
222 	unsigned int flags;
223 	bool ret = true;
224 	int sl;
225 	union {
226 		struct {
227 			u32 k[CHACHA_KEY_WORDS];
228 			__le64 iv[2];
229 		};
230 		u8 block0[POLY1305_KEY_SIZE];
231 		u8 chacha_stream[CHACHA_BLOCK_SIZE];
232 		struct {
233 			u8 mac[2][POLY1305_DIGEST_SIZE];
234 		};
235 		__le64 lens[2];
236 	} b __aligned(16);
237 
238 	if (WARN_ON(src_len > INT_MAX))
239 		return false;
240 
241 	chacha_load_key(b.k, key);
242 
243 	b.iv[0] = 0;
244 	b.iv[1] = cpu_to_le64(nonce);
245 
246 	chacha_init(&chacha_state, b.k, (u8 *)b.iv);
247 	chacha20_crypt(&chacha_state, b.block0, pad0, sizeof(b.block0));
248 	poly1305_init(&poly1305_state, b.block0);
249 
250 	if (unlikely(ad_len)) {
251 		poly1305_update(&poly1305_state, ad, ad_len);
252 		if (ad_len & 0xf)
253 			poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
254 	}
255 
256 	flags = SG_MITER_TO_SG | SG_MITER_ATOMIC;
257 
258 	sg_miter_start(&miter, src, sg_nents(src), flags);
259 
260 	for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
261 		u8 *addr = miter.addr;
262 		size_t length = min_t(size_t, sl, miter.length);
263 
264 		if (!encrypt)
265 			poly1305_update(&poly1305_state, addr, length);
266 
267 		if (unlikely(partial)) {
268 			size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
269 
270 			crypto_xor(addr, b.chacha_stream + partial, l);
271 			partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
272 
273 			addr += l;
274 			length -= l;
275 		}
276 
277 		if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
278 			size_t l = length;
279 
280 			if (unlikely(length < sl))
281 				l &= ~(CHACHA_BLOCK_SIZE - 1);
282 			chacha20_crypt(&chacha_state, addr, addr, l);
283 			addr += l;
284 			length -= l;
285 		}
286 
287 		if (unlikely(length > 0)) {
288 			chacha20_crypt(&chacha_state, b.chacha_stream, pad0,
289 				       CHACHA_BLOCK_SIZE);
290 			crypto_xor(addr, b.chacha_stream, length);
291 			partial = length;
292 		}
293 
294 		if (encrypt)
295 			poly1305_update(&poly1305_state, miter.addr,
296 					min_t(size_t, sl, miter.length));
297 	}
298 
299 	if (src_len & 0xf)
300 		poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
301 
302 	b.lens[0] = cpu_to_le64(ad_len);
303 	b.lens[1] = cpu_to_le64(src_len);
304 	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
305 
306 	if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
307 		if (encrypt) {
308 			poly1305_final(&poly1305_state,
309 				       miter.addr + miter.length + sl);
310 			ret = true;
311 		} else {
312 			poly1305_final(&poly1305_state, b.mac[0]);
313 			ret = !crypto_memneq(b.mac[0],
314 					     miter.addr + miter.length + sl,
315 					     POLY1305_DIGEST_SIZE);
316 		}
317 	}
318 
319 	sg_miter_stop(&miter);
320 
321 	if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
322 		poly1305_final(&poly1305_state, b.mac[1]);
323 		sg_copy_buffer(src, sg_nents(src), b.mac[encrypt],
324 			       sizeof(b.mac[1]), src_len, !encrypt);
325 		ret = encrypt ||
326 		      !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
327 	}
328 
329 	chacha_zeroize_state(&chacha_state);
330 	memzero_explicit(&b, sizeof(b));
331 
332 	return ret;
333 }
334 
335 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
336 					 const u8 *ad, const size_t ad_len,
337 					 const u64 nonce,
338 					 const u8 key[CHACHA20POLY1305_KEY_SIZE])
339 {
340 	return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
341 						 nonce, key, 1);
342 }
343 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
344 
345 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
346 					 const u8 *ad, const size_t ad_len,
347 					 const u64 nonce,
348 					 const u8 key[CHACHA20POLY1305_KEY_SIZE])
349 {
350 	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
351 		return false;
352 
353 	return chacha20poly1305_crypt_sg_inplace(src,
354 						 src_len - POLY1305_DIGEST_SIZE,
355 						 ad, ad_len, nonce, key, 0);
356 }
357 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
358 
359 static int __init chacha20poly1305_init(void)
360 {
361 	if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) &&
362 	    WARN_ON(!chacha20poly1305_selftest()))
363 		return -ENODEV;
364 	return 0;
365 }
366 
367 static void __exit chacha20poly1305_exit(void)
368 {
369 }
370 
371 module_init(chacha20poly1305_init);
372 module_exit(chacha20poly1305_exit);
373 MODULE_LICENSE("GPL v2");
374 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
375 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
376