1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * ChaCha20-Poly1305 AEAD, RFC7539 4 * 5 * Copyright (C) 2015 Martin Willi 6 */ 7 8 #include <crypto/internal/aead.h> 9 #include <crypto/internal/hash.h> 10 #include <crypto/internal/skcipher.h> 11 #include <crypto/scatterwalk.h> 12 #include <crypto/chacha.h> 13 #include <crypto/poly1305.h> 14 #include <linux/err.h> 15 #include <linux/kernel.h> 16 #include <linux/mm.h> 17 #include <linux/module.h> 18 #include <linux/string.h> 19 20 struct chachapoly_instance_ctx { 21 struct crypto_skcipher_spawn chacha; 22 unsigned int saltlen; 23 }; 24 25 struct chachapoly_ctx { 26 struct crypto_skcipher *chacha; 27 /* key bytes we use for the ChaCha20 IV */ 28 unsigned int saltlen; 29 u8 salt[] __counted_by(saltlen); 30 }; 31 32 struct chacha_req { 33 u8 iv[CHACHA_IV_SIZE]; 34 struct scatterlist src[1]; 35 struct skcipher_request req; /* must be last member */ 36 }; 37 38 struct chachapoly_req_ctx { 39 struct scatterlist src[2]; 40 struct scatterlist dst[2]; 41 /* the key we generate for Poly1305 using Chacha20 */ 42 u8 key[POLY1305_KEY_SIZE]; 43 /* calculated Poly1305 tag */ 44 u8 tag[POLY1305_DIGEST_SIZE]; 45 /* length of data to en/decrypt, without ICV */ 46 unsigned int cryptlen; 47 /* Actual AD, excluding IV */ 48 unsigned int assoclen; 49 /* request flags, with MAY_SLEEP cleared if needed */ 50 u32 flags; 51 union { 52 struct chacha_req chacha; 53 } u; 54 }; 55 56 static inline void async_done_continue(struct aead_request *req, int err, 57 int (*cont)(struct aead_request *)) 58 { 59 if (!err) { 60 struct chachapoly_req_ctx *rctx = aead_request_ctx(req); 61 62 rctx->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 63 err = cont(req); 64 } 65 66 if (err != -EINPROGRESS && err != -EBUSY) 67 aead_request_complete(req, err); 68 } 69 70 static void chacha_iv(u8 *iv, struct aead_request *req, u32 icb) 71 { 72 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 73 __le32 leicb = cpu_to_le32(icb); 74 75 memcpy(iv, &leicb, sizeof(leicb)); 76 memcpy(iv + sizeof(leicb), ctx->salt, ctx->saltlen); 77 memcpy(iv + sizeof(leicb) + ctx->saltlen, req->iv, 78 CHACHA_IV_SIZE - sizeof(leicb) - ctx->saltlen); 79 } 80 81 static int poly_verify_tag(struct aead_request *req) 82 { 83 struct chachapoly_req_ctx *rctx = aead_request_ctx(req); 84 u8 tag[sizeof(rctx->tag)]; 85 86 scatterwalk_map_and_copy(tag, req->src, 87 req->assoclen + rctx->cryptlen, 88 sizeof(tag), 0); 89 if (crypto_memneq(tag, rctx->tag, sizeof(tag))) 90 return -EBADMSG; 91 return 0; 92 } 93 94 static void chacha_decrypt_done(void *data, int err) 95 { 96 async_done_continue(data, err, poly_verify_tag); 97 } 98 99 static int chacha_decrypt(struct aead_request *req) 100 { 101 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 102 struct chachapoly_req_ctx *rctx = aead_request_ctx(req); 103 struct chacha_req *creq = &rctx->u.chacha; 104 struct scatterlist *src, *dst; 105 int err; 106 107 if (rctx->cryptlen == 0) 108 goto skip; 109 110 chacha_iv(creq->iv, req, 1); 111 112 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); 113 dst = src; 114 if (req->src != req->dst) 115 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); 116 117 skcipher_request_set_callback(&creq->req, rctx->flags, 118 chacha_decrypt_done, req); 119 skcipher_request_set_tfm(&creq->req, ctx->chacha); 120 skcipher_request_set_crypt(&creq->req, src, dst, 121 rctx->cryptlen, creq->iv); 122 err = crypto_skcipher_decrypt(&creq->req); 123 if (err) 124 return err; 125 126 skip: 127 return poly_verify_tag(req); 128 } 129 130 static int poly_hash(struct aead_request *req) 131 { 132 struct chachapoly_req_ctx *rctx = aead_request_ctx(req); 133 const void *zp = page_address(ZERO_PAGE(0)); 134 struct scatterlist *sg = req->src; 135 struct poly1305_desc_ctx desc; 136 struct scatter_walk walk; 137 struct { 138 union { 139 struct { 140 __le64 assoclen; 141 __le64 cryptlen; 142 }; 143 u8 u8[16]; 144 }; 145 } tail; 146 unsigned int padlen; 147 unsigned int total; 148 149 if (sg != req->dst) 150 memcpy_sglist(req->dst, sg, req->assoclen); 151 152 if (rctx->cryptlen == req->cryptlen) /* encrypting */ 153 sg = req->dst; 154 155 poly1305_init(&desc, rctx->key); 156 scatterwalk_start(&walk, sg); 157 158 total = rctx->assoclen; 159 while (total) { 160 unsigned int n = scatterwalk_next(&walk, total); 161 162 poly1305_update(&desc, walk.addr, n); 163 scatterwalk_done_src(&walk, n); 164 total -= n; 165 } 166 167 padlen = -rctx->assoclen % POLY1305_BLOCK_SIZE; 168 poly1305_update(&desc, zp, padlen); 169 170 scatterwalk_skip(&walk, req->assoclen - rctx->assoclen); 171 172 total = rctx->cryptlen; 173 while (total) { 174 unsigned int n = scatterwalk_next(&walk, total); 175 176 poly1305_update(&desc, walk.addr, n); 177 scatterwalk_done_src(&walk, n); 178 total -= n; 179 } 180 181 padlen = -rctx->cryptlen % POLY1305_BLOCK_SIZE; 182 poly1305_update(&desc, zp, padlen); 183 184 tail.assoclen = cpu_to_le64(rctx->assoclen); 185 tail.cryptlen = cpu_to_le64(rctx->cryptlen); 186 poly1305_update(&desc, tail.u8, sizeof(tail)); 187 memzero_explicit(&tail, sizeof(tail)); 188 poly1305_final(&desc, rctx->tag); 189 190 if (rctx->cryptlen != req->cryptlen) 191 return chacha_decrypt(req); 192 193 memcpy_to_scatterwalk(&walk, rctx->tag, sizeof(rctx->tag)); 194 return 0; 195 } 196 197 static void poly_genkey_done(void *data, int err) 198 { 199 async_done_continue(data, err, poly_hash); 200 } 201 202 static int poly_genkey(struct aead_request *req) 203 { 204 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 205 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); 206 struct chachapoly_req_ctx *rctx = aead_request_ctx(req); 207 struct chacha_req *creq = &rctx->u.chacha; 208 int err; 209 210 rctx->assoclen = req->assoclen; 211 212 if (crypto_aead_ivsize(tfm) == 8) { 213 if (rctx->assoclen < 8) 214 return -EINVAL; 215 rctx->assoclen -= 8; 216 } 217 218 memset(rctx->key, 0, sizeof(rctx->key)); 219 sg_init_one(creq->src, rctx->key, sizeof(rctx->key)); 220 221 chacha_iv(creq->iv, req, 0); 222 223 skcipher_request_set_callback(&creq->req, rctx->flags, 224 poly_genkey_done, req); 225 skcipher_request_set_tfm(&creq->req, ctx->chacha); 226 skcipher_request_set_crypt(&creq->req, creq->src, creq->src, 227 POLY1305_KEY_SIZE, creq->iv); 228 229 err = crypto_skcipher_decrypt(&creq->req); 230 if (err) 231 return err; 232 233 return poly_hash(req); 234 } 235 236 static void chacha_encrypt_done(void *data, int err) 237 { 238 async_done_continue(data, err, poly_genkey); 239 } 240 241 static int chacha_encrypt(struct aead_request *req) 242 { 243 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 244 struct chachapoly_req_ctx *rctx = aead_request_ctx(req); 245 struct chacha_req *creq = &rctx->u.chacha; 246 struct scatterlist *src, *dst; 247 int err; 248 249 if (req->cryptlen == 0) 250 goto skip; 251 252 chacha_iv(creq->iv, req, 1); 253 254 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); 255 dst = src; 256 if (req->src != req->dst) 257 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); 258 259 skcipher_request_set_callback(&creq->req, rctx->flags, 260 chacha_encrypt_done, req); 261 skcipher_request_set_tfm(&creq->req, ctx->chacha); 262 skcipher_request_set_crypt(&creq->req, src, dst, 263 req->cryptlen, creq->iv); 264 err = crypto_skcipher_encrypt(&creq->req); 265 if (err) 266 return err; 267 268 skip: 269 return poly_genkey(req); 270 } 271 272 static int chachapoly_encrypt(struct aead_request *req) 273 { 274 struct chachapoly_req_ctx *rctx = aead_request_ctx(req); 275 276 rctx->cryptlen = req->cryptlen; 277 rctx->flags = aead_request_flags(req); 278 279 /* encrypt call chain: 280 * - chacha_encrypt/done() 281 * - poly_genkey/done() 282 * - poly_hash() 283 */ 284 return chacha_encrypt(req); 285 } 286 287 static int chachapoly_decrypt(struct aead_request *req) 288 { 289 struct chachapoly_req_ctx *rctx = aead_request_ctx(req); 290 291 rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE; 292 rctx->flags = aead_request_flags(req); 293 294 /* decrypt call chain: 295 * - poly_genkey/done() 296 * - poly_hash() 297 * - chacha_decrypt/done() 298 * - poly_verify_tag() 299 */ 300 return poly_genkey(req); 301 } 302 303 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 304 unsigned int keylen) 305 { 306 struct chachapoly_ctx *ctx = crypto_aead_ctx(aead); 307 308 if (keylen != ctx->saltlen + CHACHA_KEY_SIZE) 309 return -EINVAL; 310 311 keylen -= ctx->saltlen; 312 memcpy(ctx->salt, key + keylen, ctx->saltlen); 313 314 crypto_skcipher_clear_flags(ctx->chacha, CRYPTO_TFM_REQ_MASK); 315 crypto_skcipher_set_flags(ctx->chacha, crypto_aead_get_flags(aead) & 316 CRYPTO_TFM_REQ_MASK); 317 return crypto_skcipher_setkey(ctx->chacha, key, keylen); 318 } 319 320 static int chachapoly_setauthsize(struct crypto_aead *tfm, 321 unsigned int authsize) 322 { 323 if (authsize != POLY1305_DIGEST_SIZE) 324 return -EINVAL; 325 326 return 0; 327 } 328 329 static int chachapoly_init(struct crypto_aead *tfm) 330 { 331 struct aead_instance *inst = aead_alg_instance(tfm); 332 struct chachapoly_instance_ctx *ictx = aead_instance_ctx(inst); 333 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); 334 struct crypto_skcipher *chacha; 335 unsigned long align; 336 337 chacha = crypto_spawn_skcipher(&ictx->chacha); 338 if (IS_ERR(chacha)) 339 return PTR_ERR(chacha); 340 341 ctx->chacha = chacha; 342 ctx->saltlen = ictx->saltlen; 343 344 align = crypto_aead_alignmask(tfm); 345 align &= ~(crypto_tfm_ctx_alignment() - 1); 346 crypto_aead_set_reqsize( 347 tfm, 348 align + offsetof(struct chachapoly_req_ctx, u) + 349 offsetof(struct chacha_req, req) + 350 sizeof(struct skcipher_request) + 351 crypto_skcipher_reqsize(chacha)); 352 353 return 0; 354 } 355 356 static void chachapoly_exit(struct crypto_aead *tfm) 357 { 358 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); 359 360 crypto_free_skcipher(ctx->chacha); 361 } 362 363 static void chachapoly_free(struct aead_instance *inst) 364 { 365 struct chachapoly_instance_ctx *ctx = aead_instance_ctx(inst); 366 367 crypto_drop_skcipher(&ctx->chacha); 368 kfree(inst); 369 } 370 371 static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, 372 const char *name, unsigned int ivsize) 373 { 374 u32 mask; 375 struct aead_instance *inst; 376 struct chachapoly_instance_ctx *ctx; 377 struct skcipher_alg_common *chacha; 378 int err; 379 380 if (ivsize > CHACHAPOLY_IV_SIZE) 381 return -EINVAL; 382 383 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); 384 if (err) 385 return err; 386 387 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 388 if (!inst) 389 return -ENOMEM; 390 ctx = aead_instance_ctx(inst); 391 ctx->saltlen = CHACHAPOLY_IV_SIZE - ivsize; 392 393 err = crypto_grab_skcipher(&ctx->chacha, aead_crypto_instance(inst), 394 crypto_attr_alg_name(tb[1]), 0, mask); 395 if (err) 396 goto err_free_inst; 397 chacha = crypto_spawn_skcipher_alg_common(&ctx->chacha); 398 399 err = -EINVAL; 400 if (strcmp(crypto_attr_alg_name(tb[2]), "poly1305") && 401 strcmp(crypto_attr_alg_name(tb[2]), "poly1305-generic")) 402 goto err_free_inst; 403 /* Need 16-byte IV size, including Initial Block Counter value */ 404 if (chacha->ivsize != CHACHA_IV_SIZE) 405 goto err_free_inst; 406 /* Not a stream cipher? */ 407 if (chacha->base.cra_blocksize != 1) 408 goto err_free_inst; 409 410 err = -ENAMETOOLONG; 411 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 412 "%s(%s,poly1305)", name, 413 chacha->base.cra_name) >= CRYPTO_MAX_ALG_NAME) 414 goto err_free_inst; 415 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 416 "%s(%s,poly1305-generic)", name, 417 chacha->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) 418 goto err_free_inst; 419 420 inst->alg.base.cra_priority = chacha->base.cra_priority; 421 inst->alg.base.cra_blocksize = 1; 422 inst->alg.base.cra_alignmask = chacha->base.cra_alignmask; 423 inst->alg.base.cra_ctxsize = sizeof(struct chachapoly_ctx) + 424 ctx->saltlen; 425 inst->alg.ivsize = ivsize; 426 inst->alg.chunksize = chacha->chunksize; 427 inst->alg.maxauthsize = POLY1305_DIGEST_SIZE; 428 inst->alg.init = chachapoly_init; 429 inst->alg.exit = chachapoly_exit; 430 inst->alg.encrypt = chachapoly_encrypt; 431 inst->alg.decrypt = chachapoly_decrypt; 432 inst->alg.setkey = chachapoly_setkey; 433 inst->alg.setauthsize = chachapoly_setauthsize; 434 435 inst->free = chachapoly_free; 436 437 err = aead_register_instance(tmpl, inst); 438 if (err) { 439 err_free_inst: 440 chachapoly_free(inst); 441 } 442 return err; 443 } 444 445 static int rfc7539_create(struct crypto_template *tmpl, struct rtattr **tb) 446 { 447 return chachapoly_create(tmpl, tb, "rfc7539", 12); 448 } 449 450 static int rfc7539esp_create(struct crypto_template *tmpl, struct rtattr **tb) 451 { 452 return chachapoly_create(tmpl, tb, "rfc7539esp", 8); 453 } 454 455 static struct crypto_template rfc7539_tmpls[] = { 456 { 457 .name = "rfc7539", 458 .create = rfc7539_create, 459 .module = THIS_MODULE, 460 }, { 461 .name = "rfc7539esp", 462 .create = rfc7539esp_create, 463 .module = THIS_MODULE, 464 }, 465 }; 466 467 static int __init chacha20poly1305_module_init(void) 468 { 469 return crypto_register_templates(rfc7539_tmpls, 470 ARRAY_SIZE(rfc7539_tmpls)); 471 } 472 473 static void __exit chacha20poly1305_module_exit(void) 474 { 475 crypto_unregister_templates(rfc7539_tmpls, 476 ARRAY_SIZE(rfc7539_tmpls)); 477 } 478 479 module_init(chacha20poly1305_module_init); 480 module_exit(chacha20poly1305_module_exit); 481 482 MODULE_LICENSE("GPL"); 483 MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); 484 MODULE_DESCRIPTION("ChaCha20-Poly1305 AEAD"); 485 MODULE_ALIAS_CRYPTO("rfc7539"); 486 MODULE_ALIAS_CRYPTO("rfc7539esp"); 487