1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Asynchronous Compression operations 4 * 5 * Copyright (c) 2016, Intel Corporation 6 * Authors: Weigang Li <weigang.li@intel.com> 7 * Giovanni Cabiddu <giovanni.cabiddu@intel.com> 8 */ 9 10 #include <crypto/internal/acompress.h> 11 #include <crypto/scatterwalk.h> 12 #include <linux/cryptouser.h> 13 #include <linux/cpumask.h> 14 #include <linux/err.h> 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/percpu.h> 18 #include <linux/scatterlist.h> 19 #include <linux/sched.h> 20 #include <linux/seq_file.h> 21 #include <linux/smp.h> 22 #include <linux/spinlock.h> 23 #include <linux/string.h> 24 #include <linux/workqueue.h> 25 #include <net/netlink.h> 26 27 #include "compress.h" 28 29 struct crypto_scomp; 30 31 enum { 32 ACOMP_WALK_SLEEP = 1 << 0, 33 ACOMP_WALK_SRC_LINEAR = 1 << 1, 34 ACOMP_WALK_DST_LINEAR = 1 << 2, 35 }; 36 37 static const struct crypto_type crypto_acomp_type; 38 39 static void acomp_reqchain_done(void *data, int err); 40 41 static inline struct acomp_alg *__crypto_acomp_alg(struct crypto_alg *alg) 42 { 43 return container_of(alg, struct acomp_alg, calg.base); 44 } 45 46 static inline struct acomp_alg *crypto_acomp_alg(struct crypto_acomp *tfm) 47 { 48 return __crypto_acomp_alg(crypto_acomp_tfm(tfm)->__crt_alg); 49 } 50 51 static int __maybe_unused crypto_acomp_report( 52 struct sk_buff *skb, struct crypto_alg *alg) 53 { 54 struct crypto_report_acomp racomp; 55 56 memset(&racomp, 0, sizeof(racomp)); 57 58 strscpy(racomp.type, "acomp", sizeof(racomp.type)); 59 60 return nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, sizeof(racomp), &racomp); 61 } 62 63 static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg) 64 __maybe_unused; 65 66 static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg) 67 { 68 seq_puts(m, "type : acomp\n"); 69 } 70 71 static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm) 72 { 73 struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm); 74 struct acomp_alg *alg = crypto_acomp_alg(acomp); 75 76 if (alg->exit) 77 alg->exit(acomp); 78 79 if (acomp_is_async(acomp)) 80 crypto_free_acomp(crypto_acomp_fb(acomp)); 81 } 82 83 static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) 84 { 85 struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm); 86 struct acomp_alg *alg = crypto_acomp_alg(acomp); 87 struct crypto_acomp *fb = NULL; 88 int err; 89 90 if (tfm->__crt_alg->cra_type != &crypto_acomp_type) 91 return crypto_init_scomp_ops_async(tfm); 92 93 if (acomp_is_async(acomp)) { 94 fb = crypto_alloc_acomp(crypto_acomp_alg_name(acomp), 0, 95 CRYPTO_ALG_ASYNC); 96 if (IS_ERR(fb)) 97 return PTR_ERR(fb); 98 99 err = -EINVAL; 100 if (crypto_acomp_reqsize(fb) > MAX_SYNC_COMP_REQSIZE) 101 goto out_free_fb; 102 103 tfm->fb = crypto_acomp_tfm(fb); 104 } 105 106 acomp->compress = alg->compress; 107 acomp->decompress = alg->decompress; 108 acomp->reqsize = alg->base.cra_reqsize; 109 110 acomp->base.exit = crypto_acomp_exit_tfm; 111 112 if (!alg->init) 113 return 0; 114 115 err = alg->init(acomp); 116 if (err) 117 goto out_free_fb; 118 119 return 0; 120 121 out_free_fb: 122 crypto_free_acomp(fb); 123 return err; 124 } 125 126 static unsigned int crypto_acomp_extsize(struct crypto_alg *alg) 127 { 128 int extsize = crypto_alg_extsize(alg); 129 130 if (alg->cra_type != &crypto_acomp_type) 131 extsize += sizeof(struct crypto_scomp *); 132 133 return extsize; 134 } 135 136 static const struct crypto_type crypto_acomp_type = { 137 .extsize = crypto_acomp_extsize, 138 .init_tfm = crypto_acomp_init_tfm, 139 #ifdef CONFIG_PROC_FS 140 .show = crypto_acomp_show, 141 #endif 142 #if IS_ENABLED(CONFIG_CRYPTO_USER) 143 .report = crypto_acomp_report, 144 #endif 145 .maskclear = ~CRYPTO_ALG_TYPE_MASK, 146 .maskset = CRYPTO_ALG_TYPE_ACOMPRESS_MASK, 147 .type = CRYPTO_ALG_TYPE_ACOMPRESS, 148 .tfmsize = offsetof(struct crypto_acomp, base), 149 .algsize = offsetof(struct acomp_alg, base), 150 }; 151 152 struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type, 153 u32 mask) 154 { 155 return crypto_alloc_tfm(alg_name, &crypto_acomp_type, type, mask); 156 } 157 EXPORT_SYMBOL_GPL(crypto_alloc_acomp); 158 159 struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type, 160 u32 mask, int node) 161 { 162 return crypto_alloc_tfm_node(alg_name, &crypto_acomp_type, type, mask, 163 node); 164 } 165 EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node); 166 167 static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt) 168 { 169 struct acomp_req_chain *state = &req->chain; 170 171 state->compl = req->base.complete; 172 state->data = req->base.data; 173 req->base.complete = cplt; 174 req->base.data = state; 175 } 176 177 static void acomp_restore_req(struct acomp_req *req) 178 { 179 struct acomp_req_chain *state = req->base.data; 180 181 req->base.complete = state->compl; 182 req->base.data = state->data; 183 } 184 185 static void acomp_reqchain_virt(struct acomp_req *req) 186 { 187 struct acomp_req_chain *state = &req->chain; 188 unsigned int slen = req->slen; 189 unsigned int dlen = req->dlen; 190 191 if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT) 192 acomp_request_set_src_dma(req, state->src, slen); 193 if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT) 194 acomp_request_set_dst_dma(req, state->dst, dlen); 195 } 196 197 static void acomp_virt_to_sg(struct acomp_req *req) 198 { 199 struct acomp_req_chain *state = &req->chain; 200 201 state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | 202 CRYPTO_ACOMP_REQ_DST_VIRT); 203 204 if (acomp_request_src_isvirt(req)) { 205 unsigned int slen = req->slen; 206 const u8 *svirt = req->svirt; 207 208 state->src = svirt; 209 sg_init_one(&state->ssg, svirt, slen); 210 acomp_request_set_src_sg(req, &state->ssg, slen); 211 } 212 213 if (acomp_request_dst_isvirt(req)) { 214 unsigned int dlen = req->dlen; 215 u8 *dvirt = req->dvirt; 216 217 state->dst = dvirt; 218 sg_init_one(&state->dsg, dvirt, dlen); 219 acomp_request_set_dst_sg(req, &state->dsg, dlen); 220 } 221 } 222 223 static int acomp_do_nondma(struct acomp_req *req, bool comp) 224 { 225 ACOMP_FBREQ_ON_STACK(fbreq, req); 226 int err; 227 228 if (comp) 229 err = crypto_acomp_compress(fbreq); 230 else 231 err = crypto_acomp_decompress(fbreq); 232 233 req->dlen = fbreq->dlen; 234 return err; 235 } 236 237 static int acomp_do_one_req(struct acomp_req *req, bool comp) 238 { 239 if (acomp_request_isnondma(req)) 240 return acomp_do_nondma(req, comp); 241 242 acomp_virt_to_sg(req); 243 return comp ? crypto_acomp_reqtfm(req)->compress(req) : 244 crypto_acomp_reqtfm(req)->decompress(req); 245 } 246 247 static int acomp_reqchain_finish(struct acomp_req *req, int err) 248 { 249 acomp_reqchain_virt(req); 250 acomp_restore_req(req); 251 return err; 252 } 253 254 static void acomp_reqchain_done(void *data, int err) 255 { 256 struct acomp_req *req = data; 257 crypto_completion_t compl; 258 259 compl = req->chain.compl; 260 data = req->chain.data; 261 262 if (err == -EINPROGRESS) 263 goto notify; 264 265 err = acomp_reqchain_finish(req, err); 266 267 notify: 268 compl(data, err); 269 } 270 271 static int acomp_do_req_chain(struct acomp_req *req, bool comp) 272 { 273 int err; 274 275 acomp_save_req(req, acomp_reqchain_done); 276 277 err = acomp_do_one_req(req, comp); 278 if (err == -EBUSY || err == -EINPROGRESS) 279 return err; 280 281 return acomp_reqchain_finish(req, err); 282 } 283 284 int crypto_acomp_compress(struct acomp_req *req) 285 { 286 struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); 287 288 if (acomp_req_on_stack(req) && acomp_is_async(tfm)) 289 return -EAGAIN; 290 if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req)) 291 return crypto_acomp_reqtfm(req)->compress(req); 292 return acomp_do_req_chain(req, true); 293 } 294 EXPORT_SYMBOL_GPL(crypto_acomp_compress); 295 296 int crypto_acomp_decompress(struct acomp_req *req) 297 { 298 struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); 299 300 if (acomp_req_on_stack(req) && acomp_is_async(tfm)) 301 return -EAGAIN; 302 if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req)) 303 return crypto_acomp_reqtfm(req)->decompress(req); 304 return acomp_do_req_chain(req, false); 305 } 306 EXPORT_SYMBOL_GPL(crypto_acomp_decompress); 307 308 void comp_prepare_alg(struct comp_alg_common *alg) 309 { 310 struct crypto_alg *base = &alg->base; 311 312 base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; 313 } 314 315 int crypto_register_acomp(struct acomp_alg *alg) 316 { 317 struct crypto_alg *base = &alg->calg.base; 318 319 comp_prepare_alg(&alg->calg); 320 321 base->cra_type = &crypto_acomp_type; 322 base->cra_flags |= CRYPTO_ALG_TYPE_ACOMPRESS; 323 324 return crypto_register_alg(base); 325 } 326 EXPORT_SYMBOL_GPL(crypto_register_acomp); 327 328 void crypto_unregister_acomp(struct acomp_alg *alg) 329 { 330 crypto_unregister_alg(&alg->base); 331 } 332 EXPORT_SYMBOL_GPL(crypto_unregister_acomp); 333 334 int crypto_register_acomps(struct acomp_alg *algs, int count) 335 { 336 int i, ret; 337 338 for (i = 0; i < count; i++) { 339 ret = crypto_register_acomp(&algs[i]); 340 if (ret) 341 goto err; 342 } 343 344 return 0; 345 346 err: 347 for (--i; i >= 0; --i) 348 crypto_unregister_acomp(&algs[i]); 349 350 return ret; 351 } 352 EXPORT_SYMBOL_GPL(crypto_register_acomps); 353 354 void crypto_unregister_acomps(struct acomp_alg *algs, int count) 355 { 356 int i; 357 358 for (i = count - 1; i >= 0; --i) 359 crypto_unregister_acomp(&algs[i]); 360 } 361 EXPORT_SYMBOL_GPL(crypto_unregister_acomps); 362 363 static void acomp_stream_workfn(struct work_struct *work) 364 { 365 struct crypto_acomp_streams *s = 366 container_of(work, struct crypto_acomp_streams, stream_work); 367 struct crypto_acomp_stream __percpu *streams = s->streams; 368 int cpu; 369 370 for_each_cpu(cpu, &s->stream_want) { 371 struct crypto_acomp_stream *ps; 372 void *ctx; 373 374 ps = per_cpu_ptr(streams, cpu); 375 if (ps->ctx) 376 continue; 377 378 ctx = s->alloc_ctx(); 379 if (IS_ERR(ctx)) 380 break; 381 382 spin_lock_bh(&ps->lock); 383 ps->ctx = ctx; 384 spin_unlock_bh(&ps->lock); 385 386 cpumask_clear_cpu(cpu, &s->stream_want); 387 } 388 } 389 390 void crypto_acomp_free_streams(struct crypto_acomp_streams *s) 391 { 392 struct crypto_acomp_stream __percpu *streams = s->streams; 393 void (*free_ctx)(void *); 394 int i; 395 396 s->streams = NULL; 397 if (!streams) 398 return; 399 400 cancel_work_sync(&s->stream_work); 401 free_ctx = s->free_ctx; 402 403 for_each_possible_cpu(i) { 404 struct crypto_acomp_stream *ps = per_cpu_ptr(streams, i); 405 406 if (!ps->ctx) 407 continue; 408 409 free_ctx(ps->ctx); 410 } 411 412 free_percpu(streams); 413 } 414 EXPORT_SYMBOL_GPL(crypto_acomp_free_streams); 415 416 int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s) 417 { 418 struct crypto_acomp_stream __percpu *streams; 419 struct crypto_acomp_stream *ps; 420 unsigned int i; 421 void *ctx; 422 423 if (s->streams) 424 return 0; 425 426 streams = alloc_percpu(struct crypto_acomp_stream); 427 if (!streams) 428 return -ENOMEM; 429 430 ctx = s->alloc_ctx(); 431 if (IS_ERR(ctx)) { 432 free_percpu(streams); 433 return PTR_ERR(ctx); 434 } 435 436 i = cpumask_first(cpu_possible_mask); 437 ps = per_cpu_ptr(streams, i); 438 ps->ctx = ctx; 439 440 for_each_possible_cpu(i) { 441 ps = per_cpu_ptr(streams, i); 442 spin_lock_init(&ps->lock); 443 } 444 445 s->streams = streams; 446 447 INIT_WORK(&s->stream_work, acomp_stream_workfn); 448 return 0; 449 } 450 EXPORT_SYMBOL_GPL(crypto_acomp_alloc_streams); 451 452 struct crypto_acomp_stream *crypto_acomp_lock_stream_bh( 453 struct crypto_acomp_streams *s) __acquires(stream) 454 { 455 struct crypto_acomp_stream __percpu *streams = s->streams; 456 int cpu = raw_smp_processor_id(); 457 struct crypto_acomp_stream *ps; 458 459 ps = per_cpu_ptr(streams, cpu); 460 spin_lock_bh(&ps->lock); 461 if (likely(ps->ctx)) 462 return ps; 463 spin_unlock(&ps->lock); 464 465 cpumask_set_cpu(cpu, &s->stream_want); 466 schedule_work(&s->stream_work); 467 468 ps = per_cpu_ptr(streams, cpumask_first(cpu_possible_mask)); 469 spin_lock(&ps->lock); 470 return ps; 471 } 472 EXPORT_SYMBOL_GPL(crypto_acomp_lock_stream_bh); 473 474 void acomp_walk_done_src(struct acomp_walk *walk, int used) 475 { 476 walk->slen -= used; 477 if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) 478 scatterwalk_advance(&walk->in, used); 479 else 480 scatterwalk_done_src(&walk->in, used); 481 482 if ((walk->flags & ACOMP_WALK_SLEEP)) 483 cond_resched(); 484 } 485 EXPORT_SYMBOL_GPL(acomp_walk_done_src); 486 487 void acomp_walk_done_dst(struct acomp_walk *walk, int used) 488 { 489 walk->dlen -= used; 490 if ((walk->flags & ACOMP_WALK_DST_LINEAR)) 491 scatterwalk_advance(&walk->out, used); 492 else 493 scatterwalk_done_dst(&walk->out, used); 494 495 if ((walk->flags & ACOMP_WALK_SLEEP)) 496 cond_resched(); 497 } 498 EXPORT_SYMBOL_GPL(acomp_walk_done_dst); 499 500 int acomp_walk_next_src(struct acomp_walk *walk) 501 { 502 unsigned int slen = walk->slen; 503 unsigned int max = UINT_MAX; 504 505 if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP)) 506 max = PAGE_SIZE; 507 if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) { 508 walk->in.__addr = (void *)(((u8 *)walk->in.sg) + 509 walk->in.offset); 510 return min(slen, max); 511 } 512 513 return slen ? scatterwalk_next(&walk->in, slen) : 0; 514 } 515 EXPORT_SYMBOL_GPL(acomp_walk_next_src); 516 517 int acomp_walk_next_dst(struct acomp_walk *walk) 518 { 519 unsigned int dlen = walk->dlen; 520 unsigned int max = UINT_MAX; 521 522 if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP)) 523 max = PAGE_SIZE; 524 if ((walk->flags & ACOMP_WALK_DST_LINEAR)) { 525 walk->out.__addr = (void *)(((u8 *)walk->out.sg) + 526 walk->out.offset); 527 return min(dlen, max); 528 } 529 530 return dlen ? scatterwalk_next(&walk->out, dlen) : 0; 531 } 532 EXPORT_SYMBOL_GPL(acomp_walk_next_dst); 533 534 int acomp_walk_virt(struct acomp_walk *__restrict walk, 535 struct acomp_req *__restrict req, bool atomic) 536 { 537 struct scatterlist *src = req->src; 538 struct scatterlist *dst = req->dst; 539 540 walk->slen = req->slen; 541 walk->dlen = req->dlen; 542 543 if (!walk->slen || !walk->dlen) 544 return -EINVAL; 545 546 walk->flags = 0; 547 if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic) 548 walk->flags |= ACOMP_WALK_SLEEP; 549 if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT)) 550 walk->flags |= ACOMP_WALK_SRC_LINEAR; 551 if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT)) 552 walk->flags |= ACOMP_WALK_DST_LINEAR; 553 554 if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) { 555 walk->in.sg = (void *)req->svirt; 556 walk->in.offset = 0; 557 } else 558 scatterwalk_start(&walk->in, src); 559 if ((walk->flags & ACOMP_WALK_DST_LINEAR)) { 560 walk->out.sg = (void *)req->dvirt; 561 walk->out.offset = 0; 562 } else 563 scatterwalk_start(&walk->out, dst); 564 565 return 0; 566 } 567 EXPORT_SYMBOL_GPL(acomp_walk_virt); 568 569 struct acomp_req *acomp_request_clone(struct acomp_req *req, 570 size_t total, gfp_t gfp) 571 { 572 struct acomp_req *nreq; 573 574 nreq = container_of(crypto_request_clone(&req->base, total, gfp), 575 struct acomp_req, base); 576 if (nreq == req) 577 return req; 578 579 if (req->src == &req->chain.ssg) 580 nreq->src = &nreq->chain.ssg; 581 if (req->dst == &req->chain.dsg) 582 nreq->dst = &nreq->chain.dsg; 583 return nreq; 584 } 585 EXPORT_SYMBOL_GPL(acomp_request_clone); 586 587 MODULE_LICENSE("GPL"); 588 MODULE_DESCRIPTION("Asynchronous compression type"); 589