1 // SPDX-License-Identifier: GPL-2.0 2 #include "bcachefs.h" 3 #include "checksum.h" 4 #include "errcode.h" 5 #include "error.h" 6 #include "super.h" 7 #include "super-io.h" 8 9 #include <linux/crc32c.h> 10 #include <linux/xxhash.h> 11 #include <linux/key.h> 12 #include <linux/random.h> 13 #include <linux/ratelimit.h> 14 #include <crypto/chacha.h> 15 #include <crypto/poly1305.h> 16 #include <keys/user-type.h> 17 18 /* 19 * bch2_checksum state is an abstraction of the checksum state calculated over different pages. 20 * it features page merging without having the checksum algorithm lose its state. 21 * for native checksum aglorithms (like crc), a default seed value will do. 22 * for hash-like algorithms, a state needs to be stored 23 */ 24 25 struct bch2_checksum_state { 26 union { 27 u64 seed; 28 struct xxh64_state h64state; 29 }; 30 unsigned int type; 31 }; 32 33 static void bch2_checksum_init(struct bch2_checksum_state *state) 34 { 35 switch (state->type) { 36 case BCH_CSUM_none: 37 case BCH_CSUM_crc32c: 38 case BCH_CSUM_crc64: 39 state->seed = 0; 40 break; 41 case BCH_CSUM_crc32c_nonzero: 42 state->seed = U32_MAX; 43 break; 44 case BCH_CSUM_crc64_nonzero: 45 state->seed = U64_MAX; 46 break; 47 case BCH_CSUM_xxhash: 48 xxh64_reset(&state->h64state, 0); 49 break; 50 default: 51 BUG(); 52 } 53 } 54 55 static u64 bch2_checksum_final(const struct bch2_checksum_state *state) 56 { 57 switch (state->type) { 58 case BCH_CSUM_none: 59 case BCH_CSUM_crc32c: 60 case BCH_CSUM_crc64: 61 return state->seed; 62 case BCH_CSUM_crc32c_nonzero: 63 return state->seed ^ U32_MAX; 64 case BCH_CSUM_crc64_nonzero: 65 return state->seed ^ U64_MAX; 66 case BCH_CSUM_xxhash: 67 return xxh64_digest(&state->h64state); 68 default: 69 BUG(); 70 } 71 } 72 73 static void bch2_checksum_update(struct bch2_checksum_state *state, const void *data, size_t len) 74 { 75 switch (state->type) { 76 case BCH_CSUM_none: 77 return; 78 case BCH_CSUM_crc32c_nonzero: 79 case BCH_CSUM_crc32c: 80 state->seed = crc32c(state->seed, data, len); 81 break; 82 case BCH_CSUM_crc64_nonzero: 83 case BCH_CSUM_crc64: 84 state->seed = crc64_be(state->seed, data, len); 85 break; 86 case BCH_CSUM_xxhash: 87 xxh64_update(&state->h64state, data, len); 88 break; 89 default: 90 BUG(); 91 } 92 } 93 94 static void bch2_chacha20_init(u32 state[CHACHA_STATE_WORDS], 95 const struct bch_key *key, struct nonce nonce) 96 { 97 u32 key_words[CHACHA_KEY_SIZE / sizeof(u32)]; 98 99 BUILD_BUG_ON(sizeof(key_words) != sizeof(*key)); 100 memcpy(key_words, key, sizeof(key_words)); 101 le32_to_cpu_array(key_words, ARRAY_SIZE(key_words)); 102 103 BUILD_BUG_ON(sizeof(nonce) != CHACHA_IV_SIZE); 104 chacha_init(state, key_words, (const u8 *)nonce.d); 105 106 memzero_explicit(key_words, sizeof(key_words)); 107 } 108 109 static void bch2_chacha20(const struct bch_key *key, struct nonce nonce, 110 void *data, size_t len) 111 { 112 u32 state[CHACHA_STATE_WORDS]; 113 114 bch2_chacha20_init(state, key, nonce); 115 chacha20_crypt(state, data, data, len); 116 memzero_explicit(state, sizeof(state)); 117 } 118 119 static void bch2_poly1305_init(struct poly1305_desc_ctx *desc, 120 struct bch_fs *c, struct nonce nonce) 121 { 122 u8 key[POLY1305_KEY_SIZE] = { 0 }; 123 124 nonce.d[3] ^= BCH_NONCE_POLY; 125 126 bch2_chacha20(&c->chacha20_key, nonce, key, sizeof(key)); 127 poly1305_init(desc, key); 128 } 129 130 struct bch_csum bch2_checksum(struct bch_fs *c, unsigned type, 131 struct nonce nonce, const void *data, size_t len) 132 { 133 switch (type) { 134 case BCH_CSUM_none: 135 case BCH_CSUM_crc32c_nonzero: 136 case BCH_CSUM_crc64_nonzero: 137 case BCH_CSUM_crc32c: 138 case BCH_CSUM_xxhash: 139 case BCH_CSUM_crc64: { 140 struct bch2_checksum_state state; 141 142 state.type = type; 143 144 bch2_checksum_init(&state); 145 bch2_checksum_update(&state, data, len); 146 147 return (struct bch_csum) { .lo = cpu_to_le64(bch2_checksum_final(&state)) }; 148 } 149 150 case BCH_CSUM_chacha20_poly1305_80: 151 case BCH_CSUM_chacha20_poly1305_128: { 152 struct poly1305_desc_ctx dctx; 153 u8 digest[POLY1305_DIGEST_SIZE]; 154 struct bch_csum ret = { 0 }; 155 156 bch2_poly1305_init(&dctx, c, nonce); 157 poly1305_update(&dctx, data, len); 158 poly1305_final(&dctx, digest); 159 160 memcpy(&ret, digest, bch_crc_bytes[type]); 161 return ret; 162 } 163 default: 164 return (struct bch_csum) {}; 165 } 166 } 167 168 int bch2_encrypt(struct bch_fs *c, unsigned type, 169 struct nonce nonce, void *data, size_t len) 170 { 171 if (!bch2_csum_type_is_encryption(type)) 172 return 0; 173 174 if (bch2_fs_inconsistent_on(!c->chacha20_key_set, 175 c, "attempting to encrypt without encryption key")) 176 return -BCH_ERR_no_encryption_key; 177 178 bch2_chacha20(&c->chacha20_key, nonce, data, len); 179 return 0; 180 } 181 182 static struct bch_csum __bch2_checksum_bio(struct bch_fs *c, unsigned type, 183 struct nonce nonce, struct bio *bio, 184 struct bvec_iter *iter) 185 { 186 struct bio_vec bv; 187 188 switch (type) { 189 case BCH_CSUM_none: 190 return (struct bch_csum) { 0 }; 191 case BCH_CSUM_crc32c_nonzero: 192 case BCH_CSUM_crc64_nonzero: 193 case BCH_CSUM_crc32c: 194 case BCH_CSUM_xxhash: 195 case BCH_CSUM_crc64: { 196 struct bch2_checksum_state state; 197 198 state.type = type; 199 bch2_checksum_init(&state); 200 201 #ifdef CONFIG_HIGHMEM 202 __bio_for_each_segment(bv, bio, *iter, *iter) { 203 void *p = kmap_local_page(bv.bv_page) + bv.bv_offset; 204 205 bch2_checksum_update(&state, p, bv.bv_len); 206 kunmap_local(p); 207 } 208 #else 209 __bio_for_each_bvec(bv, bio, *iter, *iter) 210 bch2_checksum_update(&state, page_address(bv.bv_page) + bv.bv_offset, 211 bv.bv_len); 212 #endif 213 return (struct bch_csum) { .lo = cpu_to_le64(bch2_checksum_final(&state)) }; 214 } 215 216 case BCH_CSUM_chacha20_poly1305_80: 217 case BCH_CSUM_chacha20_poly1305_128: { 218 struct poly1305_desc_ctx dctx; 219 u8 digest[POLY1305_DIGEST_SIZE]; 220 struct bch_csum ret = { 0 }; 221 222 bch2_poly1305_init(&dctx, c, nonce); 223 224 #ifdef CONFIG_HIGHMEM 225 __bio_for_each_segment(bv, bio, *iter, *iter) { 226 void *p = kmap_local_page(bv.bv_page) + bv.bv_offset; 227 228 poly1305_update(&dctx, p, bv.bv_len); 229 kunmap_local(p); 230 } 231 #else 232 __bio_for_each_bvec(bv, bio, *iter, *iter) 233 poly1305_update(&dctx, 234 page_address(bv.bv_page) + bv.bv_offset, 235 bv.bv_len); 236 #endif 237 poly1305_final(&dctx, digest); 238 239 memcpy(&ret, digest, bch_crc_bytes[type]); 240 return ret; 241 } 242 default: 243 return (struct bch_csum) {}; 244 } 245 } 246 247 struct bch_csum bch2_checksum_bio(struct bch_fs *c, unsigned type, 248 struct nonce nonce, struct bio *bio) 249 { 250 struct bvec_iter iter = bio->bi_iter; 251 252 return __bch2_checksum_bio(c, type, nonce, bio, &iter); 253 } 254 255 int __bch2_encrypt_bio(struct bch_fs *c, unsigned type, 256 struct nonce nonce, struct bio *bio) 257 { 258 struct bio_vec bv; 259 struct bvec_iter iter; 260 u32 chacha_state[CHACHA_STATE_WORDS]; 261 int ret = 0; 262 263 if (bch2_fs_inconsistent_on(!c->chacha20_key_set, 264 c, "attempting to encrypt without encryption key")) 265 return -BCH_ERR_no_encryption_key; 266 267 bch2_chacha20_init(chacha_state, &c->chacha20_key, nonce); 268 269 bio_for_each_segment(bv, bio, iter) { 270 void *p; 271 272 /* 273 * chacha_crypt() assumes that the length is a multiple of 274 * CHACHA_BLOCK_SIZE on any non-final call. 275 */ 276 if (!IS_ALIGNED(bv.bv_len, CHACHA_BLOCK_SIZE)) { 277 bch_err_ratelimited(c, "bio not aligned for encryption"); 278 ret = -EIO; 279 break; 280 } 281 282 p = bvec_kmap_local(&bv); 283 chacha20_crypt(chacha_state, p, p, bv.bv_len); 284 kunmap_local(p); 285 } 286 memzero_explicit(chacha_state, sizeof(chacha_state)); 287 return ret; 288 } 289 290 struct bch_csum bch2_checksum_merge(unsigned type, struct bch_csum a, 291 struct bch_csum b, size_t b_len) 292 { 293 struct bch2_checksum_state state; 294 295 state.type = type; 296 bch2_checksum_init(&state); 297 state.seed = le64_to_cpu(a.lo); 298 299 BUG_ON(!bch2_checksum_mergeable(type)); 300 301 while (b_len) { 302 unsigned page_len = min_t(unsigned, b_len, PAGE_SIZE); 303 304 bch2_checksum_update(&state, 305 page_address(ZERO_PAGE(0)), page_len); 306 b_len -= page_len; 307 } 308 a.lo = cpu_to_le64(bch2_checksum_final(&state)); 309 a.lo ^= b.lo; 310 a.hi ^= b.hi; 311 return a; 312 } 313 314 int bch2_rechecksum_bio(struct bch_fs *c, struct bio *bio, 315 struct bversion version, 316 struct bch_extent_crc_unpacked crc_old, 317 struct bch_extent_crc_unpacked *crc_a, 318 struct bch_extent_crc_unpacked *crc_b, 319 unsigned len_a, unsigned len_b, 320 unsigned new_csum_type) 321 { 322 struct bvec_iter iter = bio->bi_iter; 323 struct nonce nonce = extent_nonce(version, crc_old); 324 struct bch_csum merged = { 0 }; 325 struct crc_split { 326 struct bch_extent_crc_unpacked *crc; 327 unsigned len; 328 unsigned csum_type; 329 struct bch_csum csum; 330 } splits[3] = { 331 { crc_a, len_a, new_csum_type, { 0 }}, 332 { crc_b, len_b, new_csum_type, { 0 } }, 333 { NULL, bio_sectors(bio) - len_a - len_b, new_csum_type, { 0 } }, 334 }, *i; 335 bool mergeable = crc_old.csum_type == new_csum_type && 336 bch2_checksum_mergeable(new_csum_type); 337 unsigned crc_nonce = crc_old.nonce; 338 339 BUG_ON(len_a + len_b > bio_sectors(bio)); 340 BUG_ON(crc_old.uncompressed_size != bio_sectors(bio)); 341 BUG_ON(crc_is_compressed(crc_old)); 342 BUG_ON(bch2_csum_type_is_encryption(crc_old.csum_type) != 343 bch2_csum_type_is_encryption(new_csum_type)); 344 345 for (i = splits; i < splits + ARRAY_SIZE(splits); i++) { 346 iter.bi_size = i->len << 9; 347 if (mergeable || i->crc) 348 i->csum = __bch2_checksum_bio(c, i->csum_type, 349 nonce, bio, &iter); 350 else 351 bio_advance_iter(bio, &iter, i->len << 9); 352 nonce = nonce_add(nonce, i->len << 9); 353 } 354 355 if (mergeable) 356 for (i = splits; i < splits + ARRAY_SIZE(splits); i++) 357 merged = bch2_checksum_merge(new_csum_type, merged, 358 i->csum, i->len << 9); 359 else 360 merged = bch2_checksum_bio(c, crc_old.csum_type, 361 extent_nonce(version, crc_old), bio); 362 363 if (bch2_crc_cmp(merged, crc_old.csum) && !c->opts.no_data_io) { 364 struct printbuf buf = PRINTBUF; 365 prt_printf(&buf, "checksum error in %s() (memory corruption or bug?)\n" 366 " expected %0llx:%0llx got %0llx:%0llx (old type ", 367 __func__, 368 crc_old.csum.hi, 369 crc_old.csum.lo, 370 merged.hi, 371 merged.lo); 372 bch2_prt_csum_type(&buf, crc_old.csum_type); 373 prt_str(&buf, " new type "); 374 bch2_prt_csum_type(&buf, new_csum_type); 375 prt_str(&buf, ")"); 376 WARN_RATELIMIT(1, "%s", buf.buf); 377 printbuf_exit(&buf); 378 return -BCH_ERR_recompute_checksum; 379 } 380 381 for (i = splits; i < splits + ARRAY_SIZE(splits); i++) { 382 if (i->crc) 383 *i->crc = (struct bch_extent_crc_unpacked) { 384 .csum_type = i->csum_type, 385 .compression_type = crc_old.compression_type, 386 .compressed_size = i->len, 387 .uncompressed_size = i->len, 388 .offset = 0, 389 .live_size = i->len, 390 .nonce = crc_nonce, 391 .csum = i->csum, 392 }; 393 394 if (bch2_csum_type_is_encryption(new_csum_type)) 395 crc_nonce += i->len; 396 } 397 398 return 0; 399 } 400 401 /* BCH_SB_FIELD_crypt: */ 402 403 static int bch2_sb_crypt_validate(struct bch_sb *sb, struct bch_sb_field *f, 404 enum bch_validate_flags flags, struct printbuf *err) 405 { 406 struct bch_sb_field_crypt *crypt = field_to_type(f, crypt); 407 408 if (vstruct_bytes(&crypt->field) < sizeof(*crypt)) { 409 prt_printf(err, "wrong size (got %zu should be %zu)", 410 vstruct_bytes(&crypt->field), sizeof(*crypt)); 411 return -BCH_ERR_invalid_sb_crypt; 412 } 413 414 if (BCH_CRYPT_KDF_TYPE(crypt)) { 415 prt_printf(err, "bad kdf type %llu", BCH_CRYPT_KDF_TYPE(crypt)); 416 return -BCH_ERR_invalid_sb_crypt; 417 } 418 419 return 0; 420 } 421 422 static void bch2_sb_crypt_to_text(struct printbuf *out, struct bch_sb *sb, 423 struct bch_sb_field *f) 424 { 425 struct bch_sb_field_crypt *crypt = field_to_type(f, crypt); 426 427 prt_printf(out, "KFD: %llu\n", BCH_CRYPT_KDF_TYPE(crypt)); 428 prt_printf(out, "scrypt n: %llu\n", BCH_KDF_SCRYPT_N(crypt)); 429 prt_printf(out, "scrypt r: %llu\n", BCH_KDF_SCRYPT_R(crypt)); 430 prt_printf(out, "scrypt p: %llu\n", BCH_KDF_SCRYPT_P(crypt)); 431 } 432 433 const struct bch_sb_field_ops bch_sb_field_ops_crypt = { 434 .validate = bch2_sb_crypt_validate, 435 .to_text = bch2_sb_crypt_to_text, 436 }; 437 438 #ifdef __KERNEL__ 439 static int __bch2_request_key(char *key_description, struct bch_key *key) 440 { 441 struct key *keyring_key; 442 const struct user_key_payload *ukp; 443 int ret; 444 445 keyring_key = request_key(&key_type_user, key_description, NULL); 446 if (IS_ERR(keyring_key)) 447 return PTR_ERR(keyring_key); 448 449 down_read(&keyring_key->sem); 450 ukp = dereference_key_locked(keyring_key); 451 if (ukp->datalen == sizeof(*key)) { 452 memcpy(key, ukp->data, ukp->datalen); 453 ret = 0; 454 } else { 455 ret = -EINVAL; 456 } 457 up_read(&keyring_key->sem); 458 key_put(keyring_key); 459 460 return ret; 461 } 462 #else 463 #include <keyutils.h> 464 465 static int __bch2_request_key(char *key_description, struct bch_key *key) 466 { 467 key_serial_t key_id; 468 469 key_id = request_key("user", key_description, NULL, 470 KEY_SPEC_SESSION_KEYRING); 471 if (key_id >= 0) 472 goto got_key; 473 474 key_id = request_key("user", key_description, NULL, 475 KEY_SPEC_USER_KEYRING); 476 if (key_id >= 0) 477 goto got_key; 478 479 key_id = request_key("user", key_description, NULL, 480 KEY_SPEC_USER_SESSION_KEYRING); 481 if (key_id >= 0) 482 goto got_key; 483 484 return -errno; 485 got_key: 486 487 if (keyctl_read(key_id, (void *) key, sizeof(*key)) != sizeof(*key)) 488 return -1; 489 490 return 0; 491 } 492 493 #include "crypto.h" 494 #endif 495 496 int bch2_request_key(struct bch_sb *sb, struct bch_key *key) 497 { 498 struct printbuf key_description = PRINTBUF; 499 int ret; 500 501 prt_printf(&key_description, "bcachefs:"); 502 pr_uuid(&key_description, sb->user_uuid.b); 503 504 ret = __bch2_request_key(key_description.buf, key); 505 printbuf_exit(&key_description); 506 507 #ifndef __KERNEL__ 508 if (ret) { 509 char *passphrase = read_passphrase("Enter passphrase: "); 510 struct bch_encrypted_key sb_key; 511 512 bch2_passphrase_check(sb, passphrase, 513 key, &sb_key); 514 ret = 0; 515 } 516 #endif 517 518 /* stash with memfd, pass memfd fd to mount */ 519 520 return ret; 521 } 522 523 #ifndef __KERNEL__ 524 int bch2_revoke_key(struct bch_sb *sb) 525 { 526 key_serial_t key_id; 527 struct printbuf key_description = PRINTBUF; 528 529 prt_printf(&key_description, "bcachefs:"); 530 pr_uuid(&key_description, sb->user_uuid.b); 531 532 key_id = request_key("user", key_description.buf, NULL, KEY_SPEC_USER_KEYRING); 533 printbuf_exit(&key_description); 534 if (key_id < 0) 535 return errno; 536 537 keyctl_revoke(key_id); 538 539 return 0; 540 } 541 #endif 542 543 int bch2_decrypt_sb_key(struct bch_fs *c, 544 struct bch_sb_field_crypt *crypt, 545 struct bch_key *key) 546 { 547 struct bch_encrypted_key sb_key = crypt->key; 548 struct bch_key user_key; 549 int ret = 0; 550 551 /* is key encrypted? */ 552 if (!bch2_key_is_encrypted(&sb_key)) 553 goto out; 554 555 ret = bch2_request_key(c->disk_sb.sb, &user_key); 556 if (ret) { 557 bch_err(c, "error requesting encryption key: %s", bch2_err_str(ret)); 558 goto err; 559 } 560 561 /* decrypt real key: */ 562 bch2_chacha20(&user_key, bch2_sb_key_nonce(c), &sb_key, sizeof(sb_key)); 563 564 if (bch2_key_is_encrypted(&sb_key)) { 565 bch_err(c, "incorrect encryption key"); 566 ret = -EINVAL; 567 goto err; 568 } 569 out: 570 *key = sb_key.key; 571 err: 572 memzero_explicit(&sb_key, sizeof(sb_key)); 573 memzero_explicit(&user_key, sizeof(user_key)); 574 return ret; 575 } 576 577 #if 0 578 579 /* 580 * This seems to be duplicating code in cmd_remove_passphrase() in 581 * bcachefs-tools, but we might want to switch userspace to use this - and 582 * perhaps add an ioctl for calling this at runtime, so we can take the 583 * passphrase off of a mounted filesystem (which has come up). 584 */ 585 int bch2_disable_encryption(struct bch_fs *c) 586 { 587 struct bch_sb_field_crypt *crypt; 588 struct bch_key key; 589 int ret = -EINVAL; 590 591 mutex_lock(&c->sb_lock); 592 593 crypt = bch2_sb_field_get(c->disk_sb.sb, crypt); 594 if (!crypt) 595 goto out; 596 597 /* is key encrypted? */ 598 ret = 0; 599 if (bch2_key_is_encrypted(&crypt->key)) 600 goto out; 601 602 ret = bch2_decrypt_sb_key(c, crypt, &key); 603 if (ret) 604 goto out; 605 606 crypt->key.magic = cpu_to_le64(BCH_KEY_MAGIC); 607 crypt->key.key = key; 608 609 SET_BCH_SB_ENCRYPTION_TYPE(c->disk_sb.sb, 0); 610 bch2_write_super(c); 611 out: 612 mutex_unlock(&c->sb_lock); 613 614 return ret; 615 } 616 617 /* 618 * For enabling encryption on an existing filesystem: not hooked up yet, but it 619 * should be 620 */ 621 int bch2_enable_encryption(struct bch_fs *c, bool keyed) 622 { 623 struct bch_encrypted_key key; 624 struct bch_key user_key; 625 struct bch_sb_field_crypt *crypt; 626 int ret = -EINVAL; 627 628 mutex_lock(&c->sb_lock); 629 630 /* Do we already have an encryption key? */ 631 if (bch2_sb_field_get(c->disk_sb.sb, crypt)) 632 goto err; 633 634 ret = bch2_alloc_ciphers(c); 635 if (ret) 636 goto err; 637 638 key.magic = cpu_to_le64(BCH_KEY_MAGIC); 639 get_random_bytes(&key.key, sizeof(key.key)); 640 641 if (keyed) { 642 ret = bch2_request_key(c->disk_sb.sb, &user_key); 643 if (ret) { 644 bch_err(c, "error requesting encryption key: %s", bch2_err_str(ret)); 645 goto err; 646 } 647 648 ret = bch2_chacha_encrypt_key(&user_key, bch2_sb_key_nonce(c), 649 &key, sizeof(key)); 650 if (ret) 651 goto err; 652 } 653 654 ret = crypto_skcipher_setkey(&c->chacha20->base, 655 (void *) &key.key, sizeof(key.key)); 656 if (ret) 657 goto err; 658 659 crypt = bch2_sb_field_resize(&c->disk_sb, crypt, 660 sizeof(*crypt) / sizeof(u64)); 661 if (!crypt) { 662 ret = -BCH_ERR_ENOSPC_sb_crypt; 663 goto err; 664 } 665 666 crypt->key = key; 667 668 /* write superblock */ 669 SET_BCH_SB_ENCRYPTION_TYPE(c->disk_sb.sb, 1); 670 bch2_write_super(c); 671 err: 672 mutex_unlock(&c->sb_lock); 673 memzero_explicit(&user_key, sizeof(user_key)); 674 memzero_explicit(&key, sizeof(key)); 675 return ret; 676 } 677 #endif 678 679 void bch2_fs_encryption_exit(struct bch_fs *c) 680 { 681 memzero_explicit(&c->chacha20_key, sizeof(c->chacha20_key)); 682 } 683 684 int bch2_fs_encryption_init(struct bch_fs *c) 685 { 686 struct bch_sb_field_crypt *crypt; 687 int ret; 688 689 crypt = bch2_sb_field_get(c->disk_sb.sb, crypt); 690 if (!crypt) 691 return 0; 692 693 ret = bch2_decrypt_sb_key(c, crypt, &c->chacha20_key); 694 if (ret) 695 return ret; 696 c->chacha20_key_set = true; 697 return 0; 698 } 699