1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/zstd.h> 3 #include "compress.h" 4 5 struct z_erofs_zstd { 6 struct z_erofs_zstd *next; 7 u8 bounce[PAGE_SIZE]; 8 void *wksp; 9 unsigned int wkspsz; 10 }; 11 12 static DEFINE_SPINLOCK(z_erofs_zstd_lock); 13 static unsigned int z_erofs_zstd_max_dictsize; 14 static unsigned int z_erofs_zstd_nstrms, z_erofs_zstd_avail_strms; 15 static struct z_erofs_zstd *z_erofs_zstd_head; 16 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_zstd_wq); 17 18 module_param_named(zstd_streams, z_erofs_zstd_nstrms, uint, 0444); 19 20 static struct z_erofs_zstd *z_erofs_isolate_strms(bool all) 21 { 22 struct z_erofs_zstd *strm; 23 24 again: 25 spin_lock(&z_erofs_zstd_lock); 26 strm = z_erofs_zstd_head; 27 if (!strm) { 28 spin_unlock(&z_erofs_zstd_lock); 29 wait_event(z_erofs_zstd_wq, READ_ONCE(z_erofs_zstd_head)); 30 goto again; 31 } 32 z_erofs_zstd_head = all ? NULL : strm->next; 33 spin_unlock(&z_erofs_zstd_lock); 34 return strm; 35 } 36 37 static void z_erofs_zstd_exit(void) 38 { 39 while (z_erofs_zstd_avail_strms) { 40 struct z_erofs_zstd *strm, *n; 41 42 for (strm = z_erofs_isolate_strms(true); strm; strm = n) { 43 n = strm->next; 44 45 kvfree(strm->wksp); 46 kfree(strm); 47 --z_erofs_zstd_avail_strms; 48 } 49 } 50 } 51 52 static int __init z_erofs_zstd_init(void) 53 { 54 /* by default, use # of possible CPUs instead */ 55 if (!z_erofs_zstd_nstrms) 56 z_erofs_zstd_nstrms = num_possible_cpus(); 57 58 for (; z_erofs_zstd_avail_strms < z_erofs_zstd_nstrms; 59 ++z_erofs_zstd_avail_strms) { 60 struct z_erofs_zstd *strm; 61 62 strm = kzalloc(sizeof(*strm), GFP_KERNEL); 63 if (!strm) { 64 z_erofs_zstd_exit(); 65 return -ENOMEM; 66 } 67 spin_lock(&z_erofs_zstd_lock); 68 strm->next = z_erofs_zstd_head; 69 z_erofs_zstd_head = strm; 70 spin_unlock(&z_erofs_zstd_lock); 71 } 72 return 0; 73 } 74 75 static int z_erofs_load_zstd_config(struct super_block *sb, 76 struct erofs_super_block *dsb, void *data, int size) 77 { 78 static DEFINE_MUTEX(zstd_resize_mutex); 79 struct z_erofs_zstd_cfgs *zstd = data; 80 unsigned int dict_size, wkspsz; 81 struct z_erofs_zstd *strm, *head = NULL; 82 void *wksp; 83 84 if (!zstd || size < sizeof(struct z_erofs_zstd_cfgs) || zstd->format) { 85 erofs_err(sb, "unsupported zstd format, size=%u", size); 86 return -EINVAL; 87 } 88 89 if (zstd->windowlog > ilog2(Z_EROFS_ZSTD_MAX_DICT_SIZE) - 10) { 90 erofs_err(sb, "unsupported zstd window log %u", zstd->windowlog); 91 return -EINVAL; 92 } 93 dict_size = 1U << (zstd->windowlog + 10); 94 95 /* in case 2 z_erofs_load_zstd_config() race to avoid deadlock */ 96 mutex_lock(&zstd_resize_mutex); 97 if (z_erofs_zstd_max_dictsize >= dict_size) { 98 mutex_unlock(&zstd_resize_mutex); 99 return 0; 100 } 101 102 /* 1. collect/isolate all streams for the following check */ 103 while (z_erofs_zstd_avail_strms) { 104 struct z_erofs_zstd *n; 105 106 for (strm = z_erofs_isolate_strms(true); strm; strm = n) { 107 n = strm->next; 108 strm->next = head; 109 head = strm; 110 --z_erofs_zstd_avail_strms; 111 } 112 } 113 114 /* 2. walk each isolated stream and grow max dict_size if needed */ 115 wkspsz = zstd_dstream_workspace_bound(dict_size); 116 for (strm = head; strm; strm = strm->next) { 117 wksp = kvmalloc(wkspsz, GFP_KERNEL); 118 if (!wksp) 119 break; 120 kvfree(strm->wksp); 121 strm->wksp = wksp; 122 strm->wkspsz = wkspsz; 123 } 124 125 /* 3. push back all to the global list and update max dict_size */ 126 spin_lock(&z_erofs_zstd_lock); 127 DBG_BUGON(z_erofs_zstd_head); 128 z_erofs_zstd_head = head; 129 spin_unlock(&z_erofs_zstd_lock); 130 z_erofs_zstd_avail_strms = z_erofs_zstd_nstrms; 131 wake_up_all(&z_erofs_zstd_wq); 132 if (!strm) 133 z_erofs_zstd_max_dictsize = dict_size; 134 mutex_unlock(&zstd_resize_mutex); 135 return strm ? -ENOMEM : 0; 136 } 137 138 static int z_erofs_zstd_decompress(struct z_erofs_decompress_req *rq, 139 struct page **pgpl) 140 { 141 struct super_block *sb = rq->sb; 142 struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 }; 143 zstd_in_buffer in_buf = { NULL, 0, 0 }; 144 zstd_out_buffer out_buf = { NULL, 0, 0 }; 145 struct z_erofs_zstd *strm; 146 zstd_dstream *stream; 147 int zerr, err; 148 149 /* 1. get the exact compressed size */ 150 dctx.kin = kmap_local_page(*rq->in); 151 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, 152 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); 153 if (err) { 154 kunmap_local(dctx.kin); 155 return err; 156 } 157 158 /* 2. get an available ZSTD context */ 159 strm = z_erofs_isolate_strms(false); 160 161 /* 3. multi-call decompress */ 162 stream = zstd_init_dstream(z_erofs_zstd_max_dictsize, strm->wksp, strm->wkspsz); 163 if (!stream) { 164 err = -EIO; 165 goto failed_zinit; 166 } 167 168 rq->fillgaps = true; /* ZSTD doesn't support NULL output buffer */ 169 in_buf.size = min_t(u32, rq->inputsize, PAGE_SIZE - rq->pageofs_in); 170 rq->inputsize -= in_buf.size; 171 in_buf.src = dctx.kin + rq->pageofs_in; 172 dctx.bounce = strm->bounce; 173 174 do { 175 dctx.avail_out = out_buf.size - out_buf.pos; 176 dctx.inbuf_sz = in_buf.size; 177 dctx.inbuf_pos = in_buf.pos; 178 err = z_erofs_stream_switch_bufs(&dctx, &out_buf.dst, 179 (void **)&in_buf.src, pgpl); 180 if (err) 181 break; 182 183 if (out_buf.size == out_buf.pos) { 184 out_buf.size = dctx.avail_out; 185 out_buf.pos = 0; 186 } 187 in_buf.size = dctx.inbuf_sz; 188 in_buf.pos = dctx.inbuf_pos; 189 190 zerr = zstd_decompress_stream(stream, &out_buf, &in_buf); 191 if (zstd_is_error(zerr) || (!zerr && rq->outputsize)) { 192 erofs_err(sb, "failed to decompress in[%u] out[%u]: %s", 193 rq->inputsize, rq->outputsize, 194 zerr ? zstd_get_error_name(zerr) : "unexpected end of stream"); 195 err = -EFSCORRUPTED; 196 break; 197 } 198 } while (rq->outputsize || out_buf.pos < out_buf.size); 199 200 if (dctx.kout) 201 kunmap_local(dctx.kout); 202 failed_zinit: 203 kunmap_local(dctx.kin); 204 /* 4. push back ZSTD stream context to the global list */ 205 spin_lock(&z_erofs_zstd_lock); 206 strm->next = z_erofs_zstd_head; 207 z_erofs_zstd_head = strm; 208 spin_unlock(&z_erofs_zstd_lock); 209 wake_up(&z_erofs_zstd_wq); 210 return err; 211 } 212 213 const struct z_erofs_decompressor z_erofs_zstd_decomp = { 214 .config = z_erofs_load_zstd_config, 215 .decompress = z_erofs_zstd_decompress, 216 .init = z_erofs_zstd_init, 217 .exit = z_erofs_zstd_exit, 218 .name = "zstd", 219 }; 220