1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/xz.h> 3 #include "compress.h" 4 5 struct z_erofs_lzma { 6 struct z_erofs_lzma *next; 7 struct xz_dec_microlzma *state; 8 u8 bounce[PAGE_SIZE]; 9 }; 10 11 /* considering the LZMA performance, no need to use a lockless list for now */ 12 static DEFINE_SPINLOCK(z_erofs_lzma_lock); 13 static unsigned int z_erofs_lzma_max_dictsize; 14 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms; 15 static struct z_erofs_lzma *z_erofs_lzma_head; 16 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq); 17 18 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444); 19 20 static void z_erofs_lzma_exit(void) 21 { 22 /* there should be no running fs instance */ 23 while (z_erofs_lzma_avail_strms) { 24 struct z_erofs_lzma *strm; 25 26 spin_lock(&z_erofs_lzma_lock); 27 strm = z_erofs_lzma_head; 28 if (!strm) { 29 spin_unlock(&z_erofs_lzma_lock); 30 DBG_BUGON(1); 31 return; 32 } 33 z_erofs_lzma_head = NULL; 34 spin_unlock(&z_erofs_lzma_lock); 35 36 while (strm) { 37 struct z_erofs_lzma *n = strm->next; 38 39 if (strm->state) 40 xz_dec_microlzma_end(strm->state); 41 kfree(strm); 42 --z_erofs_lzma_avail_strms; 43 strm = n; 44 } 45 } 46 } 47 48 static int __init z_erofs_lzma_init(void) 49 { 50 unsigned int i; 51 52 /* by default, use # of possible CPUs instead */ 53 if (!z_erofs_lzma_nstrms) 54 z_erofs_lzma_nstrms = num_possible_cpus(); 55 56 for (i = 0; i < z_erofs_lzma_nstrms; ++i) { 57 struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL); 58 59 if (!strm) { 60 z_erofs_lzma_exit(); 61 return -ENOMEM; 62 } 63 spin_lock(&z_erofs_lzma_lock); 64 strm->next = z_erofs_lzma_head; 65 z_erofs_lzma_head = strm; 66 spin_unlock(&z_erofs_lzma_lock); 67 ++z_erofs_lzma_avail_strms; 68 } 69 return 0; 70 } 71 72 static int z_erofs_load_lzma_config(struct super_block *sb, 73 struct erofs_super_block *dsb, void *data, int size) 74 { 75 static DEFINE_MUTEX(lzma_resize_mutex); 76 struct z_erofs_lzma_cfgs *lzma = data; 77 unsigned int dict_size, i; 78 struct z_erofs_lzma *strm, *head = NULL; 79 int err; 80 81 if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) { 82 erofs_err(sb, "invalid lzma cfgs, size=%u", size); 83 return -EINVAL; 84 } 85 if (lzma->format) { 86 erofs_err(sb, "unidentified lzma format %x, please check kernel version", 87 le16_to_cpu(lzma->format)); 88 return -EINVAL; 89 } 90 dict_size = le32_to_cpu(lzma->dict_size); 91 if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) { 92 erofs_err(sb, "unsupported lzma dictionary size %u", 93 dict_size); 94 return -EINVAL; 95 } 96 97 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */ 98 mutex_lock(&lzma_resize_mutex); 99 100 if (z_erofs_lzma_max_dictsize >= dict_size) { 101 mutex_unlock(&lzma_resize_mutex); 102 return 0; 103 } 104 105 /* 1. collect/isolate all streams for the following check */ 106 for (i = 0; i < z_erofs_lzma_avail_strms; ++i) { 107 struct z_erofs_lzma *last; 108 109 again: 110 spin_lock(&z_erofs_lzma_lock); 111 strm = z_erofs_lzma_head; 112 if (!strm) { 113 spin_unlock(&z_erofs_lzma_lock); 114 wait_event(z_erofs_lzma_wq, 115 READ_ONCE(z_erofs_lzma_head)); 116 goto again; 117 } 118 z_erofs_lzma_head = NULL; 119 spin_unlock(&z_erofs_lzma_lock); 120 121 for (last = strm; last->next; last = last->next) 122 ++i; 123 last->next = head; 124 head = strm; 125 } 126 127 err = 0; 128 /* 2. walk each isolated stream and grow max dict_size if needed */ 129 for (strm = head; strm; strm = strm->next) { 130 if (strm->state) 131 xz_dec_microlzma_end(strm->state); 132 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); 133 if (!strm->state) 134 err = -ENOMEM; 135 } 136 137 /* 3. push back all to the global list and update max dict_size */ 138 spin_lock(&z_erofs_lzma_lock); 139 DBG_BUGON(z_erofs_lzma_head); 140 z_erofs_lzma_head = head; 141 spin_unlock(&z_erofs_lzma_lock); 142 wake_up_all(&z_erofs_lzma_wq); 143 144 z_erofs_lzma_max_dictsize = dict_size; 145 mutex_unlock(&lzma_resize_mutex); 146 return err; 147 } 148 149 static int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq, 150 struct page **pgpl) 151 { 152 struct super_block *sb = rq->sb; 153 struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 }; 154 struct xz_buf buf = {}; 155 struct z_erofs_lzma *strm; 156 enum xz_ret xz_err; 157 int err; 158 159 /* 1. get the exact LZMA compressed size */ 160 dctx.kin = kmap_local_page(*rq->in); 161 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, 162 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); 163 if (err) { 164 kunmap_local(dctx.kin); 165 return err; 166 } 167 168 /* 2. get an available lzma context */ 169 again: 170 spin_lock(&z_erofs_lzma_lock); 171 strm = z_erofs_lzma_head; 172 if (!strm) { 173 spin_unlock(&z_erofs_lzma_lock); 174 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head)); 175 goto again; 176 } 177 z_erofs_lzma_head = strm->next; 178 spin_unlock(&z_erofs_lzma_lock); 179 180 /* 3. multi-call decompress */ 181 xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize, 182 !rq->partial_decoding); 183 buf.in_size = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in); 184 rq->inputsize -= buf.in_size; 185 buf.in = dctx.kin + rq->pageofs_in; 186 dctx.bounce = strm->bounce; 187 do { 188 dctx.avail_out = buf.out_size - buf.out_pos; 189 dctx.inbuf_sz = buf.in_size; 190 dctx.inbuf_pos = buf.in_pos; 191 err = z_erofs_stream_switch_bufs(&dctx, (void **)&buf.out, 192 (void **)&buf.in, pgpl); 193 if (err) 194 break; 195 196 if (buf.out_size == buf.out_pos) { 197 buf.out_size = dctx.avail_out; 198 buf.out_pos = 0; 199 } 200 buf.in_size = dctx.inbuf_sz; 201 buf.in_pos = dctx.inbuf_pos; 202 203 xz_err = xz_dec_microlzma_run(strm->state, &buf); 204 DBG_BUGON(buf.out_pos > buf.out_size); 205 DBG_BUGON(buf.in_pos > buf.in_size); 206 207 if (xz_err != XZ_OK) { 208 if (xz_err == XZ_STREAM_END && !rq->outputsize) 209 break; 210 erofs_err(sb, "failed to decompress %d in[%u] out[%u]", 211 xz_err, rq->inputsize, rq->outputsize); 212 err = -EFSCORRUPTED; 213 break; 214 } 215 } while (1); 216 217 if (dctx.kout) 218 kunmap_local(dctx.kout); 219 kunmap_local(dctx.kin); 220 /* 4. push back LZMA stream context to the global list */ 221 spin_lock(&z_erofs_lzma_lock); 222 strm->next = z_erofs_lzma_head; 223 z_erofs_lzma_head = strm; 224 spin_unlock(&z_erofs_lzma_lock); 225 wake_up(&z_erofs_lzma_wq); 226 return err; 227 } 228 229 const struct z_erofs_decompressor z_erofs_lzma_decomp = { 230 .config = z_erofs_load_lzma_config, 231 .decompress = z_erofs_lzma_decompress, 232 .init = z_erofs_lzma_init, 233 .exit = z_erofs_lzma_exit, 234 .name = "lzma" 235 }; 236