1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <linux/zlib.h>
3 #include "compress.h"
4
5 struct z_erofs_deflate {
6 struct z_erofs_deflate *next;
7 struct z_stream_s z;
8 u8 bounce[PAGE_SIZE];
9 };
10
11 static DEFINE_SPINLOCK(z_erofs_deflate_lock);
12 static unsigned int z_erofs_deflate_nstrms, z_erofs_deflate_avail_strms;
13 static struct z_erofs_deflate *z_erofs_deflate_head;
14 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_deflate_wq);
15
16 module_param_named(deflate_streams, z_erofs_deflate_nstrms, uint, 0444);
17
z_erofs_deflate_exit(void)18 static void z_erofs_deflate_exit(void)
19 {
20 /* there should be no running fs instance */
21 while (z_erofs_deflate_avail_strms) {
22 struct z_erofs_deflate *strm;
23
24 spin_lock(&z_erofs_deflate_lock);
25 strm = z_erofs_deflate_head;
26 if (!strm) {
27 spin_unlock(&z_erofs_deflate_lock);
28 continue;
29 }
30 z_erofs_deflate_head = NULL;
31 spin_unlock(&z_erofs_deflate_lock);
32
33 while (strm) {
34 struct z_erofs_deflate *n = strm->next;
35
36 vfree(strm->z.workspace);
37 kfree(strm);
38 --z_erofs_deflate_avail_strms;
39 strm = n;
40 }
41 }
42 }
43
z_erofs_deflate_init(void)44 static int __init z_erofs_deflate_init(void)
45 {
46 /* by default, use # of possible CPUs instead */
47 if (!z_erofs_deflate_nstrms)
48 z_erofs_deflate_nstrms = num_possible_cpus();
49 return 0;
50 }
51
z_erofs_load_deflate_config(struct super_block * sb,struct erofs_super_block * dsb,void * data,int size)52 static int z_erofs_load_deflate_config(struct super_block *sb,
53 struct erofs_super_block *dsb, void *data, int size)
54 {
55 struct z_erofs_deflate_cfgs *dfl = data;
56 static DEFINE_MUTEX(deflate_resize_mutex);
57 static bool inited;
58
59 if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) {
60 erofs_err(sb, "invalid deflate cfgs, size=%u", size);
61 return -EINVAL;
62 }
63
64 if (dfl->windowbits > MAX_WBITS) {
65 erofs_err(sb, "unsupported windowbits %u", dfl->windowbits);
66 return -EOPNOTSUPP;
67 }
68 mutex_lock(&deflate_resize_mutex);
69 if (!inited) {
70 for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms;
71 ++z_erofs_deflate_avail_strms) {
72 struct z_erofs_deflate *strm;
73
74 strm = kzalloc(sizeof(*strm), GFP_KERNEL);
75 if (!strm)
76 goto failed;
77 /* XXX: in-kernel zlib cannot customize windowbits */
78 strm->z.workspace = vmalloc(zlib_inflate_workspacesize());
79 if (!strm->z.workspace) {
80 kfree(strm);
81 goto failed;
82 }
83
84 spin_lock(&z_erofs_deflate_lock);
85 strm->next = z_erofs_deflate_head;
86 z_erofs_deflate_head = strm;
87 spin_unlock(&z_erofs_deflate_lock);
88 }
89 inited = true;
90 }
91 mutex_unlock(&deflate_resize_mutex);
92 erofs_info(sb, "EXPERIMENTAL DEFLATE feature in use. Use at your own risk!");
93 return 0;
94 failed:
95 mutex_unlock(&deflate_resize_mutex);
96 z_erofs_deflate_exit();
97 return -ENOMEM;
98 }
99
z_erofs_deflate_decompress(struct z_erofs_decompress_req * rq,struct page ** pgpl)100 static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
101 struct page **pgpl)
102 {
103 struct super_block *sb = rq->sb;
104 struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 };
105 struct z_erofs_deflate *strm;
106 int zerr, err;
107
108 /* 1. get the exact DEFLATE compressed size */
109 dctx.kin = kmap_local_page(*rq->in);
110 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
111 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
112 if (err) {
113 kunmap_local(dctx.kin);
114 return err;
115 }
116
117 /* 2. get an available DEFLATE context */
118 again:
119 spin_lock(&z_erofs_deflate_lock);
120 strm = z_erofs_deflate_head;
121 if (!strm) {
122 spin_unlock(&z_erofs_deflate_lock);
123 wait_event(z_erofs_deflate_wq, READ_ONCE(z_erofs_deflate_head));
124 goto again;
125 }
126 z_erofs_deflate_head = strm->next;
127 spin_unlock(&z_erofs_deflate_lock);
128
129 /* 3. multi-call decompress */
130 zerr = zlib_inflateInit2(&strm->z, -MAX_WBITS);
131 if (zerr != Z_OK) {
132 err = -EIO;
133 goto failed_zinit;
134 }
135
136 rq->fillgaps = true; /* DEFLATE doesn't support NULL output buffer */
137 strm->z.avail_in = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in);
138 rq->inputsize -= strm->z.avail_in;
139 strm->z.next_in = dctx.kin + rq->pageofs_in;
140 strm->z.avail_out = 0;
141 dctx.bounce = strm->bounce;
142
143 while (1) {
144 dctx.avail_out = strm->z.avail_out;
145 dctx.inbuf_sz = strm->z.avail_in;
146 err = z_erofs_stream_switch_bufs(&dctx,
147 (void **)&strm->z.next_out,
148 (void **)&strm->z.next_in, pgpl);
149 if (err)
150 break;
151 strm->z.avail_out = dctx.avail_out;
152 strm->z.avail_in = dctx.inbuf_sz;
153
154 zerr = zlib_inflate(&strm->z, Z_SYNC_FLUSH);
155 if (zerr != Z_OK || !(rq->outputsize + strm->z.avail_out)) {
156 if (zerr == Z_OK && rq->partial_decoding)
157 break;
158 if (zerr == Z_STREAM_END && !rq->outputsize)
159 break;
160 erofs_err(sb, "failed to decompress %d in[%u] out[%u]",
161 zerr, rq->inputsize, rq->outputsize);
162 err = -EFSCORRUPTED;
163 break;
164 }
165 }
166 if (zlib_inflateEnd(&strm->z) != Z_OK && !err)
167 err = -EIO;
168 if (dctx.kout)
169 kunmap_local(dctx.kout);
170 failed_zinit:
171 kunmap_local(dctx.kin);
172 /* 4. push back DEFLATE stream context to the global list */
173 spin_lock(&z_erofs_deflate_lock);
174 strm->next = z_erofs_deflate_head;
175 z_erofs_deflate_head = strm;
176 spin_unlock(&z_erofs_deflate_lock);
177 wake_up(&z_erofs_deflate_wq);
178 return err;
179 }
180
181 const struct z_erofs_decompressor z_erofs_deflate_decomp = {
182 .config = z_erofs_load_deflate_config,
183 .decompress = z_erofs_deflate_decompress,
184 .init = z_erofs_deflate_init,
185 .exit = z_erofs_deflate_exit,
186 .name = "deflate",
187 };
188