1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <linux/zlib.h>
3 #include "compress.h"
4 
5 struct z_erofs_deflate {
6 	struct z_erofs_deflate *next;
7 	struct z_stream_s z;
8 	u8 bounce[PAGE_SIZE];
9 };
10 
11 static DEFINE_SPINLOCK(z_erofs_deflate_lock);
12 static unsigned int z_erofs_deflate_nstrms, z_erofs_deflate_avail_strms;
13 static struct z_erofs_deflate *z_erofs_deflate_head;
14 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_deflate_wq);
15 
16 module_param_named(deflate_streams, z_erofs_deflate_nstrms, uint, 0444);
17 
z_erofs_deflate_exit(void)18 static void z_erofs_deflate_exit(void)
19 {
20 	/* there should be no running fs instance */
21 	while (z_erofs_deflate_avail_strms) {
22 		struct z_erofs_deflate *strm;
23 
24 		spin_lock(&z_erofs_deflate_lock);
25 		strm = z_erofs_deflate_head;
26 		if (!strm) {
27 			spin_unlock(&z_erofs_deflate_lock);
28 			continue;
29 		}
30 		z_erofs_deflate_head = NULL;
31 		spin_unlock(&z_erofs_deflate_lock);
32 
33 		while (strm) {
34 			struct z_erofs_deflate *n = strm->next;
35 
36 			vfree(strm->z.workspace);
37 			kfree(strm);
38 			--z_erofs_deflate_avail_strms;
39 			strm = n;
40 		}
41 	}
42 }
43 
z_erofs_deflate_init(void)44 static int __init z_erofs_deflate_init(void)
45 {
46 	/* by default, use # of possible CPUs instead */
47 	if (!z_erofs_deflate_nstrms)
48 		z_erofs_deflate_nstrms = num_possible_cpus();
49 	return 0;
50 }
51 
z_erofs_load_deflate_config(struct super_block * sb,struct erofs_super_block * dsb,void * data,int size)52 static int z_erofs_load_deflate_config(struct super_block *sb,
53 			struct erofs_super_block *dsb, void *data, int size)
54 {
55 	struct z_erofs_deflate_cfgs *dfl = data;
56 	static DEFINE_MUTEX(deflate_resize_mutex);
57 	static bool inited;
58 
59 	if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) {
60 		erofs_err(sb, "invalid deflate cfgs, size=%u", size);
61 		return -EINVAL;
62 	}
63 
64 	if (dfl->windowbits > MAX_WBITS) {
65 		erofs_err(sb, "unsupported windowbits %u", dfl->windowbits);
66 		return -EOPNOTSUPP;
67 	}
68 	mutex_lock(&deflate_resize_mutex);
69 	if (!inited) {
70 		for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms;
71 		     ++z_erofs_deflate_avail_strms) {
72 			struct z_erofs_deflate *strm;
73 
74 			strm = kzalloc(sizeof(*strm), GFP_KERNEL);
75 			if (!strm)
76 				goto failed;
77 			/* XXX: in-kernel zlib cannot customize windowbits */
78 			strm->z.workspace = vmalloc(zlib_inflate_workspacesize());
79 			if (!strm->z.workspace) {
80 				kfree(strm);
81 				goto failed;
82 			}
83 
84 			spin_lock(&z_erofs_deflate_lock);
85 			strm->next = z_erofs_deflate_head;
86 			z_erofs_deflate_head = strm;
87 			spin_unlock(&z_erofs_deflate_lock);
88 		}
89 		inited = true;
90 	}
91 	mutex_unlock(&deflate_resize_mutex);
92 	erofs_info(sb, "EXPERIMENTAL DEFLATE feature in use. Use at your own risk!");
93 	return 0;
94 failed:
95 	mutex_unlock(&deflate_resize_mutex);
96 	z_erofs_deflate_exit();
97 	return -ENOMEM;
98 }
99 
z_erofs_deflate_decompress(struct z_erofs_decompress_req * rq,struct page ** pgpl)100 static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
101 				      struct page **pgpl)
102 {
103 	struct super_block *sb = rq->sb;
104 	struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 };
105 	struct z_erofs_deflate *strm;
106 	int zerr, err;
107 
108 	/* 1. get the exact DEFLATE compressed size */
109 	dctx.kin = kmap_local_page(*rq->in);
110 	err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
111 			min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
112 	if (err) {
113 		kunmap_local(dctx.kin);
114 		return err;
115 	}
116 
117 	/* 2. get an available DEFLATE context */
118 again:
119 	spin_lock(&z_erofs_deflate_lock);
120 	strm = z_erofs_deflate_head;
121 	if (!strm) {
122 		spin_unlock(&z_erofs_deflate_lock);
123 		wait_event(z_erofs_deflate_wq, READ_ONCE(z_erofs_deflate_head));
124 		goto again;
125 	}
126 	z_erofs_deflate_head = strm->next;
127 	spin_unlock(&z_erofs_deflate_lock);
128 
129 	/* 3. multi-call decompress */
130 	zerr = zlib_inflateInit2(&strm->z, -MAX_WBITS);
131 	if (zerr != Z_OK) {
132 		err = -EIO;
133 		goto failed_zinit;
134 	}
135 
136 	rq->fillgaps = true;	/* DEFLATE doesn't support NULL output buffer */
137 	strm->z.avail_in = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in);
138 	rq->inputsize -= strm->z.avail_in;
139 	strm->z.next_in = dctx.kin + rq->pageofs_in;
140 	strm->z.avail_out = 0;
141 	dctx.bounce = strm->bounce;
142 
143 	while (1) {
144 		dctx.avail_out = strm->z.avail_out;
145 		dctx.inbuf_sz = strm->z.avail_in;
146 		err = z_erofs_stream_switch_bufs(&dctx,
147 					(void **)&strm->z.next_out,
148 					(void **)&strm->z.next_in, pgpl);
149 		if (err)
150 			break;
151 		strm->z.avail_out = dctx.avail_out;
152 		strm->z.avail_in = dctx.inbuf_sz;
153 
154 		zerr = zlib_inflate(&strm->z, Z_SYNC_FLUSH);
155 		if (zerr != Z_OK || !(rq->outputsize + strm->z.avail_out)) {
156 			if (zerr == Z_OK && rq->partial_decoding)
157 				break;
158 			if (zerr == Z_STREAM_END && !rq->outputsize)
159 				break;
160 			erofs_err(sb, "failed to decompress %d in[%u] out[%u]",
161 				  zerr, rq->inputsize, rq->outputsize);
162 			err = -EFSCORRUPTED;
163 			break;
164 		}
165 	}
166 	if (zlib_inflateEnd(&strm->z) != Z_OK && !err)
167 		err = -EIO;
168 	if (dctx.kout)
169 		kunmap_local(dctx.kout);
170 failed_zinit:
171 	kunmap_local(dctx.kin);
172 	/* 4. push back DEFLATE stream context to the global list */
173 	spin_lock(&z_erofs_deflate_lock);
174 	strm->next = z_erofs_deflate_head;
175 	z_erofs_deflate_head = strm;
176 	spin_unlock(&z_erofs_deflate_lock);
177 	wake_up(&z_erofs_deflate_wq);
178 	return err;
179 }
180 
181 const struct z_erofs_decompressor z_erofs_deflate_decomp = {
182 	.config = z_erofs_load_deflate_config,
183 	.decompress = z_erofs_deflate_decompress,
184 	.init = z_erofs_deflate_init,
185 	.exit = z_erofs_deflate_exit,
186 	.name = "deflate",
187 };
188