1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* Internal definitions for network filesystem support 3 * 4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/slab.h> 9 #include <linux/seq_file.h> 10 #include <linux/netfs.h> 11 #include <linux/fscache.h> 12 #include <linux/fscache-cache.h> 13 #include <trace/events/netfs.h> 14 #include <trace/events/fscache.h> 15 16 #ifdef pr_fmt 17 #undef pr_fmt 18 #endif 19 20 #define pr_fmt(fmt) "netfs: " fmt 21 22 /* 23 * buffered_read.c 24 */ 25 void netfs_rreq_unlock_folios(struct netfs_io_request *rreq); 26 int netfs_prefetch_for_write(struct file *file, struct folio *folio, 27 size_t offset, size_t len); 28 29 /* 30 * direct_write.c 31 */ 32 ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *iter, 33 struct netfs_group *netfs_group); 34 35 /* 36 * io.c 37 */ 38 int netfs_begin_read(struct netfs_io_request *rreq, bool sync); 39 40 /* 41 * main.c 42 */ 43 extern unsigned int netfs_debug; 44 extern struct list_head netfs_io_requests; 45 extern spinlock_t netfs_proc_lock; 46 47 #ifdef CONFIG_PROC_FS 48 static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq) 49 { 50 spin_lock(&netfs_proc_lock); 51 list_add_tail_rcu(&rreq->proc_link, &netfs_io_requests); 52 spin_unlock(&netfs_proc_lock); 53 } 54 static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) 55 { 56 if (!list_empty(&rreq->proc_link)) { 57 spin_lock(&netfs_proc_lock); 58 list_del_rcu(&rreq->proc_link); 59 spin_unlock(&netfs_proc_lock); 60 } 61 } 62 #else 63 static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq) {} 64 static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) {} 65 #endif 66 67 /* 68 * misc.c 69 */ 70 #define NETFS_FLAG_PUT_MARK BIT(0) 71 #define NETFS_FLAG_PAGECACHE_MARK BIT(1) 72 int netfs_xa_store_and_mark(struct xarray *xa, unsigned long index, 73 struct folio *folio, unsigned int flags, 74 gfp_t gfp_mask); 75 int netfs_add_folios_to_buffer(struct xarray *buffer, 76 struct address_space *mapping, 77 pgoff_t index, pgoff_t to, gfp_t gfp_mask); 78 void netfs_clear_buffer(struct xarray *buffer); 79 80 /* 81 * objects.c 82 */ 83 struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, 84 struct file *file, 85 loff_t start, size_t len, 86 enum netfs_io_origin origin); 87 void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what); 88 void netfs_clear_subrequests(struct netfs_io_request *rreq, bool was_async); 89 void netfs_put_request(struct netfs_io_request *rreq, bool was_async, 90 enum netfs_rreq_ref_trace what); 91 struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq); 92 93 static inline void netfs_see_request(struct netfs_io_request *rreq, 94 enum netfs_rreq_ref_trace what) 95 { 96 trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what); 97 } 98 99 /* 100 * output.c 101 */ 102 int netfs_begin_write(struct netfs_io_request *wreq, bool may_wait, 103 enum netfs_write_trace what); 104 105 /* 106 * stats.c 107 */ 108 #ifdef CONFIG_NETFS_STATS 109 extern atomic_t netfs_n_rh_dio_read; 110 extern atomic_t netfs_n_rh_readahead; 111 extern atomic_t netfs_n_rh_readpage; 112 extern atomic_t netfs_n_rh_rreq; 113 extern atomic_t netfs_n_rh_sreq; 114 extern atomic_t netfs_n_rh_download; 115 extern atomic_t netfs_n_rh_download_done; 116 extern atomic_t netfs_n_rh_download_failed; 117 extern atomic_t netfs_n_rh_download_instead; 118 extern atomic_t netfs_n_rh_read; 119 extern atomic_t netfs_n_rh_read_done; 120 extern atomic_t netfs_n_rh_read_failed; 121 extern atomic_t netfs_n_rh_zero; 122 extern atomic_t netfs_n_rh_short_read; 123 extern atomic_t netfs_n_rh_write; 124 extern atomic_t netfs_n_rh_write_begin; 125 extern atomic_t netfs_n_rh_write_done; 126 extern atomic_t netfs_n_rh_write_failed; 127 extern atomic_t netfs_n_rh_write_zskip; 128 extern atomic_t netfs_n_wh_upload; 129 extern atomic_t netfs_n_wh_upload_done; 130 extern atomic_t netfs_n_wh_upload_failed; 131 extern atomic_t netfs_n_wh_write; 132 extern atomic_t netfs_n_wh_write_done; 133 extern atomic_t netfs_n_wh_write_failed; 134 135 int netfs_stats_show(struct seq_file *m, void *v); 136 137 static inline void netfs_stat(atomic_t *stat) 138 { 139 atomic_inc(stat); 140 } 141 142 static inline void netfs_stat_d(atomic_t *stat) 143 { 144 atomic_dec(stat); 145 } 146 147 #else 148 #define netfs_stat(x) do {} while(0) 149 #define netfs_stat_d(x) do {} while(0) 150 #endif 151 152 /* 153 * Miscellaneous functions. 154 */ 155 static inline bool netfs_is_cache_enabled(struct netfs_inode *ctx) 156 { 157 #if IS_ENABLED(CONFIG_FSCACHE) 158 struct fscache_cookie *cookie = ctx->cache; 159 160 return fscache_cookie_valid(cookie) && cookie->cache_priv && 161 fscache_cookie_enabled(cookie); 162 #else 163 return false; 164 #endif 165 } 166 167 /* 168 * Get a ref on a netfs group attached to a dirty page (e.g. a ceph snap). 169 */ 170 static inline struct netfs_group *netfs_get_group(struct netfs_group *netfs_group) 171 { 172 if (netfs_group) 173 refcount_inc(&netfs_group->ref); 174 return netfs_group; 175 } 176 177 /* 178 * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap). 179 */ 180 static inline void netfs_put_group(struct netfs_group *netfs_group) 181 { 182 if (netfs_group && refcount_dec_and_test(&netfs_group->ref)) 183 netfs_group->free(netfs_group); 184 } 185 186 /* 187 * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap). 188 */ 189 static inline void netfs_put_group_many(struct netfs_group *netfs_group, int nr) 190 { 191 if (netfs_group && refcount_sub_and_test(nr, &netfs_group->ref)) 192 netfs_group->free(netfs_group); 193 } 194 195 /* 196 * fscache-cache.c 197 */ 198 #ifdef CONFIG_PROC_FS 199 extern const struct seq_operations fscache_caches_seq_ops; 200 #endif 201 bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why); 202 void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why); 203 struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache); 204 void fscache_put_cache(struct fscache_cache *cache, enum fscache_cache_trace where); 205 206 static inline enum fscache_cache_state fscache_cache_state(const struct fscache_cache *cache) 207 { 208 return smp_load_acquire(&cache->state); 209 } 210 211 static inline bool fscache_cache_is_live(const struct fscache_cache *cache) 212 { 213 return fscache_cache_state(cache) == FSCACHE_CACHE_IS_ACTIVE; 214 } 215 216 static inline void fscache_set_cache_state(struct fscache_cache *cache, 217 enum fscache_cache_state new_state) 218 { 219 smp_store_release(&cache->state, new_state); 220 221 } 222 223 static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache, 224 enum fscache_cache_state old_state, 225 enum fscache_cache_state new_state) 226 { 227 return try_cmpxchg_release(&cache->state, &old_state, new_state); 228 } 229 230 /* 231 * fscache-cookie.c 232 */ 233 extern struct kmem_cache *fscache_cookie_jar; 234 #ifdef CONFIG_PROC_FS 235 extern const struct seq_operations fscache_cookies_seq_ops; 236 #endif 237 extern struct timer_list fscache_cookie_lru_timer; 238 239 extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix); 240 extern bool fscache_begin_cookie_access(struct fscache_cookie *cookie, 241 enum fscache_access_trace why); 242 243 static inline void fscache_see_cookie(struct fscache_cookie *cookie, 244 enum fscache_cookie_trace where) 245 { 246 trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref), 247 where); 248 } 249 250 /* 251 * fscache-main.c 252 */ 253 extern unsigned int fscache_hash(unsigned int salt, const void *data, size_t len); 254 #ifdef CONFIG_FSCACHE 255 int __init fscache_init(void); 256 void __exit fscache_exit(void); 257 #else 258 static inline int fscache_init(void) { return 0; } 259 static inline void fscache_exit(void) {} 260 #endif 261 262 /* 263 * fscache-proc.c 264 */ 265 #ifdef CONFIG_PROC_FS 266 extern int __init fscache_proc_init(void); 267 extern void fscache_proc_cleanup(void); 268 #else 269 #define fscache_proc_init() (0) 270 #define fscache_proc_cleanup() do {} while (0) 271 #endif 272 273 /* 274 * fscache-stats.c 275 */ 276 #ifdef CONFIG_FSCACHE_STATS 277 extern atomic_t fscache_n_volumes; 278 extern atomic_t fscache_n_volumes_collision; 279 extern atomic_t fscache_n_volumes_nomem; 280 extern atomic_t fscache_n_cookies; 281 extern atomic_t fscache_n_cookies_lru; 282 extern atomic_t fscache_n_cookies_lru_expired; 283 extern atomic_t fscache_n_cookies_lru_removed; 284 extern atomic_t fscache_n_cookies_lru_dropped; 285 286 extern atomic_t fscache_n_acquires; 287 extern atomic_t fscache_n_acquires_ok; 288 extern atomic_t fscache_n_acquires_oom; 289 290 extern atomic_t fscache_n_invalidates; 291 292 extern atomic_t fscache_n_relinquishes; 293 extern atomic_t fscache_n_relinquishes_retire; 294 extern atomic_t fscache_n_relinquishes_dropped; 295 296 extern atomic_t fscache_n_resizes; 297 extern atomic_t fscache_n_resizes_null; 298 299 static inline void fscache_stat(atomic_t *stat) 300 { 301 atomic_inc(stat); 302 } 303 304 static inline void fscache_stat_d(atomic_t *stat) 305 { 306 atomic_dec(stat); 307 } 308 309 #define __fscache_stat(stat) (stat) 310 311 int fscache_stats_show(struct seq_file *m); 312 #else 313 314 #define __fscache_stat(stat) (NULL) 315 #define fscache_stat(stat) do {} while (0) 316 #define fscache_stat_d(stat) do {} while (0) 317 318 static inline int fscache_stats_show(struct seq_file *m) { return 0; } 319 #endif 320 321 /* 322 * fscache-volume.c 323 */ 324 #ifdef CONFIG_PROC_FS 325 extern const struct seq_operations fscache_volumes_seq_ops; 326 #endif 327 328 struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, 329 enum fscache_volume_trace where); 330 void fscache_put_volume(struct fscache_volume *volume, 331 enum fscache_volume_trace where); 332 bool fscache_begin_volume_access(struct fscache_volume *volume, 333 struct fscache_cookie *cookie, 334 enum fscache_access_trace why); 335 void fscache_create_volume(struct fscache_volume *volume, bool wait); 336 337 /*****************************************************************************/ 338 /* 339 * debug tracing 340 */ 341 #define dbgprintk(FMT, ...) \ 342 printk("[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) 343 344 #define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__) 345 #define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__) 346 #define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__) 347 348 #ifdef __KDEBUG 349 #define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__) 350 #define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__) 351 #define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__) 352 353 #elif defined(CONFIG_NETFS_DEBUG) 354 #define _enter(FMT, ...) \ 355 do { \ 356 if (netfs_debug) \ 357 kenter(FMT, ##__VA_ARGS__); \ 358 } while (0) 359 360 #define _leave(FMT, ...) \ 361 do { \ 362 if (netfs_debug) \ 363 kleave(FMT, ##__VA_ARGS__); \ 364 } while (0) 365 366 #define _debug(FMT, ...) \ 367 do { \ 368 if (netfs_debug) \ 369 kdebug(FMT, ##__VA_ARGS__); \ 370 } while (0) 371 372 #else 373 #define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__) 374 #define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__) 375 #define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__) 376 #endif 377 378 /* 379 * assertions 380 */ 381 #if 1 /* defined(__KDEBUGALL) */ 382 383 #define ASSERT(X) \ 384 do { \ 385 if (unlikely(!(X))) { \ 386 pr_err("\n"); \ 387 pr_err("Assertion failed\n"); \ 388 BUG(); \ 389 } \ 390 } while (0) 391 392 #define ASSERTCMP(X, OP, Y) \ 393 do { \ 394 if (unlikely(!((X) OP (Y)))) { \ 395 pr_err("\n"); \ 396 pr_err("Assertion failed\n"); \ 397 pr_err("%lx " #OP " %lx is false\n", \ 398 (unsigned long)(X), (unsigned long)(Y)); \ 399 BUG(); \ 400 } \ 401 } while (0) 402 403 #define ASSERTIF(C, X) \ 404 do { \ 405 if (unlikely((C) && !(X))) { \ 406 pr_err("\n"); \ 407 pr_err("Assertion failed\n"); \ 408 BUG(); \ 409 } \ 410 } while (0) 411 412 #define ASSERTIFCMP(C, X, OP, Y) \ 413 do { \ 414 if (unlikely((C) && !((X) OP (Y)))) { \ 415 pr_err("\n"); \ 416 pr_err("Assertion failed\n"); \ 417 pr_err("%lx " #OP " %lx is false\n", \ 418 (unsigned long)(X), (unsigned long)(Y)); \ 419 BUG(); \ 420 } \ 421 } while (0) 422 423 #else 424 425 #define ASSERT(X) do {} while (0) 426 #define ASSERTCMP(X, OP, Y) do {} while (0) 427 #define ASSERTIF(C, X) do {} while (0) 428 #define ASSERTIFCMP(C, X, OP, Y) do {} while (0) 429 430 #endif /* assert or not */ 431