projects/linux·blob master · refs

602 lines · 17.1 KiB

1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/* Internal definitions for network filesystem support
3 *
4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells ([email protected])
6 */
7
8#include <linux/slab.h>
9#include <linux/seq_file.h>
10#include <linux/folio_queue.h>
11#include <linux/netfs.h>
12#include <linux/fscache.h>
13#include <linux/fscache-cache.h>
14#include <trace/events/netfs.h>
15#include <trace/events/fscache.h>
16
17#ifdef pr_fmt
18#undef pr_fmt
19#endif
20
21#define pr_fmt(fmt) "netfs: " fmt
22
23/*
24 * buffered_read.c
25 */
26int netfs_read_query_cache(struct netfs_io_request *rreq,
27 struct fscache_occupancy *occ);
28void netfs_queue_read(struct netfs_io_request *rreq,
29 struct netfs_io_subrequest *subreq);
30void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
31int netfs_prefetch_for_write(struct file *file, struct folio *folio,
32 size_t offset, size_t len);
33
34/*
35 * buffered_write.c
36 */
37void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode,
38 uoff_t pos, size_t copied);
39
40/*
41 * main.c
42 */
43extern unsigned int netfs_debug;
44extern struct list_head netfs_io_requests;
45extern spinlock_t netfs_proc_lock;
46extern mempool_t netfs_request_pool;
47extern mempool_t netfs_subrequest_pool;
48extern mempool_t netfs_folioq_pool;
49
50#ifdef CONFIG_PROC_FS
51static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq)
52{
53 spin_lock(&netfs_proc_lock);
54 list_add_tail_rcu(&rreq->proc_link, &netfs_io_requests);
55 spin_unlock(&netfs_proc_lock);
56}
57static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq)
58{
59 if (!list_empty(&rreq->proc_link)) {
60 spin_lock(&netfs_proc_lock);
61 list_del_rcu(&rreq->proc_link);
62 spin_unlock(&netfs_proc_lock);
63 }
64}
65#else
66static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq) {}
67static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) {}
68#endif
69
70/*
71 * misc.c
72 */
73struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq,
74 enum netfs_folioq_trace trace);
75void netfs_reset_iter(struct netfs_io_subrequest *subreq);
76void netfs_wake_collector(struct netfs_io_request *rreq);
77void netfs_subreq_clear_in_progress(struct netfs_io_subrequest *subreq);
78void netfs_wait_for_in_progress_stream(struct netfs_io_request *rreq,
79 struct netfs_io_stream *stream);
80ssize_t netfs_wait_for_read(struct netfs_io_request *rreq);
81ssize_t netfs_wait_for_write(struct netfs_io_request *rreq);
82void netfs_wait_for_paused_read(struct netfs_io_request *rreq);
83void netfs_wait_for_paused_write(struct netfs_io_request *rreq);
84void netfs_wait_for_put_ra_refs(struct netfs_io_request *rreq);
85int netfs_clear_stale_pre_isize(struct inode *inode, uoff_t from,
86 uoff_t to, bool nowait);
87
88/*
89 * objects.c
90 */
91struct netfs_io_request *netfs_alloc_request(struct address_space *mapping,
92 struct file *file,
93 uoff_t start, size_t len,
94 enum netfs_io_origin origin);
95void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what);
96void netfs_clear_subrequests(struct netfs_io_request *rreq);
97void netfs_put_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what);
98void netfs_put_failed_request(struct netfs_io_request *rreq);
99struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq,
100 enum netfs_io_source source);
101
102static inline void netfs_see_request(struct netfs_io_request *rreq,
103 enum netfs_rreq_ref_trace what)
104{
105 trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what);
106}
107
108static inline void netfs_see_subrequest(struct netfs_io_subrequest *subreq,
109 enum netfs_sreq_ref_trace what)
110{
111 trace_netfs_sreq_ref(subreq->rreq->debug_id, subreq->debug_index,
112 refcount_read(&subreq->ref), what);
113}
114
115/*
116 * read_collect.c
117 */
118void netfs_cancel_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio);
119void netfs_read_set_unlock_at(struct netfs_io_request *rreq);
120bool netfs_read_collection(struct netfs_io_request *rreq);
121void netfs_read_collection_worker(struct work_struct *work);
122void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error);
123void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
124
125/*
126 * read_pgpriv2.c
127 */
128#ifdef CONFIG_NETFS_PGPRIV2
129int netfs_prepare_pgpriv2_write_buffer(struct netfs_io_subrequest *subreq,
130 unsigned int max_segs);
131void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio);
132void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq);
133bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq);
134static inline bool netfs_using_pgpriv2(const struct netfs_io_request *rreq)
135{
136 return unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags));
137}
138#else
139static inline int netfs_prepare_pgpriv2_write_buffer(struct netfs_io_subrequest *subreq,
140 unsigned int max_segs)
141{
142 return -EIO;
143}
144static inline void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio)
145{
146}
147static inline void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq)
148{
149}
150static inline bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq)
151{
152 return true;
153}
154static inline bool netfs_using_pgpriv2(const struct netfs_io_request *rreq)
155{
156 return false;
157}
158#endif
159
160/*
161 * read_retry.c
162 */
163void netfs_retry_reads(struct netfs_io_request *rreq);
164void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq);
165
166/*
167 * stats.c
168 */
169#ifdef CONFIG_NETFS_STATS
170extern atomic_t netfs_n_rh_dio_read;
171extern atomic_t netfs_n_rh_readahead;
172extern atomic_t netfs_n_rh_read_folio;
173extern atomic_t netfs_n_rh_read_single;
174extern atomic_t netfs_n_rh_rreq;
175extern atomic_t netfs_n_rh_sreq;
176extern atomic_t netfs_n_rh_download;
177extern atomic_t netfs_n_rh_download_done;
178extern atomic_t netfs_n_rh_download_failed;
179extern atomic_t netfs_n_rh_download_instead;
180extern atomic_t netfs_n_rh_read;
181extern atomic_t netfs_n_rh_read_done;
182extern atomic_t netfs_n_rh_read_failed;
183extern atomic_t netfs_n_rh_zero;
184extern atomic_t netfs_n_rh_short_read;
185extern atomic_t netfs_n_rh_write;
186extern atomic_t netfs_n_rh_write_begin;
187extern atomic_t netfs_n_rh_write_done;
188extern atomic_t netfs_n_rh_write_failed;
189extern atomic_t netfs_n_rh_write_zskip;
190extern atomic_t netfs_n_rh_retry_read_req;
191extern atomic_t netfs_n_rh_retry_read_subreq;
192extern atomic_t netfs_n_wh_buffered_write;
193extern atomic_t netfs_n_wh_dio_write;
194extern atomic_t netfs_n_wh_writepages;
195extern atomic_t netfs_n_wh_copy_to_cache;
196extern atomic_t netfs_n_wh_wstream_conflict;
197extern atomic_t netfs_n_wh_upload;
198extern atomic_t netfs_n_wh_upload_done;
199extern atomic_t netfs_n_wh_upload_failed;
200extern atomic_t netfs_n_wh_write;
201extern atomic_t netfs_n_wh_write_done;
202extern atomic_t netfs_n_wh_write_failed;
203extern atomic_t netfs_n_wh_retry_write_req;
204extern atomic_t netfs_n_wh_retry_write_subreq;
205extern atomic_t netfs_n_wb_lock_skip;
206extern atomic_t netfs_n_wb_lock_wait;
207extern atomic_t netfs_n_folioq;
208
209int netfs_stats_show(struct seq_file *m, void *v);
210
211static inline void netfs_stat(atomic_t *stat)
212{
213 atomic_inc(stat);
214}
215
216static inline void netfs_stat_d(atomic_t *stat)
217{
218 atomic_dec(stat);
219}
220
221#else
222#define netfs_stat(x) do {} while(0)
223#define netfs_stat_d(x) do {} while(0)
224#endif
225
226/*
227 * write_collect.c
228 */
229int netfs_folio_written_back(struct folio *folio);
230bool netfs_write_collection(struct netfs_io_request *wreq);
231void netfs_write_collection_worker(struct work_struct *work);
232
233/*
234 * write_issue.c
235 */
236struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
237 struct file *file,
238 uoff_t start,
239 enum netfs_io_origin origin);
240void netfs_prepare_write(struct netfs_io_request *wreq,
241 struct netfs_io_stream *stream,
242 uoff_t start);
243void netfs_reissue_write(struct netfs_io_stream *stream,
244 struct netfs_io_subrequest *subreq,
245 struct iov_iter *source);
246void netfs_issue_write(struct netfs_io_request *wreq,
247 struct netfs_io_stream *stream);
248size_t netfs_advance_write(struct netfs_io_request *wreq,
249 struct netfs_io_stream *stream,
250 uoff_t start, size_t len, bool to_eof);
251
252/*
253 * write_retry.c
254 */
255void netfs_retry_writes(struct netfs_io_request *wreq);
256
257/*
258 * Miscellaneous functions.
259 */
260static inline bool netfs_is_cache_enabled(struct netfs_inode *ctx)
261{
262#if IS_ENABLED(CONFIG_FSCACHE)
263 struct fscache_cookie *cookie = ctx->cache;
264
265 return fscache_cookie_valid(cookie) && cookie->cache_priv &&
266 fscache_cookie_enabled(cookie);
267#else
268 return false;
269#endif
270}
271
272static inline bool netfs_is_cache_maybe_enabled(struct netfs_inode *ctx)
273{
274#if IS_ENABLED(CONFIG_FSCACHE)
275 struct fscache_cookie *cookie = ctx->cache;
276
277 return fscache_cookie_valid(cookie) &&
278 test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
279#else
280 return false;
281#endif
282}
283
284/*
285 * Get a ref on a netfs group attached to a dirty page (e.g. a ceph snap).
286 */
287static inline struct netfs_group *netfs_get_group(struct netfs_group *netfs_group)
288{
289 if (netfs_group && netfs_group != NETFS_FOLIO_COPY_TO_CACHE)
290 refcount_inc(&netfs_group->ref);
291 return netfs_group;
292}
293
294/*
295 * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap).
296 */
297static inline void netfs_put_group(struct netfs_group *netfs_group)
298{
299 if (netfs_group &&
300 netfs_group != NETFS_FOLIO_COPY_TO_CACHE &&
301 refcount_dec_and_test(&netfs_group->ref))
302 netfs_group->free(netfs_group);
303}
304
305/*
306 * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap).
307 */
308static inline void netfs_put_group_many(struct netfs_group *netfs_group, int nr)
309{
310 if (netfs_group &&
311 netfs_group != NETFS_FOLIO_COPY_TO_CACHE &&
312 refcount_sub_and_test(nr, &netfs_group->ref))
313 netfs_group->free(netfs_group);
314}
315
316/*
317 * Clear and wake up a NETFS_RREQ_* flag bit on a request.
318 */
319static inline void netfs_wake_rreq_flag(struct netfs_io_request *rreq,
320 unsigned int rreq_flag,
321 enum netfs_rreq_trace trace)
322{
323 if (test_bit(rreq_flag, &rreq->flags)) {
324 clear_bit_unlock(rreq_flag, &rreq->flags);
325 smp_mb__after_atomic(); /* Set flag before task state */
326 trace_netfs_rreq(rreq, trace);
327 wake_up(&rreq->waitq);
328 }
329}
330
331/*
332 * Test the NETFS_RREQ_IN_PROGRESS flag, inserting an appropriate barrier.
333 */
334static inline bool netfs_check_rreq_in_progress(const struct netfs_io_request *rreq)
335{
336 /* Order read of flags before read of anything else, such as error. */
337 return test_bit_acquire(NETFS_RREQ_IN_PROGRESS, &rreq->flags);
338}
339
340/*
341 * Test the NETFS_SREQ_IN_PROGRESS flag, inserting an appropriate barrier.
342 */
343static inline bool netfs_check_subreq_in_progress(const struct netfs_io_subrequest *subreq)
344{
345 /* Order read of flags before read of anything else, such as error. */
346 return test_bit_acquire(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
347}
348
349/*
350 * Indicate that we've generated and queued all the subrequests we're going to.
351 */
352static inline void netfs_all_subreqs_queued(struct netfs_io_request *rreq)
353{
354 smp_wmb(); /* Write lists before ALL_QUEUED. */
355 set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
356 smp_mb__after_atomic();
357 trace_netfs_rreq(rreq, netfs_rreq_trace_all_queued);
358}
359
360/*
361 * Query if all subrequests are queued.
362 */
363static inline bool netfs_are_all_subreqs_queued(const struct netfs_io_request *rreq)
364{
365 /* Read lists after ALL_QUEUED. */
366 return test_bit_acquire(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
367}
368
369/*
370 * fscache-cache.c
371 */
372#ifdef CONFIG_PROC_FS
373extern const struct seq_operations fscache_caches_seq_ops;
374#endif
375bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why);
376void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why);
377struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache);
378void fscache_put_cache(struct fscache_cache *cache, enum fscache_cache_trace where);
379
380static inline enum fscache_cache_state fscache_cache_state(const struct fscache_cache *cache)
381{
382 return smp_load_acquire(&cache->state);
383}
384
385static inline bool fscache_cache_is_live(const struct fscache_cache *cache)
386{
387 return fscache_cache_state(cache) == FSCACHE_CACHE_IS_ACTIVE;
388}
389
390static inline void fscache_set_cache_state(struct fscache_cache *cache,
391 enum fscache_cache_state new_state)
392{
393 smp_store_release(&cache->state, new_state);
394
395}
396
397static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache,
398 enum fscache_cache_state old_state,
399 enum fscache_cache_state new_state)
400{
401 return try_cmpxchg_release(&cache->state, &old_state, new_state);
402}
403
404/*
405 * fscache-cookie.c
406 */
407extern struct kmem_cache *fscache_cookie_jar;
408#ifdef CONFIG_PROC_FS
409extern const struct seq_operations fscache_cookies_seq_ops;
410#endif
411extern struct timer_list fscache_cookie_lru_timer;
412
413extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix);
414extern bool fscache_begin_cookie_access(struct fscache_cookie *cookie,
415 enum fscache_access_trace why);
416
417static inline void fscache_see_cookie(struct fscache_cookie *cookie,
418 enum fscache_cookie_trace where)
419{
420 trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref),
421 where);
422}
423
424/*
425 * fscache-main.c
426 */
427extern unsigned int fscache_hash(unsigned int salt, const void *data, size_t len);
428#ifdef CONFIG_FSCACHE
429int __init fscache_init(void);
430void __exit fscache_exit(void);
431#else
432static inline int fscache_init(void) { return 0; }
433static inline void fscache_exit(void) {}
434#endif
435
436/*
437 * fscache-proc.c
438 */
439#ifdef CONFIG_PROC_FS
440extern int __init fscache_proc_init(void);
441extern void fscache_proc_cleanup(void);
442#else
443#define fscache_proc_init() (0)
444#define fscache_proc_cleanup() do {} while (0)
445#endif
446
447/*
448 * fscache-stats.c
449 */
450#ifdef CONFIG_FSCACHE_STATS
451extern atomic_t fscache_n_volumes;
452extern atomic_t fscache_n_volumes_collision;
453extern atomic_t fscache_n_volumes_nomem;
454extern atomic_t fscache_n_cookies;
455extern atomic_t fscache_n_cookies_lru;
456extern atomic_t fscache_n_cookies_lru_expired;
457extern atomic_t fscache_n_cookies_lru_removed;
458extern atomic_t fscache_n_cookies_lru_dropped;
459
460extern atomic_t fscache_n_acquires;
461extern atomic_t fscache_n_acquires_ok;
462extern atomic_t fscache_n_acquires_oom;
463
464extern atomic_t fscache_n_invalidates;
465
466extern atomic_t fscache_n_relinquishes;
467extern atomic_t fscache_n_relinquishes_retire;
468extern atomic_t fscache_n_relinquishes_dropped;
469
470extern atomic_t fscache_n_resizes;
471extern atomic_t fscache_n_resizes_null;
472
473static inline void fscache_stat(atomic_t *stat)
474{
475 atomic_inc(stat);
476}
477
478static inline void fscache_stat_d(atomic_t *stat)
479{
480 atomic_dec(stat);
481}
482
483#define __fscache_stat(stat) (stat)
484
485int fscache_stats_show(struct seq_file *m);
486#else
487
488#define __fscache_stat(stat) (NULL)
489#define fscache_stat(stat) do {} while (0)
490#define fscache_stat_d(stat) do {} while (0)
491
492static inline int fscache_stats_show(struct seq_file *m) { return 0; }
493#endif
494
495/*
496 * fscache-volume.c
497 */
498#ifdef CONFIG_PROC_FS
499extern const struct seq_operations fscache_volumes_seq_ops;
500#endif
501
502struct fscache_volume *fscache_get_volume(struct fscache_volume *volume,
503 enum fscache_volume_trace where);
504bool fscache_begin_volume_access(struct fscache_volume *volume,
505 struct fscache_cookie *cookie,
506 enum fscache_access_trace why);
507void fscache_create_volume(struct fscache_volume *volume, bool wait);
508
509/*****************************************************************************/
510/*
511 * debug tracing
512 */
513#define dbgprintk(FMT, ...) \
514 printk("[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__)
515
516#define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__)
517#define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__)
518#define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__)
519
520#ifdef __KDEBUG
521#define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__)
522#define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__)
523#define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__)
524
525#elif defined(CONFIG_NETFS_DEBUG)
526#define _enter(FMT, ...) \
527do { \
528 if (netfs_debug) \
529 kenter(FMT, ##__VA_ARGS__); \
530} while (0)
531
532#define _leave(FMT, ...) \
533do { \
534 if (netfs_debug) \
535 kleave(FMT, ##__VA_ARGS__); \
536} while (0)
537
538#define _debug(FMT, ...) \
539do { \
540 if (netfs_debug) \
541 kdebug(FMT, ##__VA_ARGS__); \
542} while (0)
543
544#else
545#define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__)
546#define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__)
547#define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__)
548#endif
549
550/*
551 * assertions
552 */
553#if 1 /* defined(__KDEBUGALL) */
554
555#define ASSERT(X) \
556do { \
557 if (unlikely(!(X))) { \
558 pr_err("\n"); \
559 pr_err("Assertion failed\n"); \
560 BUG(); \
561 } \
562} while (0)
563
564#define ASSERTCMP(X, OP, Y) \
565do { \
566 if (unlikely(!((X) OP (Y)))) { \
567 pr_err("\n"); \
568 pr_err("Assertion failed\n"); \
569 pr_err("%lx " #OP " %lx is false\n", \
570 (unsigned long)(X), (unsigned long)(Y)); \
571 BUG(); \
572 } \
573} while (0)
574
575#define ASSERTIF(C, X) \
576do { \
577 if (unlikely((C) && !(X))) { \
578 pr_err("\n"); \
579 pr_err("Assertion failed\n"); \
580 BUG(); \
581 } \
582} while (0)
583
584#define ASSERTIFCMP(C, X, OP, Y) \
585do { \
586 if (unlikely((C) && !((X) OP (Y)))) { \
587 pr_err("\n"); \
588 pr_err("Assertion failed\n"); \
589 pr_err("%lx " #OP " %lx is false\n", \
590 (unsigned long)(X), (unsigned long)(Y)); \
591 BUG(); \
592 } \
593} while (0)
594
595#else
596
597#define ASSERT(X) do {} while (0)
598#define ASSERTCMP(X, OP, Y) do {} while (0)
599#define ASSERTIF(C, X) do {} while (0)
600#define ASSERTIFCMP(C, X, OP, Y) do {} while (0)
601
602#endif /* assert or not */