diff --git a/fs/9p/v9fs.h b/fs/9p/v9fs.h index a462bcbfc7da..2ac863dc8446 100644 --- a/fs/9p/v9fs.h +++ b/fs/9p/v9fs.h @@ -137,6 +137,7 @@ struct v9fs_session_info { struct list_head slist; /* list of sessions registered with v9fs */ struct rw_semaphore rename_sem; long session_lock_timeout; /* retry interval for blocking locks */ + struct rcu_head rcu; /* v9fs_vfs_get_link_dotl() reads ->cache in rcu pathwalk */ }; #define NDENTRY_TIMEOUT_NEVER (-1U) diff --git a/fs/9p/vfs_super.c b/fs/9p/vfs_super.c index 94d6b02c221b..79526044685c 100644 --- a/fs/9p/vfs_super.c +++ b/fs/9p/vfs_super.c @@ -173,8 +173,8 @@ static void v9fs_kill_super(struct super_block *s) v9fs_session_cancel(v9ses); v9fs_session_close(v9ses); - kfree(v9ses); - s->s_fs_info = NULL; + /* v9fs_vfs_get_link_dotl() may still read ->cache in rcu pathwalk */ + kfree_rcu(v9ses, rcu); p9_debug(P9_DEBUG_VFS, "exiting kill_super\n"); } diff --git a/fs/afs/internal.h b/fs/afs/internal.h index 330654ed16ec..3282d5d6e1c0 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -251,6 +251,8 @@ struct afs_super_info { struct afs_volume *volume; /* volume record */ enum afs_flock_mode flock_mode:8; /* File locking emulation mode */ bool dyn_root; /* True if dynamic root */ + /* afs_atcell_get_link() reads ->net_ns in rcu pathwalk */ + struct rcu_head rcu; }; static inline struct afs_super_info *AFS_FS_S(struct super_block *sb) diff --git a/fs/afs/super.c b/fs/afs/super.c index 82bb713825a0..33b195821e36 100644 --- a/fs/afs/super.c +++ b/fs/afs/super.c @@ -523,7 +523,7 @@ static void afs_destroy_sbi(struct afs_super_info *as) afs_put_volume(as->volume, afs_volume_trace_put_destroy_sbi); afs_unuse_cell(as->cell, afs_cell_trace_unuse_sbi); put_net(as->net_ns); - kfree(as); + kfree_rcu(as, rcu); } } diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 724a27e8be61..af70bb26bb54 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -1774,6 +1774,8 @@ struct ext4_sb_info { struct list_head s_es_list; /* List of inodes with reclaimable extents */ long s_es_nr_inode; struct ext4_es_stats s_es_stats; + /* ext4_get_link() reads the sbi in rcu pathwalk */ + struct rcu_head s_rcu; struct mb_cache *s_ea_block_cache; struct mb_cache *s_ea_inode_cache; diff --git a/fs/ext4/extents-test.c b/fs/ext4/extents-test.c index c3836ecb89f9..2e997f228330 100644 --- a/fs/ext4/extents-test.c +++ b/fs/ext4/extents-test.c @@ -151,6 +151,7 @@ static void extents_kunit_exit(struct kunit *test) sbi = k_ctx.k_ei->vfs_inode.i_sb->s_fs_info; ext4_es_unregister_shrinker(sbi); + ext4_es_destroy_stats(sbi); deactivate_super(sbi->s_sb); kfree(sbi); kfree(k_ctx.k_ei); @@ -340,6 +341,7 @@ out: k_ctx.k_data = NULL; ext4_es_unregister_shrinker(sbi); + ext4_es_destroy_stats(sbi); out_deactivate: deactivate_locked_super(sb); kfree(sbi); diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 76038b6c3655..3bf0a6cdbec2 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -6409,6 +6409,7 @@ EXPORT_SYMBOL_FOR_EXT4_TEST(__ext4_ext_dirty); EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_zeroout); EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_register_shrinker); EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_unregister_shrinker); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_destroy_stats); EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_map_create_blocks); EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_init_tree); EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_lookup_extent); diff --git a/fs/ext4/extents_status.c b/fs/ext4/extents_status.c index 6e4a191e8219..018b6146b9d8 100644 --- a/fs/ext4/extents_status.c +++ b/fs/ext4/extents_status.c @@ -1881,12 +1881,17 @@ err1: } void ext4_es_unregister_shrinker(struct ext4_sb_info *sbi) +{ + shrinker_free(sbi->s_es_shrinker); +} + +/* ext4_get_link() bumps the hit and miss counters in rcu pathwalk */ +void ext4_es_destroy_stats(struct ext4_sb_info *sbi) { percpu_counter_destroy(&sbi->s_es_stats.es_stats_cache_hits); percpu_counter_destroy(&sbi->s_es_stats.es_stats_cache_misses); percpu_counter_destroy(&sbi->s_es_stats.es_stats_all_cnt); percpu_counter_destroy(&sbi->s_es_stats.es_stats_shk_cnt); - shrinker_free(sbi->s_es_shrinker); } /* diff --git a/fs/ext4/extents_status.h b/fs/ext4/extents_status.h index f3396cf32b44..01cddcaaa68e 100644 --- a/fs/ext4/extents_status.h +++ b/fs/ext4/extents_status.h @@ -238,6 +238,7 @@ static inline void ext4_es_store_pblock_status(struct extent_status *es, extern int ext4_es_register_shrinker(struct ext4_sb_info *sbi); extern void ext4_es_unregister_shrinker(struct ext4_sb_info *sbi); +extern void ext4_es_destroy_stats(struct ext4_sb_info *sbi); extern int ext4_seq_es_shrinker_info_show(struct seq_file *seq, void *v); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index d636b89090a2..3ace06b4f9e3 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1279,6 +1279,14 @@ static void ext4_flex_groups_free(struct ext4_sb_info *sbi) } } +static void ext4_put_sbi_rcu(struct rcu_head *head) +{ + struct ext4_sb_info *sbi = container_of(head, struct ext4_sb_info, s_rcu); + + ext4_es_destroy_stats(sbi); + kfree(sbi); +} + static void ext4_put_super(struct super_block *sb) { struct ext4_sb_info *sbi = EXT4_SB(sb); @@ -1374,7 +1382,6 @@ static void ext4_put_super(struct super_block *sb) ext4_stop_mmpd(sbi); brelse(sbi->s_sbh); - sb->s_fs_info = NULL; /* * Now that we are completely done shutting down the * superblock, we need to actually destroy the kobject. @@ -1387,7 +1394,8 @@ static void ext4_put_super(struct super_block *sb) #if IS_ENABLED(CONFIG_UNICODE) utf8_unload(sb->s_encoding); #endif - kfree(sbi); + /* ext4_get_link() reads the sbi and the counters in rcu pathwalk */ + call_rcu(&sbi->s_rcu, ext4_put_sbi_rcu); } static struct kmem_cache *ext4_inode_cachep; @@ -5806,6 +5814,7 @@ failed_mount3a: /* Drain deferred EA inode iputs from journal replay */ flush_delayed_work(&sbi->s_ea_inode_work); ext4_es_unregister_shrinker(sbi); + ext4_es_destroy_stats(sbi); failed_mount3: /* flush s_sb_upd_work before sbi destroy */ flush_work(&sbi->s_sb_upd_work); diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c index eb9152e9fa22..14ca54e54578 100644 --- a/fs/ntfs3/dir.c +++ b/fs/ntfs3/dir.c @@ -186,11 +186,20 @@ int ntfs_nls_to_utf16(struct ntfs_sb_info *sbi, const u8 *name, u32 name_len, { int ret, slen, i; const u8 *end; - struct nls_table *nls = sbi->options->nls; + struct ntfs_mount_options *opts; + struct nls_table *nls; + bool ads; u16 *uname = uni->name; static_assert(sizeof(wchar_t) == sizeof(u16)); + /* ntfs_fs_reconfigure() may replace the options, but not ->nls */ + rcu_read_lock(); + opts = READ_ONCE(sbi->options); + nls = opts->nls; + ads = opts->ads; + rcu_read_unlock(); + if (!nls) { /* utf8 -> utf16 */ ret = _utf8s_to_utf16s(name, name_len, endian, uname, max_ulen); @@ -230,7 +239,7 @@ int ntfs_nls_to_utf16(struct ntfs_sb_info *sbi, const u8 *name, u32 name_len, uni->len = ret; uni->ads_len = 0; - if (ret > 0 && sbi->options->ads) { + if (ret > 0 && ads) { uname = uni->name; /* Find delimiter in range [1 : ret-2). */ for (i = 1; i + 1 < ret; i++) { diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h index 5811d89d67b3..44d59b444648 100644 --- a/fs/ntfs3/ntfs_fs.h +++ b/fs/ntfs3/ntfs_fs.h @@ -91,6 +91,8 @@ enum utf16_endian; struct ntfs_mount_options { char *nls_name; struct nls_table *nls; + /* ->d_hash() and ->d_compare() read the options in rcu pathwalk */ + struct rcu_head rcu; kuid_t fs_uid; kgid_t fs_gid; @@ -211,6 +213,7 @@ struct ntfs_index { /* Ntfs file system in-core superblock data. */ struct ntfs_sb_info { struct super_block *sb; + struct rcu_head rcu; u32 discard_granularity; u64 discard_granularity_mask_inv; // ~(discard_granularity_mask_inv-1) diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index f4a42a0c73a4..58ef6be3286b 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -196,7 +196,7 @@ void *ntfs_set_shared(void *ptr, u32 bytes) void *ret = NULL; int i, j = -1; - spin_lock(&s_shared_lock); + spin_lock_bh(&s_shared_lock); for (i = 0; i < ARRAY_SIZE(s_shared); i++) { if (!s_shared[i].cnt) { j = i; @@ -214,7 +214,7 @@ void *ntfs_set_shared(void *ptr, u32 bytes) s_shared[j].cnt = 1; ret = ptr; } - spin_unlock(&s_shared_lock); + spin_unlock_bh(&s_shared_lock); return ret; } @@ -231,7 +231,7 @@ void *ntfs_put_shared(void *ptr) void *ret = ptr; int i; - spin_lock(&s_shared_lock); + spin_lock_bh(&s_shared_lock); for (i = 0; i < ARRAY_SIZE(s_shared); i++) { if (s_shared[i].cnt && s_shared[i].ptr == ptr) { if (--s_shared[i].cnt) @@ -239,7 +239,7 @@ void *ntfs_put_shared(void *ptr) break; } } - spin_unlock(&s_shared_lock); + spin_unlock_bh(&s_shared_lock); return ret; } @@ -251,6 +251,11 @@ static inline void put_mount_options(struct ntfs_mount_options *options) kfree(options); } +static void put_mount_options_rcu(struct rcu_head *head) +{ + put_mount_options(container_of(head, struct ntfs_mount_options, rcu)); +} + enum Opt { Opt_uid, Opt_gid, @@ -444,6 +449,7 @@ static int ntfs_fs_reconfigure(struct fs_context *fc) struct super_block *sb = fc->root->d_sb; struct ntfs_sb_info *sbi = sb->s_fs_info; struct ntfs_mount_options *new_opts = fc->fs_private; + struct ntfs_mount_options *old_opts; int ro_rw; ro_rw = sb_rdonly(sb) && !(fc->sb_flags & SB_RDONLY); @@ -473,7 +479,15 @@ static int ntfs_fs_reconfigure(struct fs_context *fc) } sync_filesystem(sb); - swap(sbi->options, fc->fs_private); + old_opts = sbi->options; + /* pairs with the READ_ONCE() in ntfs_nls_to_utf16() */ + smp_store_release(&sbi->options, new_opts); + fc->fs_private = NULL; + /* + * ntfs_d_hash() and ntfs_d_compare() of a pathwalk in rcu mode may + * still read the old options through the sbi. + */ + call_rcu(&old_opts->rcu, put_mount_options_rcu); return 0; } @@ -708,6 +722,8 @@ static noinline void ntfs3_put_sbi(struct ntfs_sb_info *sbi) static void ntfs3_free_sbi(struct ntfs_sb_info *sbi) { + if (sbi->options) + put_mount_options(sbi->options); kfree(sbi->new_rec); kvfree(ntfs_put_shared(sbi->upcase)); kvfree(sbi->def_table); @@ -719,6 +735,11 @@ static void ntfs3_free_sbi(struct ntfs_sb_info *sbi) kfree(sbi); } +static void ntfs3_free_sbi_rcu(struct rcu_head *head) +{ + ntfs3_free_sbi(container_of(head, struct ntfs_sb_info, rcu)); +} + static void ntfs_put_super(struct super_block *sb) { struct ntfs_sb_info *sbi = sb->s_fs_info; @@ -728,11 +749,11 @@ static void ntfs_put_super(struct super_block *sb) /* Mark rw ntfs as clear, if possible. */ ntfs_set_state(sbi, NTFS_DIRTY_CLEAR); - if (sbi->options) { - put_mount_options(sbi->options); - sbi->options = NULL; - } - + /* + * The mount options stay until the sbi is freed: ->d_hash() and + * ->d_compare() of a pathwalk in rcu mode read the nls table through + * them. + */ ntfs3_put_sbi(sbi); } @@ -1939,9 +1960,11 @@ static void ntfs3_kill_sb(struct super_block *sb) kill_block_super(sb); - if (sbi->options) - put_mount_options(sbi->options); - ntfs3_free_sbi(sbi); + /* + * A pathwalk in rcu mode may still be in ->d_hash() or ->d_compare() + * and read the upcase table and the nls table through the sbi. + */ + call_rcu(&sbi->rcu, ntfs3_free_sbi_rcu); } // clang-format off diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 6e3513b13cfa..090aca946717 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -91,6 +91,14 @@ static void free_ei_rcu(struct rcu_head *rcu) } } +/* read under eventfs_srcu and, by tracefs_d_revalidate(), in rcu pathwalk */ +static void free_ei_srcu(struct rcu_head *rcu) +{ + struct eventfs_inode *ei = container_of(rcu, struct eventfs_inode, rcu); + + call_rcu(&ei->rcu, free_ei_rcu); +} + /* * eventfs_inode reference count management. * @@ -112,7 +120,7 @@ static void release_ei(struct kref *ref) entry->release(entry->name, ei->data); } - call_srcu(&eventfs_srcu, &ei->rcu, free_ei_rcu); + call_srcu(&eventfs_srcu, &ei->rcu, free_ei_srcu); } static inline void put_ei(struct eventfs_inode *ei) diff --git a/fs/unicode/utf8-core.c b/fs/unicode/utf8-core.c index f313532f5e80..74a74a48dd8b 100644 --- a/fs/unicode/utf8-core.c +++ b/fs/unicode/utf8-core.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -183,12 +184,24 @@ out_free_um: } EXPORT_SYMBOL(utf8_load); +static void utf8_unload_rcu(struct rcu_head *head) +{ + struct unicode_map *um = container_of(head, struct unicode_map, rcu); + + symbol_put(utf8_data_table); + kfree(um); +} + +/* + * A pathwalk in rcu mode may still be in ->d_hash() or ->d_compare() of a + * filesystem that is being torn down and read the map through its + * superblock, so the map and the tables it points to have to outlive the + * grace period. + */ void utf8_unload(struct unicode_map *um) { - if (um) { - symbol_put(utf8_data_table); - kfree(um); - } + if (um) + call_rcu(&um->rcu, utf8_unload_rcu); } EXPORT_SYMBOL(utf8_unload); diff --git a/include/linux/unicode.h b/include/linux/unicode.h index 64fa44fe180c..3a8cb603eb93 100644 --- a/include/linux/unicode.h +++ b/include/linux/unicode.h @@ -53,6 +53,7 @@ struct unicode_map { unsigned int version; const struct utf8data *ntab[UTF8_NMAX]; const struct utf8data_table *tables; + struct rcu_head rcu; }; int utf8_validate(const struct unicode_map *um, const struct qstr *str); diff --git a/ipc/namespace.c b/ipc/namespace.c index 1e71353bdb4a..75aa7b469348 100644 --- a/ipc/namespace.c +++ b/ipc/namespace.c @@ -96,7 +96,7 @@ fail_ipc: fail_mq_sysctls: retire_mq_sysctls(ns); fail_mq_mount: - mntput(ns->mq_mnt); + kern_unmount(ns->mq_mnt); fail_put: put_user_ns(ns->user_ns); ns_common_free(ns);