kernel/linux
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/ synced 2026-10-09 18:15:49 +00:00
Code Issues Releases Packages Activity
linux/drivers/md/bcache
History
Mingzhe Zou f9ba5dd0d9 bcache: fixup lock c->root error 
commit e34820f984 upstream.

We had a problem with io hung because it was waiting for c->root to
release the lock.

crash> cache_set.root -l cache_set.list ffffa03fde4c0050
  root = 0xffff802ef454c800
crash> btree -o 0xffff802ef454c800 | grep rw_semaphore
  [ffff802ef454c858] struct rw_semaphore lock;
crash> struct rw_semaphore ffff802ef454c858
struct rw_semaphore {
  count = {
    counter = -4294967297
  },
  wait_list = {
    next = 0xffff00006786fc28,
    prev = 0xffff00005d0efac8
  },
  wait_lock = {
    raw_lock = {
      {
        val = {
          counter = 0
        },
        {
          locked = 0 '\000',
          pending = 0 '\000'
        },
        {
          locked_pending = 0,
          tail = 0
        }
      }
    }
  },
  osq = {
    tail = {
      counter = 0
    }
  },
  owner = 0xffffa03fdc586603
}

The "counter = -4294967297" means that lock count is -1 and a write lock
is being attempted. Then, we found that there is a btree with a counter
of 1 in btree_cache_freeable.

crash> cache_set -l cache_set.list ffffa03fde4c0050 -o|grep btree_cache
  [ffffa03fde4c1140] struct list_head btree_cache;
  [ffffa03fde4c1150] struct list_head btree_cache_freeable;
  [ffffa03fde4c1160] struct list_head btree_cache_freed;
  [ffffa03fde4c1170] unsigned int btree_cache_used;
  [ffffa03fde4c1178] wait_queue_head_t btree_cache_wait;
  [ffffa03fde4c1190] struct task_struct *btree_cache_alloc_lock;
crash> list -H ffffa03fde4c1140|wc -l
973
crash> list -H ffffa03fde4c1150|wc -l
1123
crash> cache_set.btree_cache_used -l cache_set.list ffffa03fde4c0050
  btree_cache_used = 2097
crash> list -s btree -l btree.list -H ffffa03fde4c1140|grep -E -A2 "^  lock = {" > btree_cache.txt
crash> list -s btree -l btree.list -H ffffa03fde4c1150|grep -E -A2 "^  lock = {" > btree_cache_freeable.txt
[root@node-3 127.0.0.1-2023-08-04-16:40:28]# pwd
/var/crash/127.0.0.1-2023-08-04-16:40:28
[root@node-3 127.0.0.1-2023-08-04-16:40:28]# cat btree_cache.txt|grep counter|grep -v "counter = 0"
[root@node-3 127.0.0.1-2023-08-04-16:40:28]# cat btree_cache_freeable.txt|grep counter|grep -v "counter = 0"
      counter = 1

We found that this is a bug in bch_sectors_dirty_init() when locking c->root:
    (1). Thread X has locked c->root(A) write.
    (2). Thread Y failed to lock c->root(A), waiting for the lock(c->root A).
    (3). Thread X bch_btree_set_root() changes c->root from A to B.
    (4). Thread X releases the lock(c->root A).
    (5). Thread Y successfully locks c->root(A).
    (6). Thread Y releases the lock(c->root B).

        down_write locked ---(1)----------------------┐
                |                                     |
                |   down_read waiting ---(2)----┐     |
                |           |               ┌-------------┐ ┌-------------┐
        bch_btree_set_root ===(3)========>> | c->root   A | | c->root   B |
                |           |               └-------------┘ └-------------┘
            up_write ---(4)---------------------┘     |            |
                            |                         |            |
                    down_read locked ---(5)-----------┘            |
                            |                                      |
                        up_read ---(6)-----------------------------┘

Since c->root may change, the correct steps to lock c->root should be
the same as bch_root_usage(), compare after locking.

static unsigned int bch_root_usage(struct cache_set *c)
{
        unsigned int bytes = 0;
        struct bkey *k;
        struct btree *b;
        struct btree_iter iter;

        goto lock_root;

        do {
                rw_unlock(false, b);
lock_root:
                b = c->root;
                rw_lock(false, b, b->level);
        } while (b != c->root);

        for_each_key_filter(&b->keys, k, &iter, bch_ptr_bad)
                bytes += bkey_bytes(k);

        rw_unlock(false, b);

        return (bytes * 100) / btree_bytes(c);
}

Fixes: b144e45fc5 ("bcache: make bch_sectors_dirty_init() to be multithreaded")
Signed-off-by: Mingzhe Zou <mingzhe.zou@easystack.cn>
Cc:  <stable@vger.kernel.org>
Signed-off-by: Coly Li <colyli@suse.de>
Link: https://lore.kernel.org/r/20231120052503.6122-7-colyli@suse.de
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
History 2023-12-03 07:32:12 +01:00
..
alloc.c block: decouple REQ_OP_SECURE_ERASE from REQ_OP_DISCARD 2022-04-17 19:49:59 -06:00
bcache.h bcache:: fix repeated words in comments 2022-09-19 11:12:35 -06:00
bcache_ondisk.h bcache: Silence memcpy() run-time false positive warnings 2023-02-09 11:28:14 +01:00
bset.c bcache: bset: Fix comment typos 2022-09-19 11:12:35 -06:00
bset.h bcache: move uapi header bcache.h to bcache code directory 2021-10-29 06:43:21 -06:00
btree.c bcache: check return value from btree_node_alloc_replacement() 2023-12-03 07:32:12 +01:00
btree.h bcache: fixup btree_cache_wait list damage 2023-07-19 16:22:10 +02:00
closure.c bcache: Convert to DEFINE_SHOW_ATTRIBUTE 2020-10-02 14:25:29 -06:00
closure.h bcache: fix typo in code comments of closure_return_with_destructor() 2018-10-08 08:19:43 -06:00
debug.c block: turn bio_kmalloc into a simple kmalloc wrapper 2022-04-17 19:30:41 -06:00
debug.h bcache: add identifier names to arguments of function definitions 2018-08-11 15:46:41 -06:00
extents.c bcache: remove PTR_CACHE 2021-04-11 08:37:55 -06:00
extents.h bcache: add identifier names to arguments of function definitions 2018-08-11 15:46:41 -06:00
features.c bcache: move uapi header bcache.h to bcache code directory 2021-10-29 06:43:21 -06:00
features.h bcache: move uapi header bcache.h to bcache code directory 2021-10-29 06:43:21 -06:00
io.c block: pass a block_device and opf to bio_init 2022-02-02 07:49:59 -07:00
journal.c bcache: Silence memcpy() run-time false positive warnings 2023-02-09 11:28:14 +01:00
journal.h bcache: avoid journal no-space deadlock by reserving 1 journal bucket 2022-05-24 06:19:33 -06:00
Kconfig bcache: remove EXPERIMENTAL for Kconfig option 'Asynchronous device registration' 2022-08-02 17:22:41 -06:00
Makefile bcache: add sysfs file to display feature sets information of cache set 2020-07-25 07:38:21 -06:00
movinggc.c block: pass a block_device and opf to bio_init 2022-02-02 07:49:59 -07:00
request.c treewide: use prandom_u32_max() when possible, part 1 2022-10-11 17:42:55 -06:00
request.h block: switch polling to be bio based 2021-10-18 06:17:36 -06:00
stats.c bcache: use default_groups in kobj_type 2022-01-26 15:56:18 +01:00
stats.h bcache: remove unused bch_mark_cache_readahead function def in stats.h 2022-09-19 11:12:35 -06:00
super.c bcache: Remove unnecessary NULL point check in node allocations 2023-07-19 16:22:10 +02:00
sysfs.c bcache: prevent potential division by zero error 2023-12-03 07:32:12 +01:00
sysfs.h bcache: use default_groups in kobj_type 2022-01-26 15:56:18 +01:00
trace.c License cleanup: add SPDX GPL-2.0 license identifier to files with no license 2017-11-02 11:10:55 +01:00
util.c treewide: Use fallthrough pseudo-keyword 2020-08-23 17:36:59 -05:00
util.h for-5.16/bdev-size-2021-10-29 2021-11-01 09:50:37 -07:00
writeback.c bcache: fixup lock c->root error 2023-12-03 07:32:12 +01:00
writeback.h bcache: improve multithreaded bch_sectors_dirty_init() 2022-05-24 06:19:33 -06:00