diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 9514c2e5f01e..fac9337dc543 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -718,6 +718,86 @@ void bch2_do_discards(struct bch_fs *c) percpu_ref_put(&c->writes); } +static int invalidate_one_bucket(struct btree_trans *trans, struct bch_dev *ca) +{ + struct bch_fs *c = trans->c; + struct btree_iter lru_iter, alloc_iter = { NULL }; + struct bkey_s_c k; + struct bkey_i_alloc_v4 *a; + u64 bucket, idx; + int ret; + + bch2_trans_iter_init(trans, &lru_iter, BTREE_ID_lru, + POS(ca->dev_idx, 0), 0); + k = bch2_btree_iter_peek(&lru_iter); + ret = bkey_err(k); + if (ret) + goto out; + + if (!k.k || k.k->p.inode != ca->dev_idx) + goto out; + + if (bch2_fs_inconsistent_on(k.k->type != KEY_TYPE_lru, c, + "non lru key in lru btree")) + goto out; + + idx = k.k->p.offset; + bucket = le64_to_cpu(bkey_s_c_to_lru(k).v->idx); + + a = bch2_trans_start_alloc_update(trans, &alloc_iter, + POS(ca->dev_idx, bucket)); + ret = PTR_ERR_OR_ZERO(a); + if (ret) + goto out; + + if (bch2_fs_inconsistent_on(idx != alloc_lru_idx(a->v), c, + "invalidating bucket with wrong lru idx (got %llu should be %llu", + idx, alloc_lru_idx(a->v))) + goto out; + + SET_BCH_ALLOC_V4_NEED_INC_GEN(&a->v, false); + a->v.gen++; + a->v.data_type = 0; + a->v.dirty_sectors = 0; + a->v.cached_sectors = 0; + a->v.io_time[READ] = atomic64_read(&c->io_clock[READ].now); + a->v.io_time[WRITE] = atomic64_read(&c->io_clock[WRITE].now); + + ret = bch2_trans_update(trans, &alloc_iter, &a->k_i, + BTREE_TRIGGER_BUCKET_INVALIDATE); +out: + bch2_trans_iter_exit(trans, &alloc_iter); + bch2_trans_iter_exit(trans, &lru_iter); + return ret; +} + +static void bch2_do_invalidates_work(struct work_struct *work) +{ + struct bch_fs *c = container_of(work, struct bch_fs, invalidate_work); + struct bch_dev *ca; + struct btree_trans trans; + unsigned i; + int ret = 0; + + bch2_trans_init(&trans, c, 0, 0); + + for_each_member_device(ca, c, i) + while (!ret && should_invalidate_buckets(ca)) + ret = __bch2_trans_do(&trans, NULL, NULL, + BTREE_INSERT_USE_RESERVE| + BTREE_INSERT_NOFAIL, + invalidate_one_bucket(&trans, ca)); + + bch2_trans_exit(&trans); + percpu_ref_put(&c->writes); +} + +void bch2_do_invalidates(struct bch_fs *c) +{ + if (percpu_ref_tryget(&c->writes)) + queue_work(system_long_wq, &c->invalidate_work); +} + static int bch2_dev_freespace_init(struct bch_fs *c, struct bch_dev *ca) { struct btree_trans trans; @@ -1002,4 +1082,5 @@ void bch2_fs_allocator_background_init(struct bch_fs *c) { spin_lock_init(&c->freelist_lock); INIT_WORK(&c->discard_work, bch2_do_discards_work); + INIT_WORK(&c->invalidate_work, bch2_do_invalidates_work); } diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index 8ba9bf853c2f..d4883d3cd642 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -115,6 +115,17 @@ int bch2_trans_mark_alloc(struct btree_trans *, struct bkey_s_c, struct bkey_i *, unsigned); void bch2_do_discards(struct bch_fs *); +static inline bool should_invalidate_buckets(struct bch_dev *ca) +{ + struct bch_dev_usage u = bch2_dev_usage_read(ca); + + return u.d[BCH_DATA_cached].buckets && + u.buckets_unavailable + u.d[BCH_DATA_cached].buckets < + ca->mi.nbuckets >> 7; +} + +void bch2_do_invalidates(struct bch_fs *); + int bch2_fs_freespace_init(struct bch_fs *); void bch2_recalc_capacity(struct bch_fs *); diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index ca48b3f86304..66d9c209252e 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -759,6 +759,7 @@ struct bch_fs { struct buckets_waiting_for_journal buckets_waiting_for_journal; struct work_struct discard_work; + struct work_struct invalidate_work; /* GARBAGE COLLECTION */ struct task_struct *gc_thread; diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 0e86b45b6c55..bfab5d88550b 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -548,6 +548,11 @@ int bch2_mark_alloc(struct btree_trans *trans, !new_a.journal_seq) bch2_do_discards(c); + if (!old_a.data_type && + new_a.data_type && + should_invalidate_buckets(ca)) + bch2_do_invalidates(c); + if (bucket_state(new_a) == BUCKET_need_gc_gens) { atomic_inc(&c->kick_gc); wake_up_process(c->gc_thread);