Skip to content

Commit 6e80d4f

Browse files
dennisszhoukdave
authored andcommitted
btrfs: handle empty block_group removal for async discard
block_group removal is a little tricky. It can race with the extent allocator, the cleaner thread, and balancing. The current path is for a block_group to be added to the unused_bgs list. Then, when the cleaner thread comes around, it starts a transaction and then proceeds with removing the block_group. Extents that are pinned are subsequently removed from the pinned trees and then eventually a discard is issued for the entire block_group. Async discard introduces another player into the game, the discard workqueue. While it has none of the racing issues, the new problem is ensuring we don't leave free space untrimmed prior to forgetting the block_group. This is handled by placing fully free block_groups on a separate discard queue. This is necessary to maintain discarding order as in the future we will slowly trim even fully free block_groups. The ordering helps us make progress on the same block_group rather than say the last fully freed block_group or needing to search through the fully freed block groups at the beginning of a list and insert after. The new order of events is a fully freed block group gets placed on the unused discard queue first. Once it's processed, it will be placed on the unusued_bgs list and then the original sequence of events will happen, just without the final whole block_group discard. The mount flags can change when processing unused_bgs, so when flipping from DISCARD to DISCARD_ASYNC, the unused_bgs must be punted to the discard_list to be trimmed. If we flip off DISCARD_ASYNC, we punt free block groups on the discard_list to the unused_bg queue which will do the final discard for us. Reviewed-by: Josef Bacik <[email protected]> Signed-off-by: Dennis Zhou <[email protected]> Reviewed-by: David Sterba <[email protected]> Signed-off-by: David Sterba <[email protected]>
1 parent b0643e5 commit 6e80d4f

File tree

7 files changed

+206
-9
lines changed

7 files changed

+206
-9
lines changed

fs/btrfs/block-group.c

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1246,6 +1246,7 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
12461246
struct btrfs_block_group *block_group;
12471247
struct btrfs_space_info *space_info;
12481248
struct btrfs_trans_handle *trans;
1249+
const bool async_trim_enabled = btrfs_test_opt(fs_info, DISCARD_ASYNC);
12491250
int ret = 0;
12501251

12511252
if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
@@ -1275,6 +1276,22 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
12751276

12761277
/* Don't want to race with allocators so take the groups_sem */
12771278
down_write(&space_info->groups_sem);
1279+
1280+
/*
1281+
* Async discard moves the final block group discard to be prior
1282+
* to the unused_bgs code path. Therefore, if it's not fully
1283+
* trimmed, punt it back to the async discard lists.
1284+
*/
1285+
if (btrfs_test_opt(fs_info, DISCARD_ASYNC) &&
1286+
!btrfs_is_free_space_trimmed(block_group)) {
1287+
trace_btrfs_skip_unused_block_group(block_group);
1288+
up_write(&space_info->groups_sem);
1289+
/* Requeue if we failed because of async discard */
1290+
btrfs_discard_queue_work(&fs_info->discard_ctl,
1291+
block_group);
1292+
goto next;
1293+
}
1294+
12781295
spin_lock(&block_group->lock);
12791296
if (block_group->reserved || block_group->pinned ||
12801297
block_group->used || block_group->ro ||
@@ -1378,6 +1395,16 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
13781395
spin_unlock(&block_group->lock);
13791396
spin_unlock(&space_info->lock);
13801397

1398+
/*
1399+
* The normal path here is an unused block group is passed here,
1400+
* then trimming is handled in the transaction commit path.
1401+
* Async discard interposes before this to do the trimming
1402+
* before coming down the unused block group path as trimming
1403+
* will no longer be done later in the transaction commit path.
1404+
*/
1405+
if (!async_trim_enabled && btrfs_test_opt(fs_info, DISCARD_ASYNC))
1406+
goto flip_async;
1407+
13811408
/* DISCARD can flip during remount */
13821409
trimming = btrfs_test_opt(fs_info, DISCARD_SYNC);
13831410

@@ -1422,6 +1449,13 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
14221449
spin_lock(&fs_info->unused_bgs_lock);
14231450
}
14241451
spin_unlock(&fs_info->unused_bgs_lock);
1452+
return;
1453+
1454+
flip_async:
1455+
btrfs_end_transaction(trans);
1456+
mutex_unlock(&fs_info->delete_unused_bgs_mutex);
1457+
btrfs_put_block_group(block_group);
1458+
btrfs_discard_punt_unused_bgs_list(fs_info);
14251459
}
14261460

14271461
void btrfs_mark_bg_unused(struct btrfs_block_group *bg)
@@ -1626,6 +1660,8 @@ static struct btrfs_block_group *btrfs_create_block_group_cache(
16261660
cache->full_stripe_len = btrfs_full_stripe_len(fs_info, start);
16271661
set_free_space_tree_thresholds(cache);
16281662

1663+
cache->discard_index = BTRFS_DISCARD_INDEX_UNUSED;
1664+
16291665
atomic_set(&cache->count, 1);
16301666
spin_lock_init(&cache->lock);
16311667
init_rwsem(&cache->data_rwsem);
@@ -1792,7 +1828,10 @@ static int read_one_block_group(struct btrfs_fs_info *info,
17921828
inc_block_group_ro(cache, 1);
17931829
} else if (cache->used == 0) {
17941830
ASSERT(list_empty(&cache->bg_list));
1795-
btrfs_mark_bg_unused(cache);
1831+
if (btrfs_test_opt(info, DISCARD_ASYNC))
1832+
btrfs_discard_queue_work(&info->discard_ctl, cache);
1833+
else
1834+
btrfs_mark_bg_unused(cache);
17961835
}
17971836
return 0;
17981837
error:
@@ -2755,8 +2794,10 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans,
27552794
* dirty list to avoid races between cleaner kthread and space
27562795
* cache writeout.
27572796
*/
2758-
if (!alloc && old_val == 0)
2759-
btrfs_mark_bg_unused(cache);
2797+
if (!alloc && old_val == 0) {
2798+
if (!btrfs_test_opt(info, DISCARD_ASYNC))
2799+
btrfs_mark_bg_unused(cache);
2800+
}
27602801

27612802
btrfs_put_block_group(cache);
27622803
total -= num_bytes;

fs/btrfs/ctree.h

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -443,9 +443,14 @@ struct btrfs_full_stripe_locks_tree {
443443
/* Discard control. */
444444
/*
445445
* Async discard uses multiple lists to differentiate the discard filter
446-
* parameters.
446+
* parameters. Index 0 is for completely free block groups where we need to
447+
* ensure the entire block group is trimmed without being lossy. Indices
448+
* afterwards represent monotonically decreasing discard filter sizes to
449+
* prioritize what should be discarded next.
447450
*/
448-
#define BTRFS_NR_DISCARD_LISTS 1
451+
#define BTRFS_NR_DISCARD_LISTS 2
452+
#define BTRFS_DISCARD_INDEX_UNUSED 0
453+
#define BTRFS_DISCARD_INDEX_START 1
449454

450455
struct btrfs_discard_ctl {
451456
struct workqueue_struct *discard_workers;

fs/btrfs/discard.c

Lines changed: 112 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
/* This is an initial delay to give some chance for block reuse */
1515
#define BTRFS_DISCARD_DELAY (120ULL * NSEC_PER_SEC)
16+
#define BTRFS_DISCARD_UNUSED_DELAY (10ULL * NSEC_PER_SEC)
1617

1718
static struct list_head *get_discard_list(struct btrfs_discard_ctl *discard_ctl,
1819
struct btrfs_block_group *block_group)
@@ -30,16 +31,41 @@ static void add_to_discard_list(struct btrfs_discard_ctl *discard_ctl,
3031
return;
3132
}
3233

33-
if (list_empty(&block_group->discard_list))
34+
if (list_empty(&block_group->discard_list) ||
35+
block_group->discard_index == BTRFS_DISCARD_INDEX_UNUSED) {
36+
if (block_group->discard_index == BTRFS_DISCARD_INDEX_UNUSED)
37+
block_group->discard_index = BTRFS_DISCARD_INDEX_START;
3438
block_group->discard_eligible_time = (ktime_get_ns() +
3539
BTRFS_DISCARD_DELAY);
40+
}
3641

3742
list_move_tail(&block_group->discard_list,
3843
get_discard_list(discard_ctl, block_group));
3944

4045
spin_unlock(&discard_ctl->lock);
4146
}
4247

48+
static void add_to_discard_unused_list(struct btrfs_discard_ctl *discard_ctl,
49+
struct btrfs_block_group *block_group)
50+
{
51+
spin_lock(&discard_ctl->lock);
52+
53+
if (!btrfs_run_discard_work(discard_ctl)) {
54+
spin_unlock(&discard_ctl->lock);
55+
return;
56+
}
57+
58+
list_del_init(&block_group->discard_list);
59+
60+
block_group->discard_index = BTRFS_DISCARD_INDEX_UNUSED;
61+
block_group->discard_eligible_time = (ktime_get_ns() +
62+
BTRFS_DISCARD_UNUSED_DELAY);
63+
list_add_tail(&block_group->discard_list,
64+
&discard_ctl->discard_list[BTRFS_DISCARD_INDEX_UNUSED]);
65+
66+
spin_unlock(&discard_ctl->lock);
67+
}
68+
4369
static bool remove_from_discard_list(struct btrfs_discard_ctl *discard_ctl,
4470
struct btrfs_block_group *block_group)
4571
{
@@ -154,7 +180,10 @@ void btrfs_discard_queue_work(struct btrfs_discard_ctl *discard_ctl,
154180
if (!block_group || !btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC))
155181
return;
156182

157-
add_to_discard_list(discard_ctl, block_group);
183+
if (block_group->used == 0)
184+
add_to_discard_unused_list(discard_ctl, block_group);
185+
else
186+
add_to_discard_list(discard_ctl, block_group);
158187

159188
if (!delayed_work_pending(&discard_ctl->work))
160189
btrfs_discard_schedule_work(discard_ctl, false);
@@ -198,6 +227,29 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,
198227
spin_unlock(&discard_ctl->lock);
199228
}
200229

230+
/**
231+
* btrfs_finish_discard_pass - determine next step of a block_group
232+
* @discard_ctl: discard control
233+
* @block_group: block_group of interest
234+
*
235+
* This determines the next step for a block group after it's finished going
236+
* through a pass on a discard list. If it is unused and fully trimmed, we can
237+
* mark it unused and send it to the unused_bgs path. Otherwise, pass it onto
238+
* the appropriate filter list or let it fall off.
239+
*/
240+
static void btrfs_finish_discard_pass(struct btrfs_discard_ctl *discard_ctl,
241+
struct btrfs_block_group *block_group)
242+
{
243+
remove_from_discard_list(discard_ctl, block_group);
244+
245+
if (block_group->used == 0) {
246+
if (btrfs_is_free_space_trimmed(block_group))
247+
btrfs_mark_bg_unused(block_group);
248+
else
249+
add_to_discard_unused_list(discard_ctl, block_group);
250+
}
251+
}
252+
201253
/**
202254
* btrfs_discard_workfn - discard work function
203255
* @work: work
@@ -219,7 +271,7 @@ static void btrfs_discard_workfn(struct work_struct *work)
219271
btrfs_trim_block_group(block_group, &trimmed, block_group->start,
220272
btrfs_block_group_end(block_group), 0);
221273

222-
remove_from_discard_list(discard_ctl, block_group);
274+
btrfs_finish_discard_pass(discard_ctl, block_group);
223275
btrfs_discard_schedule_work(discard_ctl, false);
224276
}
225277

@@ -239,13 +291,69 @@ bool btrfs_run_discard_work(struct btrfs_discard_ctl *discard_ctl)
239291
test_bit(BTRFS_FS_DISCARD_RUNNING, &fs_info->flags));
240292
}
241293

294+
/**
295+
* btrfs_discard_punt_unused_bgs_list - punt unused_bgs list to discard lists
296+
* @fs_info: fs_info of interest
297+
*
298+
* The unused_bgs list needs to be punted to the discard lists because the
299+
* order of operations is changed. In the normal sychronous discard path, the
300+
* block groups are trimmed via a single large trim in transaction commit. This
301+
* is ultimately what we are trying to avoid with asynchronous discard. Thus,
302+
* it must be done before going down the unused_bgs path.
303+
*/
304+
void btrfs_discard_punt_unused_bgs_list(struct btrfs_fs_info *fs_info)
305+
{
306+
struct btrfs_block_group *block_group, *next;
307+
308+
spin_lock(&fs_info->unused_bgs_lock);
309+
/* We enabled async discard, so punt all to the queue */
310+
list_for_each_entry_safe(block_group, next, &fs_info->unused_bgs,
311+
bg_list) {
312+
list_del_init(&block_group->bg_list);
313+
btrfs_discard_queue_work(&fs_info->discard_ctl, block_group);
314+
}
315+
spin_unlock(&fs_info->unused_bgs_lock);
316+
}
317+
318+
/**
319+
* btrfs_discard_purge_list - purge discard lists
320+
* @discard_ctl: discard control
321+
*
322+
* If we are disabling async discard, we may have intercepted block groups that
323+
* are completely free and ready for the unused_bgs path. As discarding will
324+
* now happen in transaction commit or not at all, we can safely mark the
325+
* corresponding block groups as unused and they will be sent on their merry
326+
* way to the unused_bgs list.
327+
*/
328+
static void btrfs_discard_purge_list(struct btrfs_discard_ctl *discard_ctl)
329+
{
330+
struct btrfs_block_group *block_group, *next;
331+
int i;
332+
333+
spin_lock(&discard_ctl->lock);
334+
for (i = 0; i < BTRFS_NR_DISCARD_LISTS; i++) {
335+
list_for_each_entry_safe(block_group, next,
336+
&discard_ctl->discard_list[i],
337+
discard_list) {
338+
list_del_init(&block_group->discard_list);
339+
spin_unlock(&discard_ctl->lock);
340+
if (block_group->used == 0)
341+
btrfs_mark_bg_unused(block_group);
342+
spin_lock(&discard_ctl->lock);
343+
}
344+
}
345+
spin_unlock(&discard_ctl->lock);
346+
}
347+
242348
void btrfs_discard_resume(struct btrfs_fs_info *fs_info)
243349
{
244350
if (!btrfs_test_opt(fs_info, DISCARD_ASYNC)) {
245351
btrfs_discard_cleanup(fs_info);
246352
return;
247353
}
248354

355+
btrfs_discard_punt_unused_bgs_list(fs_info);
356+
249357
set_bit(BTRFS_FS_DISCARD_RUNNING, &fs_info->flags);
250358
}
251359

@@ -270,4 +378,5 @@ void btrfs_discard_cleanup(struct btrfs_fs_info *fs_info)
270378
{
271379
btrfs_discard_stop(fs_info);
272380
cancel_delayed_work_sync(&fs_info->discard_ctl.work);
381+
btrfs_discard_purge_list(&fs_info->discard_ctl);
273382
}

fs/btrfs/discard.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ struct btrfs_fs_info;
77
struct btrfs_discard_ctl;
88
struct btrfs_block_group;
99

10+
/* Work operations */
1011
void btrfs_discard_cancel_work(struct btrfs_discard_ctl *discard_ctl,
1112
struct btrfs_block_group *block_group);
1213
void btrfs_discard_queue_work(struct btrfs_discard_ctl *discard_ctl,
@@ -15,6 +16,8 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,
1516
bool override);
1617
bool btrfs_run_discard_work(struct btrfs_discard_ctl *discard_ctl);
1718

19+
/* Setup/cleanup operations */
20+
void btrfs_discard_punt_unused_bgs_list(struct btrfs_fs_info *fs_info);
1821
void btrfs_discard_resume(struct btrfs_fs_info *fs_info);
1922
void btrfs_discard_stop(struct btrfs_fs_info *fs_info);
2023
void btrfs_discard_init(struct btrfs_fs_info *fs_info);

fs/btrfs/free-space-cache.c

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2707,6 +2707,37 @@ void btrfs_remove_free_space_cache(struct btrfs_block_group *block_group)
27072707

27082708
}
27092709

2710+
/**
2711+
* btrfs_is_free_space_trimmed - see if everything is trimmed
2712+
* @block_group: block_group of interest
2713+
*
2714+
* Walk @block_group's free space rb_tree to determine if everything is trimmed.
2715+
*/
2716+
bool btrfs_is_free_space_trimmed(struct btrfs_block_group *block_group)
2717+
{
2718+
struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
2719+
struct btrfs_free_space *info;
2720+
struct rb_node *node;
2721+
bool ret = true;
2722+
2723+
spin_lock(&ctl->tree_lock);
2724+
node = rb_first(&ctl->free_space_offset);
2725+
2726+
while (node) {
2727+
info = rb_entry(node, struct btrfs_free_space, offset_index);
2728+
2729+
if (!btrfs_free_space_trimmed(info)) {
2730+
ret = false;
2731+
break;
2732+
}
2733+
2734+
node = rb_next(node);
2735+
}
2736+
2737+
spin_unlock(&ctl->tree_lock);
2738+
return ret;
2739+
}
2740+
27102741
u64 btrfs_find_space_for_alloc(struct btrfs_block_group *block_group,
27112742
u64 offset, u64 bytes, u64 empty_size,
27122743
u64 *max_extent_size)
@@ -2793,6 +2824,8 @@ int btrfs_return_cluster_to_free_space(
27932824
ret = __btrfs_return_cluster_to_free_space(block_group, cluster);
27942825
spin_unlock(&ctl->tree_lock);
27952826

2827+
btrfs_discard_queue_work(&block_group->fs_info->discard_ctl, block_group);
2828+
27962829
/* finally drop our ref */
27972830
btrfs_put_block_group(block_group);
27982831
return ret;

fs/btrfs/free-space-cache.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ int btrfs_remove_free_space(struct btrfs_block_group *block_group,
119119
u64 bytenr, u64 size);
120120
void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl);
121121
void btrfs_remove_free_space_cache(struct btrfs_block_group *block_group);
122+
bool btrfs_is_free_space_trimmed(struct btrfs_block_group *block_group);
122123
u64 btrfs_find_space_for_alloc(struct btrfs_block_group *block_group,
123124
u64 offset, u64 bytes, u64 empty_size,
124125
u64 *max_extent_size);

fs/btrfs/scrub.c

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include <linux/sched/mm.h>
99
#include <crypto/hash.h>
1010
#include "ctree.h"
11+
#include "discard.h"
1112
#include "volumes.h"
1213
#include "disk-io.h"
1314
#include "ordered-data.h"
@@ -3659,7 +3660,11 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx,
36593660
if (!cache->removed && !cache->ro && cache->reserved == 0 &&
36603661
cache->used == 0) {
36613662
spin_unlock(&cache->lock);
3662-
btrfs_mark_bg_unused(cache);
3663+
if (btrfs_test_opt(fs_info, DISCARD_ASYNC))
3664+
btrfs_discard_queue_work(&fs_info->discard_ctl,
3665+
cache);
3666+
else
3667+
btrfs_mark_bg_unused(cache);
36633668
} else {
36643669
spin_unlock(&cache->lock);
36653670
}

0 commit comments

Comments
 (0)