2017-12-17 22:00:59 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2008-04-29 18:13:32 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-04-29 18:13:32 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2021-07-19 13:59:14 +08:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:21:05 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2009-01-05 21:49:55 -05:00
|
|
|
|
2009-01-22 18:13:05 +01:00
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
2017-02-02 19:15:33 +01:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
2013-10-17 21:11:01 -04:00
|
|
|
|
2024-12-01 17:08:41 -08:00
|
|
|
|
2014-02-22 06:18:17 -05:00
|
|
|
|
2016-04-25 23:22:35 -04:00
|
|
|
|
2020-05-23 09:30:11 +02:00
|
|
|
|
2010-05-17 06:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-04-02 11:37:42 +09:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2017-10-09 12:15:34 -07:00
|
|
|
|
2019-07-22 09:26:24 -07:00
|
|
|
|
2017-10-09 12:15:34 -07:00
|
|
|
|
2018-08-27 01:15:11 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2008-07-11 19:27:31 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2016-03-13 17:18:12 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2008-09-08 22:25:24 -04:00
|
|
|
|
2008-07-13 21:03:29 -04:00
|
|
|
|
2008-09-08 22:25:24 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2012-03-19 23:11:43 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2020-05-10 11:54:55 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-11-28 13:03:30 -05:00
|
|
|
|
2020-05-10 11:54:55 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-11-28 13:03:30 -05:00
|
|
|
|
2020-05-10 11:54:55 +05:30
|
|
|
|
2012-11-28 13:03:30 -05:00
|
|
|
|
|
|
|
|
|
2020-11-07 23:58:11 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-06-09 01:55:03 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:42 +05:30
|
|
|
|
2023-06-08 10:39:35 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:42 +05:30
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:50 +05:30
|
|
|
|
2023-06-08 10:39:35 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:50 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-06-08 10:39:35 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:50 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-06-08 10:39:35 -04:00
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:50 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-06-08 10:39:35 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:50 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-06-08 10:39:35 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:50 +05:30
|
|
|
|
|
|
|
|
|
2023-06-08 10:39:35 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:42 +05:30
|
|
|
|
|
|
|
|
|
2009-09-30 00:32:42 -04:00
|
|
|
|
2010-05-17 07:00:00 -04:00
|
|
|
|
2009-09-30 00:32:42 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2009-08-09 16:46:13 -04:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2009-08-09 16:46:13 -04:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2009-08-09 16:46:13 -04:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2009-08-09 16:46:13 -04:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2009-08-09 16:46:13 -04:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2009-08-09 16:46:13 -04:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2009-08-09 16:46:13 -04:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2009-08-09 16:46:13 -04:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2009-08-09 16:46:13 -04:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2009-08-09 16:46:13 -04:00
|
|
|
|
2008-07-14 17:52:37 -04:00
|
|
|
|
2009-08-09 16:46:13 -04:00
|
|
|
|
2009-08-09 22:01:13 -04:00
|
|
|
|
|
|
|
|
|
2011-05-25 07:41:26 -04:00
|
|
|
|
|
|
|
|
|
2013-04-09 22:11:22 -04:00
|
|
|
|
|
|
|
|
|
2020-05-20 12:10:36 +05:30
|
|
|
|
|
|
|
|
|
2024-04-24 14:19:03 +08:00
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:50 +05:30
|
|
|
|
2024-04-24 14:19:03 +08:00
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:50 +05:30
|
|
|
|
2024-04-24 14:19:03 +08:00
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:50 +05:30
|
|
|
|
2023-05-30 18:03:49 +05:30
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-11-05 00:14:04 -05:00
|
|
|
|
2009-07-13 10:24:17 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2008-11-05 00:14:04 -05:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
|
|
|
|
|
2010-05-16 19:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-05-10 11:54:51 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-01-27 09:58:04 +08:00
|
|
|
|
2010-05-16 19:00:00 -04:00
|
|
|
|
2024-01-27 09:58:04 +08:00
|
|
|
|
|
|
|
|
|
2010-05-16 19:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-08-28 11:13:24 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-27 21:30:10 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-10-16 13:07:10 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-03-05 10:40:22 -05:00
|
|
|
|
2014-04-20 23:45:47 -04:00
|
|
|
|
2013-06-04 13:21:11 -04:00
|
|
|
|
2012-03-05 10:40:22 -05:00
|
|
|
|
2009-09-28 15:49:08 -04:00
|
|
|
|
2010-07-27 11:56:06 -04:00
|
|
|
|
2013-06-04 13:21:11 -04:00
|
|
|
|
|
|
|
|
|
2009-09-28 15:49:08 -04:00
|
|
|
|
2013-06-04 14:23:41 -04:00
|
|
|
|
|
|
|
|
|
2009-09-28 15:48:29 -04:00
|
|
|
|
2021-07-19 13:59:14 +08:00
|
|
|
|
2019-10-16 13:07:10 +05:30
|
|
|
|
2009-09-28 15:49:08 -04:00
|
|
|
|
|
|
|
|
|
2010-10-27 21:30:10 -04:00
|
|
|
|
2015-07-21 23:50:24 -04:00
|
|
|
|
2010-10-27 21:30:10 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2011-05-24 19:00:39 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-07-18 08:38:01 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
|
|
|
|
|
2010-10-27 21:29:12 -04:00
|
|
|
|
2016-11-18 13:00:24 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2008-09-08 22:25:24 -04:00
|
|
|
|
2011-09-09 18:34:51 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2011-09-09 18:34:51 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2007-07-17 21:42:41 -04:00
|
|
|
|
2016-09-15 11:55:01 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2011-09-09 18:44:51 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-12-20 09:29:35 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-08-22 23:22:14 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-12-20 09:29:35 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-09-09 18:44:51 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
|
|
|
|
|
2009-01-05 22:20:24 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Ext4: Uninitialized Block Groups
In pass1 of e2fsck, every inode table in the fileystem is scanned and checked,
regardless of whether it is in use. This is this the most time consuming part
of the filesystem check. The unintialized block group feature can greatly
reduce e2fsck time by eliminating checking of uninitialized inodes.
With this feature, there is a a high water mark of used inodes for each block
group. Block and inode bitmaps can be uninitialized on disk via a flag in the
group descriptor to avoid reading or scanning them at e2fsck time. A checksum
of each group descriptor is used to ensure that corruption in the group
descriptor's bit flags does not cause incorrect operation.
The feature is enabled through a mkfs option
mke2fs /dev/ -O uninit_groups
A patch adding support for uninitialized block groups to e2fsprogs tools has
been posted to the linux-ext4 mailing list.
The patches have been stress tested with fsstress and fsx. In performance
tests testing e2fsck time, we have seen that e2fsck time on ext3 grows
linearly with the total number of inodes in the filesytem. In ext4 with the
uninitialized block groups feature, the e2fsck time is constant, based
solely on the number of used inodes rather than the total inode count.
Since typical ext4 filesystems only use 1-10% of their inodes, this feature can
greatly reduce e2fsck time for users. With performance improvement of 2-20
times, depending on how full the filesystem is.
The attached graph shows the major improvements in e2fsck times in filesystems
with a large total inode count, but few inodes in use.
In each group descriptor if we have
EXT4_BG_INODE_UNINIT set in bg_flags:
Inode table is not initialized/used in this group. So we can skip
the consistency check during fsck.
EXT4_BG_BLOCK_UNINIT set in bg_flags:
No block in the group is used. So we can skip the block bitmap
verification for this group.
We also add two new fields to group descriptor as a part of
uninitialized group patch.
__le16 bg_itable_unused; /* Unused inodes count */
__le16 bg_checksum; /* crc16(sb_uuid+group+desc) */
bg_itable_unused:
If we have EXT4_BG_INODE_UNINIT not set in bg_flags
then bg_itable_unused will give the offset within
the inode table till the inodes are used. This can be
used by fsck to skip list of inodes that are marked unused.
bg_checksum:
Now that we depend on bg_flags and bg_itable_unused to determine
the block and inode usage, we need to make sure group descriptor
is not corrupt. We add checksum to group descriptor to
detect corruption. If the descriptor is found to be corrupt, we
mark all the blocks and inodes in the group used.
Signed-off-by: Avantika Mathur <mathur@us.ibm.com>
Signed-off-by: Andreas Dilger <adilger@clusterfs.com>
Signed-off-by: Mingming Cao <cmm@us.ibm.com>
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
2007-10-16 18:38:25 -04:00
|
|
|
|
2012-04-29 18:23:10 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-01-05 22:20:24 -05:00
|
|
|
|
Ext4: Uninitialized Block Groups
In pass1 of e2fsck, every inode table in the fileystem is scanned and checked,
regardless of whether it is in use. This is this the most time consuming part
of the filesystem check. The unintialized block group feature can greatly
reduce e2fsck time by eliminating checking of uninitialized inodes.
With this feature, there is a a high water mark of used inodes for each block
group. Block and inode bitmaps can be uninitialized on disk via a flag in the
group descriptor to avoid reading or scanning them at e2fsck time. A checksum
of each group descriptor is used to ensure that corruption in the group
descriptor's bit flags does not cause incorrect operation.
The feature is enabled through a mkfs option
mke2fs /dev/ -O uninit_groups
A patch adding support for uninitialized block groups to e2fsprogs tools has
been posted to the linux-ext4 mailing list.
The patches have been stress tested with fsstress and fsx. In performance
tests testing e2fsck time, we have seen that e2fsck time on ext3 grows
linearly with the total number of inodes in the filesytem. In ext4 with the
uninitialized block groups feature, the e2fsck time is constant, based
solely on the number of used inodes rather than the total inode count.
Since typical ext4 filesystems only use 1-10% of their inodes, this feature can
greatly reduce e2fsck time for users. With performance improvement of 2-20
times, depending on how full the filesystem is.
The attached graph shows the major improvements in e2fsck times in filesystems
with a large total inode count, but few inodes in use.
In each group descriptor if we have
EXT4_BG_INODE_UNINIT set in bg_flags:
Inode table is not initialized/used in this group. So we can skip
the consistency check during fsck.
EXT4_BG_BLOCK_UNINIT set in bg_flags:
No block in the group is used. So we can skip the block bitmap
verification for this group.
We also add two new fields to group descriptor as a part of
uninitialized group patch.
__le16 bg_itable_unused; /* Unused inodes count */
__le16 bg_checksum; /* crc16(sb_uuid+group+desc) */
bg_itable_unused:
If we have EXT4_BG_INODE_UNINIT not set in bg_flags
then bg_itable_unused will give the offset within
the inode table till the inodes are used. This can be
used by fsck to skip list of inodes that are marked unused.
bg_checksum:
Now that we depend on bg_flags and bg_itable_unused to determine
the block and inode usage, we need to make sure group descriptor
is not corrupt. We add checksum to group descriptor to
detect corruption. If the descriptor is found to be corrupt, we
mark all the blocks and inodes in the group used.
Signed-off-by: Avantika Mathur <mathur@us.ibm.com>
Signed-off-by: Andreas Dilger <adilger@clusterfs.com>
Signed-off-by: Mingming Cao <cmm@us.ibm.com>
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
2007-10-16 18:38:25 -04:00
|
|
|
|
2006-10-11 01:21:15 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-01-05 22:20:24 -05:00
|
|
|
|
2012-04-29 18:23:10 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2012-04-29 18:33:10 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-07-11 19:27:31 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-03-11 23:39:59 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-07-11 19:27:31 -04:00
|
|
|
|
|
|
|
|
|
Ext4: Uninitialized Block Groups
In pass1 of e2fsck, every inode table in the fileystem is scanned and checked,
regardless of whether it is in use. This is this the most time consuming part
of the filesystem check. The unintialized block group feature can greatly
reduce e2fsck time by eliminating checking of uninitialized inodes.
With this feature, there is a a high water mark of used inodes for each block
group. Block and inode bitmaps can be uninitialized on disk via a flag in the
group descriptor to avoid reading or scanning them at e2fsck time. A checksum
of each group descriptor is used to ensure that corruption in the group
descriptor's bit flags does not cause incorrect operation.
The feature is enabled through a mkfs option
mke2fs /dev/ -O uninit_groups
A patch adding support for uninitialized block groups to e2fsprogs tools has
been posted to the linux-ext4 mailing list.
The patches have been stress tested with fsstress and fsx. In performance
tests testing e2fsck time, we have seen that e2fsck time on ext3 grows
linearly with the total number of inodes in the filesytem. In ext4 with the
uninitialized block groups feature, the e2fsck time is constant, based
solely on the number of used inodes rather than the total inode count.
Since typical ext4 filesystems only use 1-10% of their inodes, this feature can
greatly reduce e2fsck time for users. With performance improvement of 2-20
times, depending on how full the filesystem is.
The attached graph shows the major improvements in e2fsck times in filesystems
with a large total inode count, but few inodes in use.
In each group descriptor if we have
EXT4_BG_INODE_UNINIT set in bg_flags:
Inode table is not initialized/used in this group. So we can skip
the consistency check during fsck.
EXT4_BG_BLOCK_UNINIT set in bg_flags:
No block in the group is used. So we can skip the block bitmap
verification for this group.
We also add two new fields to group descriptor as a part of
uninitialized group patch.
__le16 bg_itable_unused; /* Unused inodes count */
__le16 bg_checksum; /* crc16(sb_uuid+group+desc) */
bg_itable_unused:
If we have EXT4_BG_INODE_UNINIT not set in bg_flags
then bg_itable_unused will give the offset within
the inode table till the inodes are used. This can be
used by fsck to skip list of inodes that are marked unused.
bg_checksum:
Now that we depend on bg_flags and bg_itable_unused to determine
the block and inode usage, we need to make sure group descriptor
is not corrupt. We add checksum to group descriptor to
detect corruption. If the descriptor is found to be corrupt, we
mark all the blocks and inodes in the group used.
Signed-off-by: Avantika Mathur <mathur@us.ibm.com>
Signed-off-by: Andreas Dilger <adilger@clusterfs.com>
Signed-off-by: Mingming Cao <cmm@us.ibm.com>
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
2007-10-16 18:38:25 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:21:14 -07:00
|
|
|
|
2006-10-11 01:21:15 -07:00
|
|
|
|
2006-10-11 01:21:14 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2011-09-09 18:34:51 -04:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:21:14 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-01-19 16:00:58 -05:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2006-10-11 01:21:03 -07:00
|
|
|
|
2019-07-22 09:26:24 -07:00
|
|
|
|
2010-01-25 03:31:32 -05:00
|
|
|
|
ext4: remove EXT4_EOFBLOCKS_FL and associated code
The EXT4_EOFBLOCKS_FL inode flag is used to indicate whether a file
contains unwritten blocks past i_size. It's set when ext4_fallocate
is called with the KEEP_SIZE flag to extend a file with an unwritten
extent. However, this flag hasn't been useful functionally since
March, 2012, when a decision was made to remove it from ext4.
All traces of EXT4_EOFBLOCKS_FL were removed from e2fsprogs version
1.42.2 by commit 010dc7b90d97 ("e2fsck: remove EXT4_EOFBLOCKS_FL flag
handling") at that time. Now that enough time has passed to make
e2fsprogs versions containing this modification common, this patch now
removes the code associated with EXT4_EOFBLOCKS_FL from the kernel as
well.
This change has two implications. First, because pre-1.42.2 e2fsck
versions only look for a problem if EXT4_EOFBLOCKS_FL is set, and
because that bit will never be set by newer kernels containing this
patch, old versions of e2fsck won't have a compatibility problem with
files created by newer kernels.
Second, newer kernels will not clear EXT4_EOFBLOCKS_FL inode flag bits
belonging to a file written by an older kernel. If set, it will remain
in that state until the file is deleted. Because e2fsck versions since
1.42.2 don't check the flag at all, no adverse effect is expected.
However, pre-1.42.2 e2fsck versions that do check the flag may report
that it is set when it ought not to be after a file has been truncated
or had its unwritten blocks written. In this case, the old version of
e2fsck will offer to clear the flag. No adverse effect would then
occur whether the user chooses to clear the flag or not.
Signed-off-by: Eric Whitney <enwlinux@gmail.com>
Link: https://lore.kernel.org/r/20200211210216.24960-1-enwlinux@gmail.com
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2020-02-11 16:02:16 -05:00
|
|
|
|
2020-05-28 08:00:02 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-12-10 14:04:46 -05:00
|
|
|
|
2015-10-17 16:15:18 -04:00
|
|
|
|
2020-05-10 14:52:52 -07:00
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2020-07-12 20:10:12 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2009-02-15 18:57:26 -05:00
|
|
|
|
|
|
|
|
|
2011-08-31 11:54:51 -04:00
|
|
|
|
2009-02-15 18:57:26 -05:00
|
|
|
|
2016-01-08 16:01:21 -05:00
|
|
|
|
2020-05-28 08:00:02 -07:00
|
|
|
|
|
|
|
|
|
2009-02-15 18:57:26 -05:00
|
|
|
|
2009-02-15 18:09:20 -05:00
|
|
|
|
2019-06-10 00:13:32 -04:00
|
|
|
|
|
|
|
|
|
2009-02-15 18:09:20 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-02-11 00:35:06 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-05-28 08:00:02 -07:00
|
|
|
|
|
|
|
|
|
2020-08-28 16:43:30 +08:00
|
|
|
|
2020-05-28 08:00:02 -07:00
|
|
|
|
2009-02-15 18:09:20 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-04-11 07:44:12 -04:00
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-07-22 09:26:24 -07:00
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
ext4: remove EXT4_EOFBLOCKS_FL and associated code
The EXT4_EOFBLOCKS_FL inode flag is used to indicate whether a file
contains unwritten blocks past i_size. It's set when ext4_fallocate
is called with the KEEP_SIZE flag to extend a file with an unwritten
extent. However, this flag hasn't been useful functionally since
March, 2012, when a decision was made to remove it from ext4.
All traces of EXT4_EOFBLOCKS_FL were removed from e2fsprogs version
1.42.2 by commit 010dc7b90d97 ("e2fsck: remove EXT4_EOFBLOCKS_FL flag
handling") at that time. Now that enough time has passed to make
e2fsprogs versions containing this modification common, this patch now
removes the code associated with EXT4_EOFBLOCKS_FL from the kernel as
well.
This change has two implications. First, because pre-1.42.2 e2fsck
versions only look for a problem if EXT4_EOFBLOCKS_FL is set, and
because that bit will never be set by newer kernels containing this
patch, old versions of e2fsck won't have a compatibility problem with
files created by newer kernels.
Second, newer kernels will not clear EXT4_EOFBLOCKS_FL inode flag bits
belonging to a file written by an older kernel. If set, it will remain
in that state until the file is deleted. Because e2fsck versions since
1.42.2 don't check the flag at all, no adverse effect is expected.
However, pre-1.42.2 e2fsck versions that do check the flag may report
that it is set when it ought not to be after a file has been truncated
or had its unwritten blocks written. In this case, the old version of
e2fsck will offer to clear the flag. No adverse effect would then
occur whether the user chooses to clear the flag or not.
Signed-off-by: Eric Whitney <enwlinux@gmail.com>
Link: https://lore.kernel.org/r/20200211210216.24960-1-enwlinux@gmail.com
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2020-02-11 16:02:16 -05:00
|
|
|
|
2020-05-28 08:00:02 -07:00
|
|
|
|
2012-12-10 14:04:46 -05:00
|
|
|
|
2015-10-17 16:15:18 -04:00
|
|
|
|
2020-05-10 14:52:52 -07:00
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: ensure Inode flags consistency are checked at build time
Flags being used by atomic operations in inode flags (e.g.
ext4_test_inode_flag(), should be consistent with that actually stored
in inodes, i.e.: EXT4_XXX_FL.
It ensures that this consistency is checked at build-time, not at
run-time.
Currently, the flags consistency are being checked at run-time, but,
there is no real reason to not do a build-time check instead of a
run-time check. The code is comparing macro defined values with enum
type variables, where both are constants, so, there is no problem in
comparing constants at build-time.
enum variables are treated as constants by the C compiler, according
to the C99 specs (see www.open-std.org/jtc1/sc22/wg14/www/docs/n1124.pdf
sec. 6.2.5, item 16), so, there is no real problem in comparing an
enumeration type at build time
Signed-off-by: Carlos Maiolino <cmaiolino@redhat.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2012-12-10 16:30:45 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2022-10-31 13:58:33 +08:00
|
|
|
|
ext4: ensure Inode flags consistency are checked at build time
Flags being used by atomic operations in inode flags (e.g.
ext4_test_inode_flag(), should be consistent with that actually stored
in inodes, i.e.: EXT4_XXX_FL.
It ensures that this consistency is checked at build-time, not at
run-time.
Currently, the flags consistency are being checked at run-time, but,
there is no real reason to not do a build-time check instead of a
run-time check. The code is comparing macro defined values with enum
type variables, where both are constants, so, there is no problem in
comparing constants at build-time.
enum variables are treated as constants by the C compiler, according
to the C99 specs (see www.open-std.org/jtc1/sc22/wg14/www/docs/n1124.pdf
sec. 6.2.5, item 16), so, there is no real problem in comparing an
enumeration type at build time
Signed-off-by: Carlos Maiolino <cmaiolino@redhat.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2012-12-10 16:30:45 -05:00
|
|
|
|
|
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-01-19 16:00:58 -05:00
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-07-22 09:26:24 -07:00
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
2012-12-10 14:04:46 -05:00
|
|
|
|
2015-10-17 16:15:18 -04:00
|
|
|
|
2020-05-10 14:52:52 -07:00
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-05-17 06:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2017-10-29 09:38:46 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2012-01-03 23:32:52 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-07-17 21:42:41 -04:00
|
|
|
|
2010-07-27 11:56:07 -04:00
|
|
|
|
2007-07-17 21:42:41 -04:00
|
|
|
|
2014-04-20 23:45:47 -04:00
|
|
|
|
2009-05-14 00:58:52 -04:00
|
|
|
|
2009-05-14 13:57:08 -04:00
|
|
|
|
2014-04-20 23:45:47 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-14 00:58:52 -04:00
|
|
|
|
2013-04-09 22:11:22 -04:00
|
|
|
|
|
|
|
|
|
2009-06-09 00:17:05 -04:00
|
|
|
|
2009-09-28 15:49:08 -04:00
|
|
|
|
2014-04-20 23:45:47 -04:00
|
|
|
|
2009-09-28 15:49:08 -04:00
|
|
|
|
2010-03-02 13:28:44 -05:00
|
|
|
|
2010-01-15 01:27:59 -05:00
|
|
|
|
2010-03-02 13:28:44 -05:00
|
|
|
|
2014-04-20 23:45:47 -04:00
|
|
|
|
2010-03-02 13:28:44 -05:00
|
|
|
|
|
|
|
|
|
2014-04-20 23:45:47 -04:00
|
|
|
|
2013-04-09 22:11:22 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-05-25 07:41:54 -04:00
|
|
|
|
|
|
|
|
|
2014-03-18 18:05:35 -04:00
|
|
|
|
2015-12-07 15:04:57 -05:00
|
|
|
|
2015-12-07 15:10:26 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-04-24 00:56:08 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2022-04-24 22:09:35 +08:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2013-08-16 21:23:41 -04:00
|
|
|
|
2013-08-16 22:05:14 -04:00
|
|
|
|
2014-09-01 14:43:09 -04:00
|
|
|
|
2013-08-16 21:23:41 -04:00
|
|
|
|
2013-08-16 22:05:14 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-08-16 21:23:41 -04:00
|
|
|
|
2014-09-04 18:09:29 -04:00
|
|
|
|
|
|
|
|
|
2020-05-07 10:50:28 -07:00
|
|
|
|
2013-08-16 21:23:41 -04:00
|
|
|
|
2009-11-23 07:17:05 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2018-01-07 23:36:19 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-09-09 18:50:51 -04:00
|
|
|
|
|
|
|
|
|
2018-10-01 14:25:08 -04:00
|
|
|
|
2009-11-23 07:17:05 -05:00
|
|
|
|
2010-05-17 05:00:00 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-05-17 06:00:00 -04:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2010-05-17 05:00:00 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
tree-wide: fix comment/printk typos
"gadget", "through", "command", "maintain", "maintain", "controller", "address",
"between", "initiali[zs]e", "instead", "function", "select", "already",
"equal", "access", "management", "hierarchy", "registration", "interest",
"relative", "memory", "offset", "already",
Signed-off-by: Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
Signed-off-by: Jiri Kosina <jkosina@suse.cz>
2010-11-01 15:38:34 -04:00
|
|
|
|
ext4: limit block allocations for indirect-block files to < 2^32
Today, the ext4 allocator will happily allocate blocks past
2^32 for indirect-block files, which results in the block
numbers getting truncated, and corruption ensues.
This patch limits such allocations to < 2^32, and adds
BUG_ONs if we do get blocks larger than that.
This should address RH Bug 519471, ext4 bitmap allocator
must limit blocks to < 2^32
* ext4_find_goal() is modified to choose a goal < UINT_MAX,
so that our starting point is in an acceptable range.
* ext4_xattr_block_set() is modified such that the goal block
is < UINT_MAX, as above.
* ext4_mb_regular_allocator() is modified so that the group
search does not continue into groups which are too high
* ext4_mb_use_preallocated() has a check that we don't use
preallocated space which is too far out
* ext4_alloc_blocks() and ext4_xattr_block_set() add some BUG_ONs
No attempt has been made to limit inode locations to < 2^32,
so we may wind up with blocks far from their inodes. Doing
this much already will lead to some odd ENOSPC issues when the
"lower 32" gets full, and further restricting inodes could
make that even weirder.
For high inodes, choosing a goal of the original, % UINT_MAX,
may be a bit odd, but then we're in an odd situation anyway,
and I don't know of a better heuristic.
Signed-off-by: Eric Sandeen <sandeen@redhat.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2009-09-16 14:45:10 -04:00
|
|
|
|
|
|
|
|
|
2018-09-01 12:45:04 -04:00
|
|
|
|
2020-05-05 17:43:14 +02:00
|
|
|
|
2018-09-01 12:45:04 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:26 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:26 -05:00
|
|
|
|
2006-10-11 01:21:09 -07:00
|
|
|
|
2007-05-24 13:04:54 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2012-04-29 18:23:10 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
2006-10-11 01:21:09 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-04-29 18:23:10 -04:00
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2015-10-17 16:15:18 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: Fix handling of extended tv_sec
In ext4, the bottom two bits of {a,c,m}time_extra are used to extend
the {a,c,m}time fields, deferring the year 2038 problem to the year
2446.
When decoding these extended fields, for times whose bottom 32 bits
would represent a negative number, sign extension causes the 64-bit
extended timestamp to be negative as well, which is not what's
intended. This patch corrects that issue, so that the only negative
{a,c,m}times are those between 1901 and 1970 (as per 32-bit signed
timestamps).
Some older kernels might have written pre-1970 dates with 1,1 in the
extra bits. This patch treats those incorrectly-encoded dates as
pre-1970, instead of post-2311, until kernel 4.20 is released.
Hopefully by then e2fsck will have fixed up the bad data.
Also add a comment explaining the encoding of ext4's extra {a,c,m}time
bits.
Signed-off-by: David Turner <novalis@novalis.org>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
Reported-by: Mark Harris <mh8928@yahoo.com>
Bugzilla: https://bugzilla.kernel.org/show_bug.cgi?id=23732
Cc: stable@vger.kernel.org
2015-11-24 14:34:37 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-07-05 15:01:07 -04:00
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
2023-07-05 15:01:07 -04:00
|
|
|
|
|
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
|
|
|
|
|
2023-07-05 15:01:07 -04:00
|
|
|
|
|
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
2023-07-05 15:01:07 -04:00
|
|
|
|
|
|
|
|
|
2019-08-12 13:44:49 -04:00
|
|
|
|
2023-07-05 15:01:07 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
|
|
|
|
|
2023-07-05 15:01:07 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-10-04 14:52:20 -04:00
|
|
|
|
|
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
2023-10-04 14:52:20 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-07-05 15:01:07 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
2023-10-04 14:52:20 -04:00
|
|
|
|
vfs: change inode times to use struct timespec64
struct timespec is not y2038 safe. Transition vfs to use
y2038 safe struct timespec64 instead.
The change was made with the help of the following cocinelle
script. This catches about 80% of the changes.
All the header file and logic changes are included in the
first 5 rules. The rest are trivial substitutions.
I avoid changing any of the function signatures or any other
filesystem specific data structures to keep the patch simple
for review.
The script can be a little shorter by combining different cases.
But, this version was sufficient for my usecase.
virtual patch
@ depends on patch @
identifier now;
@@
- struct timespec
+ struct timespec64
current_time ( ... )
{
- struct timespec now = current_kernel_time();
+ struct timespec64 now = current_kernel_time64();
...
- return timespec_trunc(
+ return timespec64_trunc(
... );
}
@ depends on patch @
identifier xtime;
@@
struct \( iattr \| inode \| kstat \) {
...
- struct timespec xtime;
+ struct timespec64 xtime;
...
}
@ depends on patch @
identifier t;
@@
struct inode_operations {
...
int (*update_time) (...,
- struct timespec t,
+ struct timespec64 t,
...);
...
}
@ depends on patch @
identifier t;
identifier fn_update_time =~ "update_time$";
@@
fn_update_time (...,
- struct timespec *t,
+ struct timespec64 *t,
...) { ... }
@ depends on patch @
identifier t;
@@
lease_get_mtime( ... ,
- struct timespec *t
+ struct timespec64 *t
) { ... }
@te depends on patch forall@
identifier ts;
local idexpression struct inode *inode_node;
identifier i_xtime =~ "^i_[acm]time$";
identifier ia_xtime =~ "^ia_[acm]time$";
identifier fn_update_time =~ "update_time$";
identifier fn;
expression e, E3;
local idexpression struct inode *node1;
local idexpression struct inode *node2;
local idexpression struct iattr *attr1;
local idexpression struct iattr *attr2;
local idexpression struct iattr attr;
identifier i_xtime1 =~ "^i_[acm]time$";
identifier i_xtime2 =~ "^i_[acm]time$";
identifier ia_xtime1 =~ "^ia_[acm]time$";
identifier ia_xtime2 =~ "^ia_[acm]time$";
@@
(
(
- struct timespec ts;
+ struct timespec64 ts;
|
- struct timespec ts = current_time(inode_node);
+ struct timespec64 ts = current_time(inode_node);
)
<+... when != ts
(
- timespec_equal(&inode_node->i_xtime, &ts)
+ timespec64_equal(&inode_node->i_xtime, &ts)
|
- timespec_equal(&ts, &inode_node->i_xtime)
+ timespec64_equal(&ts, &inode_node->i_xtime)
|
- timespec_compare(&inode_node->i_xtime, &ts)
+ timespec64_compare(&inode_node->i_xtime, &ts)
|
- timespec_compare(&ts, &inode_node->i_xtime)
+ timespec64_compare(&ts, &inode_node->i_xtime)
|
ts = current_time(e)
|
fn_update_time(..., &ts,...)
|
inode_node->i_xtime = ts
|
node1->i_xtime = ts
|
ts = inode_node->i_xtime
|
<+... attr1->ia_xtime ...+> = ts
|
ts = attr1->ia_xtime
|
ts.tv_sec
|
ts.tv_nsec
|
btrfs_set_stack_timespec_sec(..., ts.tv_sec)
|
btrfs_set_stack_timespec_nsec(..., ts.tv_nsec)
|
- ts = timespec64_to_timespec(
+ ts =
...
-)
|
- ts = ktime_to_timespec(
+ ts = ktime_to_timespec64(
...)
|
- ts = E3
+ ts = timespec_to_timespec64(E3)
|
- ktime_get_real_ts(&ts)
+ ktime_get_real_ts64(&ts)
|
fn(...,
- ts
+ timespec64_to_timespec(ts)
,...)
)
...+>
(
<... when != ts
- return ts;
+ return timespec64_to_timespec(ts);
...>
)
|
- timespec_equal(&node1->i_xtime1, &node2->i_xtime2)
+ timespec64_equal(&node1->i_xtime2, &node2->i_xtime2)
|
- timespec_equal(&node1->i_xtime1, &attr2->ia_xtime2)
+ timespec64_equal(&node1->i_xtime2, &attr2->ia_xtime2)
|
- timespec_compare(&node1->i_xtime1, &node2->i_xtime2)
+ timespec64_compare(&node1->i_xtime1, &node2->i_xtime2)
|
node1->i_xtime1 =
- timespec_trunc(attr1->ia_xtime1,
+ timespec64_trunc(attr1->ia_xtime1,
...)
|
- attr1->ia_xtime1 = timespec_trunc(attr2->ia_xtime2,
+ attr1->ia_xtime1 = timespec64_trunc(attr2->ia_xtime2,
...)
|
- ktime_get_real_ts(&attr1->ia_xtime1)
+ ktime_get_real_ts64(&attr1->ia_xtime1)
|
- ktime_get_real_ts(&attr.ia_xtime1)
+ ktime_get_real_ts64(&attr.ia_xtime1)
)
@ depends on patch @
struct inode *node;
struct iattr *attr;
identifier fn;
identifier i_xtime =~ "^i_[acm]time$";
identifier ia_xtime =~ "^ia_[acm]time$";
expression e;
@@
(
- fn(node->i_xtime);
+ fn(timespec64_to_timespec(node->i_xtime));
|
fn(...,
- node->i_xtime);
+ timespec64_to_timespec(node->i_xtime));
|
- e = fn(attr->ia_xtime);
+ e = fn(timespec64_to_timespec(attr->ia_xtime));
)
@ depends on patch forall @
struct inode *node;
struct iattr *attr;
identifier i_xtime =~ "^i_[acm]time$";
identifier ia_xtime =~ "^ia_[acm]time$";
identifier fn;
@@
{
+ struct timespec ts;
<+...
(
+ ts = timespec64_to_timespec(node->i_xtime);
fn (...,
- &node->i_xtime,
+ &ts,
...);
|
+ ts = timespec64_to_timespec(attr->ia_xtime);
fn (...,
- &attr->ia_xtime,
+ &ts,
...);
)
...+>
}
@ depends on patch forall @
struct inode *node;
struct iattr *attr;
struct kstat *stat;
identifier ia_xtime =~ "^ia_[acm]time$";
identifier i_xtime =~ "^i_[acm]time$";
identifier xtime =~ "^[acm]time$";
identifier fn, ret;
@@
{
+ struct timespec ts;
<+...
(
+ ts = timespec64_to_timespec(node->i_xtime);
ret = fn (...,
- &node->i_xtime,
+ &ts,
...);
|
+ ts = timespec64_to_timespec(node->i_xtime);
ret = fn (...,
- &node->i_xtime);
+ &ts);
|
+ ts = timespec64_to_timespec(attr->ia_xtime);
ret = fn (...,
- &attr->ia_xtime,
+ &ts,
...);
|
+ ts = timespec64_to_timespec(attr->ia_xtime);
ret = fn (...,
- &attr->ia_xtime);
+ &ts);
|
+ ts = timespec64_to_timespec(stat->xtime);
ret = fn (...,
- &stat->xtime);
+ &ts);
)
...+>
}
@ depends on patch @
struct inode *node;
struct inode *node2;
identifier i_xtime1 =~ "^i_[acm]time$";
identifier i_xtime2 =~ "^i_[acm]time$";
identifier i_xtime3 =~ "^i_[acm]time$";
struct iattr *attrp;
struct iattr *attrp2;
struct iattr attr ;
identifier ia_xtime1 =~ "^ia_[acm]time$";
identifier ia_xtime2 =~ "^ia_[acm]time$";
struct kstat *stat;
struct kstat stat1;
struct timespec64 ts;
identifier xtime =~ "^[acmb]time$";
expression e;
@@
(
( node->i_xtime2 \| attrp->ia_xtime2 \| attr.ia_xtime2 \) = node->i_xtime1 ;
|
node->i_xtime2 = \( node2->i_xtime1 \| timespec64_trunc(...) \);
|
node->i_xtime2 = node->i_xtime1 = node->i_xtime3 = \(ts \| current_time(...) \);
|
node->i_xtime1 = node->i_xtime3 = \(ts \| current_time(...) \);
|
stat->xtime = node2->i_xtime1;
|
stat1.xtime = node2->i_xtime1;
|
( node->i_xtime2 \| attrp->ia_xtime2 \) = attrp->ia_xtime1 ;
|
( attrp->ia_xtime1 \| attr.ia_xtime1 \) = attrp2->ia_xtime2;
|
- e = node->i_xtime1;
+ e = timespec64_to_timespec( node->i_xtime1 );
|
- e = attrp->ia_xtime1;
+ e = timespec64_to_timespec( attrp->ia_xtime1 );
|
node->i_xtime1 = current_time(...);
|
node->i_xtime2 = node->i_xtime1 = node->i_xtime3 =
- e;
+ timespec_to_timespec64(e);
|
node->i_xtime1 = node->i_xtime3 =
- e;
+ timespec_to_timespec64(e);
|
- node->i_xtime1 = e;
+ node->i_xtime1 = timespec_to_timespec64(e);
)
Signed-off-by: Deepa Dinamani <deepa.kernel@gmail.com>
Cc: <anton@tuxera.com>
Cc: <balbi@kernel.org>
Cc: <bfields@fieldses.org>
Cc: <darrick.wong@oracle.com>
Cc: <dhowells@redhat.com>
Cc: <dsterba@suse.com>
Cc: <dwmw2@infradead.org>
Cc: <hch@lst.de>
Cc: <hirofumi@mail.parknet.co.jp>
Cc: <hubcap@omnibond.com>
Cc: <jack@suse.com>
Cc: <jaegeuk@kernel.org>
Cc: <jaharkes@cs.cmu.edu>
Cc: <jslaby@suse.com>
Cc: <keescook@chromium.org>
Cc: <mark@fasheh.com>
Cc: <miklos@szeredi.hu>
Cc: <nico@linaro.org>
Cc: <reiserfs-devel@vger.kernel.org>
Cc: <richard@nod.at>
Cc: <sage@redhat.com>
Cc: <sfrench@samba.org>
Cc: <swhiteho@redhat.com>
Cc: <tj@kernel.org>
Cc: <trond.myklebust@primarydata.com>
Cc: <tytso@mit.edu>
Cc: <viro@zeniv.linux.org.uk>
2018-05-08 19:36:02 -07:00
|
|
|
|
2023-10-04 14:52:20 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
|
|
|
|
|
2023-07-05 15:01:07 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
vfs: change inode times to use struct timespec64
struct timespec is not y2038 safe. Transition vfs to use
y2038 safe struct timespec64 instead.
The change was made with the help of the following cocinelle
script. This catches about 80% of the changes.
All the header file and logic changes are included in the
first 5 rules. The rest are trivial substitutions.
I avoid changing any of the function signatures or any other
filesystem specific data structures to keep the patch simple
for review.
The script can be a little shorter by combining different cases.
But, this version was sufficient for my usecase.
virtual patch
@ depends on patch @
identifier now;
@@
- struct timespec
+ struct timespec64
current_time ( ... )
{
- struct timespec now = current_kernel_time();
+ struct timespec64 now = current_kernel_time64();
...
- return timespec_trunc(
+ return timespec64_trunc(
... );
}
@ depends on patch @
identifier xtime;
@@
struct \( iattr \| inode \| kstat \) {
...
- struct timespec xtime;
+ struct timespec64 xtime;
...
}
@ depends on patch @
identifier t;
@@
struct inode_operations {
...
int (*update_time) (...,
- struct timespec t,
+ struct timespec64 t,
...);
...
}
@ depends on patch @
identifier t;
identifier fn_update_time =~ "update_time$";
@@
fn_update_time (...,
- struct timespec *t,
+ struct timespec64 *t,
...) { ... }
@ depends on patch @
identifier t;
@@
lease_get_mtime( ... ,
- struct timespec *t
+ struct timespec64 *t
) { ... }
@te depends on patch forall@
identifier ts;
local idexpression struct inode *inode_node;
identifier i_xtime =~ "^i_[acm]time$";
identifier ia_xtime =~ "^ia_[acm]time$";
identifier fn_update_time =~ "update_time$";
identifier fn;
expression e, E3;
local idexpression struct inode *node1;
local idexpression struct inode *node2;
local idexpression struct iattr *attr1;
local idexpression struct iattr *attr2;
local idexpression struct iattr attr;
identifier i_xtime1 =~ "^i_[acm]time$";
identifier i_xtime2 =~ "^i_[acm]time$";
identifier ia_xtime1 =~ "^ia_[acm]time$";
identifier ia_xtime2 =~ "^ia_[acm]time$";
@@
(
(
- struct timespec ts;
+ struct timespec64 ts;
|
- struct timespec ts = current_time(inode_node);
+ struct timespec64 ts = current_time(inode_node);
)
<+... when != ts
(
- timespec_equal(&inode_node->i_xtime, &ts)
+ timespec64_equal(&inode_node->i_xtime, &ts)
|
- timespec_equal(&ts, &inode_node->i_xtime)
+ timespec64_equal(&ts, &inode_node->i_xtime)
|
- timespec_compare(&inode_node->i_xtime, &ts)
+ timespec64_compare(&inode_node->i_xtime, &ts)
|
- timespec_compare(&ts, &inode_node->i_xtime)
+ timespec64_compare(&ts, &inode_node->i_xtime)
|
ts = current_time(e)
|
fn_update_time(..., &ts,...)
|
inode_node->i_xtime = ts
|
node1->i_xtime = ts
|
ts = inode_node->i_xtime
|
<+... attr1->ia_xtime ...+> = ts
|
ts = attr1->ia_xtime
|
ts.tv_sec
|
ts.tv_nsec
|
btrfs_set_stack_timespec_sec(..., ts.tv_sec)
|
btrfs_set_stack_timespec_nsec(..., ts.tv_nsec)
|
- ts = timespec64_to_timespec(
+ ts =
...
-)
|
- ts = ktime_to_timespec(
+ ts = ktime_to_timespec64(
...)
|
- ts = E3
+ ts = timespec_to_timespec64(E3)
|
- ktime_get_real_ts(&ts)
+ ktime_get_real_ts64(&ts)
|
fn(...,
- ts
+ timespec64_to_timespec(ts)
,...)
)
...+>
(
<... when != ts
- return ts;
+ return timespec64_to_timespec(ts);
...>
)
|
- timespec_equal(&node1->i_xtime1, &node2->i_xtime2)
+ timespec64_equal(&node1->i_xtime2, &node2->i_xtime2)
|
- timespec_equal(&node1->i_xtime1, &attr2->ia_xtime2)
+ timespec64_equal(&node1->i_xtime2, &attr2->ia_xtime2)
|
- timespec_compare(&node1->i_xtime1, &node2->i_xtime2)
+ timespec64_compare(&node1->i_xtime1, &node2->i_xtime2)
|
node1->i_xtime1 =
- timespec_trunc(attr1->ia_xtime1,
+ timespec64_trunc(attr1->ia_xtime1,
...)
|
- attr1->ia_xtime1 = timespec_trunc(attr2->ia_xtime2,
+ attr1->ia_xtime1 = timespec64_trunc(attr2->ia_xtime2,
...)
|
- ktime_get_real_ts(&attr1->ia_xtime1)
+ ktime_get_real_ts64(&attr1->ia_xtime1)
|
- ktime_get_real_ts(&attr.ia_xtime1)
+ ktime_get_real_ts64(&attr.ia_xtime1)
)
@ depends on patch @
struct inode *node;
struct iattr *attr;
identifier fn;
identifier i_xtime =~ "^i_[acm]time$";
identifier ia_xtime =~ "^ia_[acm]time$";
expression e;
@@
(
- fn(node->i_xtime);
+ fn(timespec64_to_timespec(node->i_xtime));
|
fn(...,
- node->i_xtime);
+ timespec64_to_timespec(node->i_xtime));
|
- e = fn(attr->ia_xtime);
+ e = fn(timespec64_to_timespec(attr->ia_xtime));
)
@ depends on patch forall @
struct inode *node;
struct iattr *attr;
identifier i_xtime =~ "^i_[acm]time$";
identifier ia_xtime =~ "^ia_[acm]time$";
identifier fn;
@@
{
+ struct timespec ts;
<+...
(
+ ts = timespec64_to_timespec(node->i_xtime);
fn (...,
- &node->i_xtime,
+ &ts,
...);
|
+ ts = timespec64_to_timespec(attr->ia_xtime);
fn (...,
- &attr->ia_xtime,
+ &ts,
...);
)
...+>
}
@ depends on patch forall @
struct inode *node;
struct iattr *attr;
struct kstat *stat;
identifier ia_xtime =~ "^ia_[acm]time$";
identifier i_xtime =~ "^i_[acm]time$";
identifier xtime =~ "^[acm]time$";
identifier fn, ret;
@@
{
+ struct timespec ts;
<+...
(
+ ts = timespec64_to_timespec(node->i_xtime);
ret = fn (...,
- &node->i_xtime,
+ &ts,
...);
|
+ ts = timespec64_to_timespec(node->i_xtime);
ret = fn (...,
- &node->i_xtime);
+ &ts);
|
+ ts = timespec64_to_timespec(attr->ia_xtime);
ret = fn (...,
- &attr->ia_xtime,
+ &ts,
...);
|
+ ts = timespec64_to_timespec(attr->ia_xtime);
ret = fn (...,
- &attr->ia_xtime);
+ &ts);
|
+ ts = timespec64_to_timespec(stat->xtime);
ret = fn (...,
- &stat->xtime);
+ &ts);
)
...+>
}
@ depends on patch @
struct inode *node;
struct inode *node2;
identifier i_xtime1 =~ "^i_[acm]time$";
identifier i_xtime2 =~ "^i_[acm]time$";
identifier i_xtime3 =~ "^i_[acm]time$";
struct iattr *attrp;
struct iattr *attrp2;
struct iattr attr ;
identifier ia_xtime1 =~ "^ia_[acm]time$";
identifier ia_xtime2 =~ "^ia_[acm]time$";
struct kstat *stat;
struct kstat stat1;
struct timespec64 ts;
identifier xtime =~ "^[acmb]time$";
expression e;
@@
(
( node->i_xtime2 \| attrp->ia_xtime2 \| attr.ia_xtime2 \) = node->i_xtime1 ;
|
node->i_xtime2 = \( node2->i_xtime1 \| timespec64_trunc(...) \);
|
node->i_xtime2 = node->i_xtime1 = node->i_xtime3 = \(ts \| current_time(...) \);
|
node->i_xtime1 = node->i_xtime3 = \(ts \| current_time(...) \);
|
stat->xtime = node2->i_xtime1;
|
stat1.xtime = node2->i_xtime1;
|
( node->i_xtime2 \| attrp->ia_xtime2 \) = attrp->ia_xtime1 ;
|
( attrp->ia_xtime1 \| attr.ia_xtime1 \) = attrp2->ia_xtime2;
|
- e = node->i_xtime1;
+ e = timespec64_to_timespec( node->i_xtime1 );
|
- e = attrp->ia_xtime1;
+ e = timespec64_to_timespec( attrp->ia_xtime1 );
|
node->i_xtime1 = current_time(...);
|
node->i_xtime2 = node->i_xtime1 = node->i_xtime3 =
- e;
+ timespec_to_timespec64(e);
|
node->i_xtime1 = node->i_xtime3 =
- e;
+ timespec_to_timespec64(e);
|
- node->i_xtime1 = e;
+ node->i_xtime1 = timespec_to_timespec64(e);
)
Signed-off-by: Deepa Dinamani <deepa.kernel@gmail.com>
Cc: <anton@tuxera.com>
Cc: <balbi@kernel.org>
Cc: <bfields@fieldses.org>
Cc: <darrick.wong@oracle.com>
Cc: <dhowells@redhat.com>
Cc: <dsterba@suse.com>
Cc: <dwmw2@infradead.org>
Cc: <hch@lst.de>
Cc: <hirofumi@mail.parknet.co.jp>
Cc: <hubcap@omnibond.com>
Cc: <jack@suse.com>
Cc: <jaegeuk@kernel.org>
Cc: <jaharkes@cs.cmu.edu>
Cc: <jslaby@suse.com>
Cc: <keescook@chromium.org>
Cc: <mark@fasheh.com>
Cc: <miklos@szeredi.hu>
Cc: <nico@linaro.org>
Cc: <reiserfs-devel@vger.kernel.org>
Cc: <richard@nod.at>
Cc: <sage@redhat.com>
Cc: <sfrench@samba.org>
Cc: <swhiteho@redhat.com>
Cc: <tj@kernel.org>
Cc: <trond.myklebust@primarydata.com>
Cc: <tytso@mit.edu>
Cc: <viro@zeniv.linux.org.uk>
2018-05-08 19:36:02 -07:00
|
|
|
|
2023-07-05 15:01:07 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:21:09 -07:00
|
|
|
|
2008-01-28 23:58:26 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-04-29 18:23:10 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:21:09 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-11-08 15:18:54 -05:00
|
|
|
|
2020-10-15 13:37:55 -07:00
|
|
|
|
2012-11-08 15:18:54 -05:00
|
|
|
|
2016-04-01 01:31:28 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-05-23 23:49:51 -04:00
|
|
|
|
2016-04-01 01:31:28 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-05-23 23:49:51 -04:00
|
|
|
|
2016-04-01 01:31:28 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2017-07-30 22:30:11 -04:00
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-01-10 12:13:42 -05:00
|
|
|
|
2011-01-10 12:18:25 -05:00
|
|
|
|
2010-01-24 14:34:07 -05:00
|
|
|
|
2011-01-10 12:18:25 -05:00
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2022-01-21 15:06:11 +08:00
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
2020-10-15 13:37:57 -07:00
|
|
|
|
|
|
|
|
|
2022-02-21 13:26:15 +05:30
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:57 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-06-03 21:15:24 +08:00
|
|
|
|
2020-10-15 13:37:57 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-01-10 12:29:43 -05:00
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
2014-04-21 14:37:55 -04:00
|
|
|
|
|
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
2018-07-29 15:51:00 -04:00
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
2018-07-29 15:51:00 -04:00
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
2020-08-17 15:36:15 +08:00
|
|
|
|
2024-06-03 21:15:24 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: Use rbtrees to manage PAs instead of inode i_prealloc_list
Currently, the kernel uses i_prealloc_list to hold all the inode
preallocations. This is known to cause degradation in performance in
workloads which perform large number of sparse writes on a single file.
This is mainly because functions like ext4_mb_normalize_request() and
ext4_mb_use_preallocated() iterate over this complete list, resulting in
slowdowns when large number of PAs are present.
Patch 27bc446e2 partially fixed this by enforcing a limit of 512 for
the inode preallocation list and adding logic to continually trim the
list if it grows above the threshold, however our testing revealed that
a hardcoded value is not suitable for all kinds of workloads.
To optimize this, add an rbtree to the inode and hold the inode
preallocations in this rbtree. This will make iterating over inode PAs
faster and scale much better than a linked list. Additionally, we also
had to remove the LRU logic that was added during trimming of the list
(in ext4_mb_release_context()) as it will add extra overhead in rbtree.
The discards now happen in the lowest-logical-offset-first order.
** Locking notes **
With the introduction of rbtree to maintain inode PAs, we can't use RCU
to walk the tree for searching since it can result in partial traversals
which might miss some nodes(or entire subtrees) while discards happen
in parallel (which happens under a lock). Hence this patch converts the
ei->i_prealloc_lock spin_lock to rw_lock.
Almost all the codepaths that read/modify the PA rbtrees are protected
by the higher level inode->i_data_sem (except
ext4_mb_discard_group_preallocations() and ext4_clear_inode()) IIUC, the
only place we need lock protection is when one thread is reading
"searching" the PA rbtree (earlier protected under rcu_read_lock()) and
another is "deleting" the PAs in ext4_mb_discard_group_preallocations()
function (which iterates all the PAs using the grp->bb_prealloc_list and
deletes PAs from the tree without taking any inode lock (i_data_sem)).
So, this patch converts all rcu_read_lock/unlock() paths for inode list
PA to use read_lock() and all places where we were using
ei->i_prealloc_lock spinlock will now be using write_lock().
Note that this makes the fast path (searching of the right PA e.g.
ext4_mb_use_preallocated() or ext4_mb_normalize_request()), now use
read_lock() instead of rcu_read_lock/unlock(). Ths also will now block
due to slow discard path (ext4_mb_discard_group_preallocations()) which
uses write_lock().
But this is not as bad as it looks. This is because -
1. The slow path only occurs when the normal allocation failed and we
can say that we are low on disk space. One can argue this scenario
won't be much frequent.
2. ext4_mb_discard_group_preallocations(), locks and unlocks the rwlock
for deleting every individual PA. This gives enough opportunity for
the fast path to acquire the read_lock for searching the PA inode
list.
Suggested-by: Ritesh Harjani (IBM) <ritesh.list@gmail.com>
Signed-off-by: Ojaswin Mujoo <ojaswin@linux.ibm.com>
Reviewed-by: Ritesh Harjani (IBM) <ritesh.list@gmail.com>
Reviewed-by: Jan Kara <jack@suse.cz>
Link: https://lore.kernel.org/r/4137bce8f6948fedd8bae134dabae24acfe699c6.1679731817.git.ojaswin@linux.ibm.com
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2023-03-25 13:43:41 +05:30
|
|
|
|
|
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
2012-11-08 15:18:54 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-11-25 11:45:37 -05:00
|
|
|
|
2014-09-01 22:26:49 -04:00
|
|
|
|
2014-11-25 11:45:37 -05:00
|
|
|
|
2014-11-25 11:51:23 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-11-08 15:18:54 -05:00
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2018-10-01 14:17:41 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-12-10 14:04:46 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-12-14 15:21:14 +03:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-09-28 15:48:29 -04:00
|
|
|
|
2013-06-04 14:21:02 -04:00
|
|
|
|
2010-03-04 16:14:02 -05:00
|
|
|
|
2013-06-04 14:21:02 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-01-10 12:13:42 -05:00
|
|
|
|
|
|
|
|
|
2009-12-08 23:51:10 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-04-29 18:31:10 -04:00
|
|
|
|
2014-09-29 14:58:25 +02:00
|
|
|
|
2024-02-06 15:08:19 +01:00
|
|
|
|
2014-09-29 14:58:25 +02:00
|
|
|
|
|
|
|
|
|
2012-04-29 18:31:10 -04:00
|
|
|
|
|
|
|
|
|
2015-04-12 00:43:56 -04:00
|
|
|
|
2016-01-08 16:01:21 -05:00
|
|
|
|
2009-05-01 13:44:33 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:59 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2008-02-10 01:11:44 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2013-04-03 22:12:52 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2017-06-22 11:55:14 -04:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-03-03 17:56:23 -05:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2015-02-16 15:59:38 -08:00
|
|
|
|
2020-05-28 07:59:57 -07:00
|
|
|
|
2015-02-16 15:59:38 -08:00
|
|
|
|
2020-05-28 07:59:57 -07:00
|
|
|
|
2015-02-16 15:59:38 -08:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-03-16 23:12:23 -04:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2016-09-05 23:08:16 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-03-04 16:14:02 -05:00
|
|
|
|
2009-11-02 10:15:27 -08:00
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2018-06-12 23:34:57 -04:00
|
|
|
|
2021-04-01 10:21:29 -07:00
|
|
|
|
2008-07-11 19:27:31 -04:00
|
|
|
|
2008-10-10 22:12:43 -04:00
|
|
|
|
2009-05-17 15:38:01 -04:00
|
|
|
|
2009-11-19 14:25:42 -05:00
|
|
|
|
ext4: add support for lazy inode table initialization
When the lazy_itable_init extended option is passed to mke2fs, it
considerably speeds up filesystem creation because inode tables are
not zeroed out. The fact that parts of the inode table are
uninitialized is not a problem so long as the block group descriptors,
which contain information regarding how much of the inode table has
been initialized, has not been corrupted However, if the block group
checksums are not valid, e2fsck must scan the entire inode table, and
the the old, uninitialized data could potentially cause e2fsck to
report false problems.
Hence, it is important for the inode tables to be initialized as soon
as possble. This commit adds this feature so that mke2fs can safely
use the lazy inode table initialization feature to speed up formatting
file systems.
This is done via a new new kernel thread called ext4lazyinit, which is
created on demand and destroyed, when it is no longer needed. There
is only one thread for all ext4 filesystems in the system. When the
first filesystem with inititable mount option is mounted, ext4lazyinit
thread is created, then the filesystem can register its request in the
request list.
This thread then walks through the list of requests picking up
scheduled requests and invoking ext4_init_inode_table(). Next schedule
time for the request is computed by multiplying the time it took to
zero out last inode table with wait multiplier, which can be set with
the (init_itable=n) mount option (default is 10). We are doing
this so we do not take the whole I/O bandwidth. When the thread is no
longer necessary (request list is empty) it frees the appropriate
structures and exits (and can be created later later by another
filesystem).
We do not disturb regular inode allocations in any way, it just do not
care whether the inode table is, or is not zeroed. But when zeroing, we
have to skip used inodes, obviously. Also we should prevent new inode
allocations from the group, while zeroing is on the way. For that we
take write alloc_sem lock in ext4_init_inode_table() and read alloc_sem
in the ext4_claim_inode, so when we are unlucky and allocator hits the
group which is currently being zeroed, it just has to wait.
This can be suppresed using the mount option no_init_itable.
Signed-off-by: Lukas Czerner <lczerner@redhat.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2010-10-27 21:30:05 -04:00
|
|
|
|
2008-10-10 22:12:43 -04:00
|
|
|
|
2013-04-03 22:12:52 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-09-03 18:22:38 -04:00
|
|
|
|
|
|
|
|
|
2013-04-03 22:12:52 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-03-24 14:09:06 -04:00
|
|
|
|
|
|
|
|
|
ext4: do not allow journal_opts for fs w/o journal
It is appeared that we can pass journal related mount options and such options
be shown in /proc/mounts
Example:
#mkfs.ext4 -F /dev/vdb
#tune2fs -O ^has_journal /dev/vdb
#mount /dev/vdb /mnt/ -ocommit=20,journal_async_commit
#cat /proc/mounts | grep /mnt
/dev/vdb /mnt ext4 rw,relatime,journal_checksum,journal_async_commit,commit=20,data=ordered 0 0
But options:"journal_checksum,journal_async_commit,commit=20,data=ordered" has
nothing with reality because there is no journal at all.
This patch disallow following options for journalless configurations:
- journal_checksum
- journal_async_commit
- commit=%ld
- data={writeback,ordered,journal}
Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
Reviewed-by: Andreas Dilger <adilger@dilger.ca>
2015-10-18 23:50:26 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:54 -07:00
|
|
|
|
2020-11-11 10:32:09 -08:00
|
|
|
|
|
|
|
|
|
ext4: improve cr 0 / cr 1 group scanning
Instead of traversing through groups linearly, scan groups in specific
orders at cr 0 and cr 1. At cr 0, we want to find groups that have the
largest free order >= the order of the request. So, with this patch,
we maintain lists for each possible order and insert each group into a
list based on the largest free order in its buddy bitmap. During cr 0
allocation, we traverse these lists in the increasing order of largest
free orders. This allows us to find a group with the best available cr
0 match in constant time. If nothing can be found, we fallback to cr 1
immediately.
At CR1, the story is slightly different. We want to traverse in the
order of increasing average fragment size. For CR1, we maintain a rb
tree of groupinfos which is sorted by average fragment size. Instead
of traversing linearly, at CR1, we traverse in the order of increasing
average fragment size, starting at the most optimal group. This brings
down cr 1 search complexity to log(num groups).
For cr >= 2, we just perform the linear search as before. Also, in
case of lock contention, we intermittently fallback to linear search
even in CR 0 and CR 1 cases. This allows us to proceed during the
allocation path even in case of high contention.
There is an opportunity to do optimization at CR2 too. That's because
at CR2 we only consider groups where bb_free counter (number of free
blocks) is greater than the request extent size. That's left as future
work.
All the changes introduced in this patch are protected under a new
mount option "mb_optimize_scan".
With this patchset, following experiment was performed:
Created a highly fragmented disk of size 65TB. The disk had no
contiguous 2M regions. Following command was run consecutively for 3
times:
time dd if=/dev/urandom of=file bs=2M count=10
Here are the results with and without cr 0/1 optimizations introduced
in this patch:
|---------+------------------------------+---------------------------|
| | Without CR 0/1 Optimizations | With CR 0/1 Optimizations |
|---------+------------------------------+---------------------------|
| 1st run | 5m1.871s | 2m47.642s |
| 2nd run | 2m28.390s | 0m0.611s |
| 3rd run | 2m26.530s | 0m1.255s |
|---------+------------------------------+---------------------------|
Signed-off-by: Harshad Shirwadkar <harshadshirwadkar@gmail.com>
Reported-by: kernel test robot <lkp@intel.com>
Reported-by: Dan Carpenter <dan.carpenter@oracle.com>
Reviewed-by: Andreas Dilger <adilger@dilger.ca>
Link: https://lore.kernel.org/r/20210401172129.189766-6-harshadshirwadkar@gmail.com
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2021-04-01 10:21:27 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-06-16 18:50:50 +02:00
|
|
|
|
2020-10-15 13:37:54 -07:00
|
|
|
|
2010-12-15 20:26:48 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2010-12-15 20:30:48 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: use proper little-endian bitops
ext4_{set,clear}_bit() is defined as __test_and_{set,clear}_bit_le() for
ext4. Only two ext4_{set,clear}_bit() calls check the return value. The
rest of calls ignore the return value and they can be replaced with
__{set,clear}_bit_le().
This changes ext4_{set,clear}_bit() from __test_and_{set,clear}_bit_le()
to __{set,clear}_bit_le() and introduces ext4_test_and_{set,clear}_bit()
for the two places where old bit needs to be returned.
This ext4_{set,clear}_bit() change is considered safe, because if someone
uses these macros without noticing the change, new ext4_{set,clear}_bit
don't have return value and causes compiler errors where the return value
is used.
This also removes unused ext4_find_first_zero_bit().
Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2011-12-28 20:32:07 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 16:42:07 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2022-02-16 12:32:47 +05:30
|
|
|
|
2011-07-26 22:05:53 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2012-04-29 18:23:10 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-12-13 14:56:18 +01:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-09-09 18:34:51 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2011-09-09 18:34:51 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:21:14 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-05-23 15:54:12 -07:00
|
|
|
|
2018-08-27 01:15:11 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-07-27 11:56:04 -04:00
|
|
|
|
2006-10-11 01:21:14 -07:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
|
|
|
|
|
2023-03-01 13:38:42 +00:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-05-24 18:31:25 -04:00
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
|
|
|
|
|
2008-07-11 19:27:31 -04:00
|
|
|
|
2012-04-29 18:23:10 -04:00
|
|
|
|
2015-04-11 07:44:12 -04:00
|
|
|
|
|
|
|
|
|
2009-02-28 19:39:58 -05:00
|
|
|
|
2010-06-29 11:00:23 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-07-27 11:56:03 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2018-08-27 01:15:11 -04:00
|
|
|
|
2010-07-27 11:56:03 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2018-08-27 01:15:11 -04:00
|
|
|
|
2010-08-01 23:14:20 -04:00
|
|
|
|
|
|
|
|
|
2011-09-09 18:34:51 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-05-12 10:16:06 -04:00
|
|
|
|
2015-01-19 16:00:58 -05:00
|
|
|
|
2015-04-11 07:44:12 -04:00
|
|
|
|
|
|
|
|
|
2015-10-17 16:15:18 -04:00
|
|
|
|
2015-10-17 16:16:02 -04:00
|
|
|
|
2018-07-29 15:51:48 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-11-19 21:54:15 -05:00
|
|
|
|
|
|
|
|
|
2019-04-25 14:05:42 -04:00
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
|
|
|
|
|
2012-04-29 18:23:10 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2010-07-27 11:56:03 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2009-06-13 10:09:48 -04:00
|
|
|
|
2014-09-11 11:15:15 -04:00
|
|
|
|
2016-01-08 16:01:22 -05:00
|
|
|
|
2014-09-11 11:15:15 -04:00
|
|
|
|
2019-04-25 14:05:42 -04:00
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:04 +02:00
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
2021-08-16 11:57:04 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:08 +02:00
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-09-09 18:34:51 -04:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: limit block allocations for indirect-block files to < 2^32
Today, the ext4 allocator will happily allocate blocks past
2^32 for indirect-block files, which results in the block
numbers getting truncated, and corruption ensues.
This patch limits such allocations to < 2^32, and adds
BUG_ONs if we do get blocks larger than that.
This should address RH Bug 519471, ext4 bitmap allocator
must limit blocks to < 2^32
* ext4_find_goal() is modified to choose a goal < UINT_MAX,
so that our starting point is in an acceptable range.
* ext4_xattr_block_set() is modified such that the goal block
is < UINT_MAX, as above.
* ext4_mb_regular_allocator() is modified so that the group
search does not continue into groups which are too high
* ext4_mb_use_preallocated() has a check that we don't use
preallocated space which is too far out
* ext4_alloc_blocks() and ext4_xattr_block_set() add some BUG_ONs
No attempt has been made to limit inode locations to < 2^32,
so we may wind up with blocks far from their inodes. Doing
this much already will lead to some odd ENOSPC issues when the
"lower 32" gets full, and further restricting inodes could
make that even weirder.
For high inodes, choosing a goal of the original, % UINT_MAX,
may be a bit odd, but then we're in an odd situation anyway,
and I don't know of a better heuristic.
Signed-off-by: Eric Sandeen <sandeen@redhat.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2009-09-16 14:45:10 -04:00
|
|
|
|
2012-07-09 16:27:05 -04:00
|
|
|
|
2011-09-09 18:34:51 -04:00
|
|
|
|
|
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-08-27 01:47:11 +08:00
|
|
|
|
2020-02-15 16:40:37 -05:00
|
|
|
|
2009-06-13 10:09:41 -04:00
|
|
|
|
2010-12-15 20:30:48 -05:00
|
|
|
|
2020-11-05 19:59:09 -08:00
|
|
|
|
2012-03-04 19:27:31 -05:00
|
|
|
|
2023-01-29 11:49:39 +08:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
2013-04-09 22:11:22 -04:00
|
|
|
|
2012-02-07 15:41:49 -08:00
|
|
|
|
|
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-06-13 11:45:35 -04:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
2021-05-27 16:55:57 -07:00
|
|
|
|
2011-09-09 18:56:51 -04:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
2011-09-09 18:56:51 -04:00
|
|
|
|
2021-02-18 10:11:32 -05:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-05-30 22:56:46 -04:00
|
|
|
|
2021-04-30 21:50:46 +03:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2017-02-05 01:27:48 -05:00
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-01-23 14:26:38 +01:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
2018-10-12 09:28:09 -04:00
|
|
|
|
|
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-09-24 11:03:43 +08:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-02-18 19:08:50 -08:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-09-05 01:31:50 -04:00
|
|
|
|
2016-06-26 18:24:01 -04:00
|
|
|
|
2023-06-12 20:40:17 +08:00
|
|
|
|
2017-06-22 23:54:33 -04:00
|
|
|
|
2021-07-24 15:41:23 +08:00
|
|
|
|
|
|
|
|
|
2021-08-30 15:52:46 +08:00
|
|
|
|
2022-09-08 11:21:28 +02:00
|
|
|
|
|
|
|
|
|
ext4: improve cr 0 / cr 1 group scanning
Instead of traversing through groups linearly, scan groups in specific
orders at cr 0 and cr 1. At cr 0, we want to find groups that have the
largest free order >= the order of the request. So, with this patch,
we maintain lists for each possible order and insert each group into a
list based on the largest free order in its buddy bitmap. During cr 0
allocation, we traverse these lists in the increasing order of largest
free orders. This allows us to find a group with the best available cr
0 match in constant time. If nothing can be found, we fallback to cr 1
immediately.
At CR1, the story is slightly different. We want to traverse in the
order of increasing average fragment size. For CR1, we maintain a rb
tree of groupinfos which is sorted by average fragment size. Instead
of traversing linearly, at CR1, we traverse in the order of increasing
average fragment size, starting at the most optimal group. This brings
down cr 1 search complexity to log(num groups).
For cr >= 2, we just perform the linear search as before. Also, in
case of lock contention, we intermittently fallback to linear search
even in CR 0 and CR 1 cases. This allows us to proceed during the
allocation path even in case of high contention.
There is an opportunity to do optimization at CR2 too. That's because
at CR2 we only consider groups where bb_free counter (number of free
blocks) is greater than the request extent size. That's left as future
work.
All the changes introduced in this patch are protected under a new
mount option "mb_optimize_scan".
With this patchset, following experiment was performed:
Created a highly fragmented disk of size 65TB. The disk had no
contiguous 2M regions. Following command was run consecutively for 3
times:
time dd if=/dev/urandom of=file bs=2M count=10
Here are the results with and without cr 0/1 optimizations introduced
in this patch:
|---------+------------------------------+---------------------------|
| | Without CR 0/1 Optimizations | With CR 0/1 Optimizations |
|---------+------------------------------+---------------------------|
| 1st run | 5m1.871s | 2m47.642s |
| 2nd run | 2m28.390s | 0m0.611s |
| 3rd run | 2m26.530s | 0m1.255s |
|---------+------------------------------+---------------------------|
Signed-off-by: Harshad Shirwadkar <harshadshirwadkar@gmail.com>
Reported-by: kernel test robot <lkp@intel.com>
Reported-by: Dan Carpenter <dan.carpenter@oracle.com>
Reviewed-by: Andreas Dilger <adilger@dilger.ca>
Link: https://lore.kernel.org/r/20210401172129.189766-6-harshadshirwadkar@gmail.com
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2021-04-01 10:21:27 -07:00
|
|
|
|
|
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: improve cr 0 / cr 1 group scanning
Instead of traversing through groups linearly, scan groups in specific
orders at cr 0 and cr 1. At cr 0, we want to find groups that have the
largest free order >= the order of the request. So, with this patch,
we maintain lists for each possible order and insert each group into a
list based on the largest free order in its buddy bitmap. During cr 0
allocation, we traverse these lists in the increasing order of largest
free orders. This allows us to find a group with the best available cr
0 match in constant time. If nothing can be found, we fallback to cr 1
immediately.
At CR1, the story is slightly different. We want to traverse in the
order of increasing average fragment size. For CR1, we maintain a rb
tree of groupinfos which is sorted by average fragment size. Instead
of traversing linearly, at CR1, we traverse in the order of increasing
average fragment size, starting at the most optimal group. This brings
down cr 1 search complexity to log(num groups).
For cr >= 2, we just perform the linear search as before. Also, in
case of lock contention, we intermittently fallback to linear search
even in CR 0 and CR 1 cases. This allows us to proceed during the
allocation path even in case of high contention.
There is an opportunity to do optimization at CR2 too. That's because
at CR2 we only consider groups where bb_free counter (number of free
blocks) is greater than the request extent size. That's left as future
work.
All the changes introduced in this patch are protected under a new
mount option "mb_optimize_scan".
With this patchset, following experiment was performed:
Created a highly fragmented disk of size 65TB. The disk had no
contiguous 2M regions. Following command was run consecutively for 3
times:
time dd if=/dev/urandom of=file bs=2M count=10
Here are the results with and without cr 0/1 optimizations introduced
in this patch:
|---------+------------------------------+---------------------------|
| | Without CR 0/1 Optimizations | With CR 0/1 Optimizations |
|---------+------------------------------+---------------------------|
| 1st run | 5m1.871s | 2m47.642s |
| 2nd run | 2m28.390s | 0m0.611s |
| 3rd run | 2m26.530s | 0m1.255s |
|---------+------------------------------+---------------------------|
Signed-off-by: Harshad Shirwadkar <harshadshirwadkar@gmail.com>
Reported-by: kernel test robot <lkp@intel.com>
Reported-by: Dan Carpenter <dan.carpenter@oracle.com>
Reviewed-by: Andreas Dilger <adilger@dilger.ca>
Link: https://lore.kernel.org/r/20210401172129.189766-6-harshadshirwadkar@gmail.com
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2021-04-01 10:21:27 -07:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-08-17 09:48:17 -04:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-04-21 10:54:07 +03:00
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:50 +05:30
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:43 +05:30
|
|
|
|
2021-04-01 10:21:25 -07:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
2023-05-30 18:03:44 +05:30
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:50 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-05-30 18:03:42 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-04-01 10:21:23 -07:00
|
|
|
|
|
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: Avoid group preallocation for closed files
Currently the group preallocation code tries to find a large (512)
free block from which to do per-cpu group allocation for small files.
The problem with this scheme is that it leaves the filesystem horribly
fragmented. In the worst case, if the filesystem is unmounted and
remounted (after a system shutdown, for example) we forget the fact
that wee were using a particular (now-partially filled) 512 block
extent. So the next time we try to allocate space for a small file,
we will find *another* completely free 512 block chunk to allocate
small files. Given that there are 32,768 blocks in a block group,
after 64 iterations of "mount, write one 4k file in a directory,
unmount", the block group will have 64 files, each separated by 511
blocks, and the block group will no longer have any free 512
completely free chunks of blocks for group preallocation space.
So if we try to allocate blocks for a file that has been closed, such
that we know the final size of the file, and the filesystem is not
busy, avoid using group preallocation.
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2009-09-18 13:34:02 -04:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
2010-02-02 14:39:01 +09:00
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-08-17 09:54:17 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
2020-02-18 19:08:51 -08:00
|
|
|
|
2012-09-05 01:29:50 -04:00
|
|
|
|
2009-09-28 15:48:41 -04:00
|
|
|
|
2013-06-04 14:21:02 -04:00
|
|
|
|
|
|
|
|
|
2010-07-27 11:56:04 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: add support for lazy inode table initialization
When the lazy_itable_init extended option is passed to mke2fs, it
considerably speeds up filesystem creation because inode tables are
not zeroed out. The fact that parts of the inode table are
uninitialized is not a problem so long as the block group descriptors,
which contain information regarding how much of the inode table has
been initialized, has not been corrupted However, if the block group
checksums are not valid, e2fsck must scan the entire inode table, and
the the old, uninitialized data could potentially cause e2fsck to
report false problems.
Hence, it is important for the inode tables to be initialized as soon
as possble. This commit adds this feature so that mke2fs can safely
use the lazy inode table initialization feature to speed up formatting
file systems.
This is done via a new new kernel thread called ext4lazyinit, which is
created on demand and destroyed, when it is no longer needed. There
is only one thread for all ext4 filesystems in the system. When the
first filesystem with inititable mount option is mounted, ext4lazyinit
thread is created, then the filesystem can register its request in the
request list.
This thread then walks through the list of requests picking up
scheduled requests and invoking ext4_init_inode_table(). Next schedule
time for the request is computed by multiplying the time it took to
zero out last inode table with wait multiplier, which can be set with
the (init_itable=n) mount option (default is 10). We are doing
this so we do not take the whole I/O bandwidth. When the thread is no
longer necessary (request list is empty) it frees the appropriate
structures and exits (and can be created later later by another
filesystem).
We do not disturb regular inode allocations in any way, it just do not
care whether the inode table is, or is not zeroed. But when zeroing, we
have to skip used inodes, obviously. Also we should prevent new inode
allocations from the group, while zeroing is on the way. For that we
take write alloc_sem lock in ext4_init_inode_table() and read alloc_sem
in the ext4_claim_inode, so when we are unlucky and allocator hits the
group which is currently being zeroed, it just has to wait.
This can be suppresed using the mount option no_init_itable.
Signed-off-by: Lukas Czerner <lczerner@redhat.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2010-10-27 21:30:05 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-05-24 18:31:25 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-11 00:03:38 -04:00
|
|
|
|
|
|
|
|
|
2021-11-03 15:51:21 +01:00
|
|
|
|
2012-04-29 18:27:10 -04:00
|
|
|
|
2012-04-29 18:29:10 -04:00
|
|
|
|
|
|
|
|
|
2013-02-18 00:32:55 -05:00
|
|
|
|
|
|
|
|
|
2023-09-11 17:44:29 +08:00
|
|
|
|
2014-11-25 11:51:23 -05:00
|
|
|
|
2014-11-25 11:45:37 -05:00
|
|
|
|
2014-09-01 22:26:49 -04:00
|
|
|
|
2017-06-22 11:28:55 -04:00
|
|
|
|
2017-06-22 11:44:55 -04:00
|
|
|
|
2014-11-25 11:45:37 -05:00
|
|
|
|
2013-10-17 21:11:01 -04:00
|
|
|
|
2021-08-16 11:57:04 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-10-17 21:11:01 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-07-25 12:33:13 +00:00
|
|
|
|
|
|
|
|
|
2016-04-25 23:22:35 -04:00
|
|
|
|
fscrypt: handle test_dummy_encryption in more logical way
The behavior of the test_dummy_encryption mount option is that when a
new file (or directory or symlink) is created in an unencrypted
directory, it's automatically encrypted using a dummy encryption policy.
That's it; in particular, the encryption (or lack thereof) of existing
files (or directories or symlinks) doesn't change.
Unfortunately the implementation of test_dummy_encryption is a bit weird
and confusing. When test_dummy_encryption is enabled and a file is
being created in an unencrypted directory, we set up an encryption key
(->i_crypt_info) for the directory. This isn't actually used to do any
encryption, however, since the directory is still unencrypted! Instead,
->i_crypt_info is only used for inheriting the encryption policy.
One consequence of this is that the filesystem ends up providing a
"dummy context" (policy + nonce) instead of a "dummy policy". In
commit ed318a6cc0b6 ("fscrypt: support test_dummy_encryption=v2"), I
mistakenly thought this was required. However, actually the nonce only
ends up being used to derive a key that is never used.
Another consequence of this implementation is that it allows for
'inode->i_crypt_info != NULL && !IS_ENCRYPTED(inode)', which is an edge
case that can be forgotten about. For example, currently
FS_IOC_GET_ENCRYPTION_POLICY on an unencrypted directory may return the
dummy encryption policy when the filesystem is mounted with
test_dummy_encryption. That seems like the wrong thing to do, since
again, the directory itself is not actually encrypted.
Therefore, switch to a more logical and maintainable implementation
where the dummy encryption policy inheritance is done without setting up
keys for unencrypted directories. This involves:
- Adding a function fscrypt_policy_to_inherit() which returns the
encryption policy to inherit from a directory. This can be a real
policy, a dummy policy, or no policy.
- Replacing struct fscrypt_dummy_context, ->get_dummy_context(), etc.
with struct fscrypt_dummy_policy, ->get_dummy_policy(), etc.
- Making fscrypt_fname_encrypted_size() take an fscrypt_policy instead
of an inode.
Acked-by: Jaegeuk Kim <jaegeuk@kernel.org>
Acked-by: Jeff Layton <jlayton@kernel.org>
Link: https://lore.kernel.org/r/20200917041136.178600-13-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@google.com>
2020-09-16 21:11:35 -07:00
|
|
|
|
|
|
|
|
|
fscrypt: support test_dummy_encryption=v2
v1 encryption policies are deprecated in favor of v2, and some new
features (e.g. encryption+casefolding) are only being added for v2.
Therefore, the "test_dummy_encryption" mount option (which is used for
encryption I/O testing with xfstests) needs to support v2 policies.
To do this, extend its syntax to be "test_dummy_encryption=v1" or
"test_dummy_encryption=v2". The existing "test_dummy_encryption" (no
argument) also continues to be accepted, to specify the default setting
-- currently v1, but the next patch changes it to v2.
To cleanly support both v1 and v2 while also making it easy to support
specifying other encryption settings in the future (say, accepting
"$contents_mode:$filenames_mode:v2"), make ext4 and f2fs maintain a
pointer to the dummy fscrypt_context rather than using mount flags.
To avoid concurrency issues, don't allow test_dummy_encryption to be set
or changed during a remount. (The former restriction is new, but
xfstests doesn't run into it, so no one should notice.)
Tested with 'gce-xfstests -c {ext4,f2fs}/encrypt -g auto'. On ext4,
there are two regressions, both of which are test bugs: ext4/023 and
ext4/028 fail because they set an xattr and expect it to be stored
inline, but the increase in size of the fscrypt_context from
24 to 40 bytes causes this xattr to be spilled into an external block.
Link: https://lore.kernel.org/r/20200512233251.118314-4-ebiggers@kernel.org
Acked-by: Jaegeuk Kim <jaegeuk@kernel.org>
Reviewed-by: Theodore Ts'o <tytso@mit.edu>
Signed-off-by: Eric Biggers <ebiggers@google.com>
2020-05-12 16:32:50 -07:00
|
|
|
|
2020-02-19 10:30:47 -08:00
|
|
|
|
|
|
|
|
|
2023-05-24 15:25:38 +08:00
|
|
|
|
|
|
|
|
|
2020-02-19 10:30:47 -08:00
|
|
|
|
2020-02-19 10:30:46 -08:00
|
|
|
|
2017-08-24 16:42:48 -07:00
|
|
|
|
2021-11-29 11:21:59 +01:00
|
|
|
|
2019-11-21 13:09:43 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-06-20 10:54:23 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:57 -07:00
|
|
|
|
2020-11-27 12:34:00 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-08-22 23:43:38 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-11-27 12:34:00 +01:00
|
|
|
|
2023-08-22 23:43:38 -04:00
|
|
|
|
2020-11-27 12:34:00 +01:00
|
|
|
|
2024-11-04 16:22:57 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-12-23 12:21:38 -08:00
|
|
|
|
2020-10-15 13:37:57 -07:00
|
|
|
|
2021-12-23 12:21:38 -08:00
|
|
|
|
2020-10-15 13:37:57 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2022-01-17 17:36:54 +08:00
|
|
|
|
2020-10-15 13:37:59 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-03 16:33:44 -04:00
|
|
|
|
|
|
|
|
|
2008-09-08 22:25:24 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2023-05-04 14:47:23 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2010-01-24 14:34:07 -05:00
|
|
|
|
2020-02-15 16:40:37 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-11-05 19:59:09 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2022-01-17 17:36:55 +08:00
|
|
|
|
2020-11-05 19:59:09 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-11-21 13:09:43 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-11-19 21:54:15 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-01-24 14:34:07 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-05-17 08:00:00 -04:00
|
|
|
|
2012-12-10 14:04:46 -05:00
|
|
|
|
2013-08-16 22:05:14 -04:00
|
|
|
|
2017-08-24 14:25:02 -04:00
|
|
|
|
2019-07-22 09:26:24 -07:00
|
|
|
|
2020-10-15 13:37:57 -07:00
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
2010-01-24 14:34:07 -05:00
|
|
|
|
|
|
|
|
|
2011-01-10 12:18:25 -05:00
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
|
|
|
|
|
2011-01-10 12:18:25 -05:00
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-01-10 12:18:25 -05:00
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-01-10 12:18:25 -05:00
|
|
|
|
2010-01-24 14:34:07 -05:00
|
|
|
|
|
|
|
|
|
2013-11-11 22:40:40 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-01-10 12:18:25 -05:00
|
|
|
|
2013-11-11 22:40:40 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-01-10 12:18:25 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2019-07-22 09:26:24 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2018-01-21 18:04:24 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-05-12 10:16:06 -04:00
|
|
|
|
2020-11-05 19:58:50 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:54 -07:00
|
|
|
|
2019-10-24 14:54:37 -07:00
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:26 -05:00
|
|
|
|
Ext4: Uninitialized Block Groups
In pass1 of e2fsck, every inode table in the fileystem is scanned and checked,
regardless of whether it is in use. This is this the most time consuming part
of the filesystem check. The unintialized block group feature can greatly
reduce e2fsck time by eliminating checking of uninitialized inodes.
With this feature, there is a a high water mark of used inodes for each block
group. Block and inode bitmaps can be uninitialized on disk via a flag in the
group descriptor to avoid reading or scanning them at e2fsck time. A checksum
of each group descriptor is used to ensure that corruption in the group
descriptor's bit flags does not cause incorrect operation.
The feature is enabled through a mkfs option
mke2fs /dev/ -O uninit_groups
A patch adding support for uninitialized block groups to e2fsprogs tools has
been posted to the linux-ext4 mailing list.
The patches have been stress tested with fsstress and fsx. In performance
tests testing e2fsck time, we have seen that e2fsck time on ext3 grows
linearly with the total number of inodes in the filesytem. In ext4 with the
uninitialized block groups feature, the e2fsck time is constant, based
solely on the number of used inodes rather than the total inode count.
Since typical ext4 filesystems only use 1-10% of their inodes, this feature can
greatly reduce e2fsck time for users. With performance improvement of 2-20
times, depending on how full the filesystem is.
The attached graph shows the major improvements in e2fsck times in filesystems
with a large total inode count, but few inodes in use.
In each group descriptor if we have
EXT4_BG_INODE_UNINIT set in bg_flags:
Inode table is not initialized/used in this group. So we can skip
the consistency check during fsck.
EXT4_BG_BLOCK_UNINIT set in bg_flags:
No block in the group is used. So we can skip the block bitmap
verification for this group.
We also add two new fields to group descriptor as a part of
uninitialized group patch.
__le16 bg_itable_unused; /* Unused inodes count */
__le16 bg_checksum; /* crc16(sb_uuid+group+desc) */
bg_itable_unused:
If we have EXT4_BG_INODE_UNINIT not set in bg_flags
then bg_itable_unused will give the offset within
the inode table till the inodes are used. This can be
used by fsck to skip list of inodes that are marked unused.
bg_checksum:
Now that we depend on bg_flags and bg_itable_unused to determine
the block and inode usage, we need to make sure group descriptor
is not corrupt. We add checksum to group descriptor to
detect corruption. If the descriptor is found to be corrupt, we
mark all the blocks and inodes in the group used.
Signed-off-by: Avantika Mathur <mathur@us.ibm.com>
Signed-off-by: Andreas Dilger <adilger@clusterfs.com>
Signed-off-by: Mingming Cao <cmm@us.ibm.com>
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
2007-10-16 18:38:25 -04:00
|
|
|
|
2007-07-18 08:38:01 -04:00
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
2011-05-24 19:00:39 -04:00
|
|
|
|
2011-09-09 18:34:51 -04:00
|
|
|
|
2012-04-29 18:23:10 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-01-04 22:01:53 -05:00
|
|
|
|
2015-02-12 22:31:21 -05:00
|
|
|
|
2015-10-17 16:15:18 -04:00
|
|
|
|
2019-07-22 09:26:24 -07:00
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:21:03 -07:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
2008-01-28 23:58:26 -05:00
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
2010-01-25 03:31:32 -05:00
|
|
|
|
|
|
|
|
|
2015-10-17 16:16:02 -04:00
|
|
|
|
2012-01-04 22:01:53 -05:00
|
|
|
|
2012-12-10 14:04:46 -05:00
|
|
|
|
2015-01-19 16:00:58 -05:00
|
|
|
|
2019-04-25 14:05:42 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2019-02-14 17:52:18 -05:00
|
|
|
|
|
|
|
|
|
2015-10-17 16:18:43 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-02-14 17:52:18 -05:00
|
|
|
|
2015-10-17 16:18:43 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-02-14 17:52:18 -05:00
|
|
|
|
2015-10-17 16:18:43 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-02-14 17:52:18 -05:00
|
|
|
|
2015-10-17 16:18:43 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:54 -07:00
|
|
|
|
2019-10-24 14:54:37 -07:00
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
2015-10-17 16:18:43 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-07-22 09:26:24 -07:00
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
2015-10-17 16:18:43 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-04-25 14:05:42 -04:00
|
|
|
|
2015-10-17 16:18:43 -04:00
|
|
|
|
2011-04-18 17:29:14 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:21:03 -07:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
2011-05-24 18:31:25 -04:00
|
|
|
|
2017-06-21 21:10:32 -04:00
|
|
|
|
2015-04-16 01:56:00 -04:00
|
|
|
|
|
|
|
|
|
2015-10-17 16:16:02 -04:00
|
|
|
|
2019-04-25 14:05:42 -04:00
|
|
|
|
2017-06-21 21:09:57 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
Ext4: Uninitialized Block Groups
In pass1 of e2fsck, every inode table in the fileystem is scanned and checked,
regardless of whether it is in use. This is this the most time consuming part
of the filesystem check. The unintialized block group feature can greatly
reduce e2fsck time by eliminating checking of uninitialized inodes.
With this feature, there is a a high water mark of used inodes for each block
group. Block and inode bitmaps can be uninitialized on disk via a flag in the
group descriptor to avoid reading or scanning them at e2fsck time. A checksum
of each group descriptor is used to ensure that corruption in the group
descriptor's bit flags does not cause incorrect operation.
The feature is enabled through a mkfs option
mke2fs /dev/ -O uninit_groups
A patch adding support for uninitialized block groups to e2fsprogs tools has
been posted to the linux-ext4 mailing list.
The patches have been stress tested with fsstress and fsx. In performance
tests testing e2fsck time, we have seen that e2fsck time on ext3 grows
linearly with the total number of inodes in the filesytem. In ext4 with the
uninitialized block groups feature, the e2fsck time is constant, based
solely on the number of used inodes rather than the total inode count.
Since typical ext4 filesystems only use 1-10% of their inodes, this feature can
greatly reduce e2fsck time for users. With performance improvement of 2-20
times, depending on how full the filesystem is.
The attached graph shows the major improvements in e2fsck times in filesystems
with a large total inode count, but few inodes in use.
In each group descriptor if we have
EXT4_BG_INODE_UNINIT set in bg_flags:
Inode table is not initialized/used in this group. So we can skip
the consistency check during fsck.
EXT4_BG_BLOCK_UNINIT set in bg_flags:
No block in the group is used. So we can skip the block bitmap
verification for this group.
We also add two new fields to group descriptor as a part of
uninitialized group patch.
__le16 bg_itable_unused; /* Unused inodes count */
__le16 bg_checksum; /* crc16(sb_uuid+group+desc) */
bg_itable_unused:
If we have EXT4_BG_INODE_UNINIT not set in bg_flags
then bg_itable_unused will give the offset within
the inode table till the inodes are used. This can be
used by fsck to skip list of inodes that are marked unused.
bg_checksum:
Now that we depend on bg_flags and bg_itable_unused to determine
the block and inode usage, we need to make sure group descriptor
is not corrupt. We add checksum to group descriptor to
detect corruption. If the descriptor is found to be corrupt, we
mark all the blocks and inodes in the group used.
Signed-off-by: Avantika Mathur <mathur@us.ibm.com>
Signed-off-by: Andreas Dilger <adilger@clusterfs.com>
Signed-off-by: Mingming Cao <cmm@us.ibm.com>
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
2007-10-16 18:38:25 -04:00
|
|
|
|
2007-07-18 08:38:01 -04:00
|
|
|
|
2007-07-18 09:15:20 -04:00
|
|
|
|
2008-01-28 23:58:26 -05:00
|
|
|
|
2011-09-09 19:06:51 -04:00
|
|
|
|
2012-05-27 08:12:42 -04:00
|
|
|
|
ext4: make quota as first class supported feature
This patch adds support for quotas as a first class feature in ext4;
which is to say, the quota files are stored in hidden inodes as file
system metadata, instead of as separate files visible in the file system
directory hierarchy.
It is based on the proposal at:
https://ext4.wiki.kernel.org/index.php/Design_For_1st_Class_Quota_in_Ext4
This patch introduces a new feature - EXT4_FEATURE_RO_COMPAT_QUOTA
which, when turned on, enables quota accounting at mount time
iteself. Also, the quota inodes are stored in two additional superblock
fields. Some changes introduced by this patch that should be pointed
out are:
1) Two new ext4-superblock fields - s_usr_quota_inum and
s_grp_quota_inum for storing the quota inodes in use.
2) Default quota inodes are: inode#3 for tracking userquota and inode#4
for tracking group quota. The superblock fields can be set to use
other inodes as well.
3) If the QUOTA feature and corresponding quota inodes are set in
superblock, the quota usage tracking is turned on at mount time. On
'quotaon' ioctl, the quota limits enforcement is turned
on. 'quotaoff' ioctl turns off only the limits enforcement in this
case.
4) When QUOTA feature is in use, the quota mount options 'quota',
'usrquota', 'grpquota' are ignored by the kernel.
5) mke2fs or tune2fs can be used to set the QUOTA feature and initialize
quota inodes. The default reserved inodes will not be visible to user
as regular files.
6) The quota-tools will need to be modified to support hidden quota
files on ext4. E2fsprogs will also include support for creating and
fixing quota files.
7) Support is only for the new V2 quota file format.
Tested-by: Jan Kara <jack@suse.cz>
Reviewed-by: Jan Kara <jack@suse.cz>
Reviewed-by: Johann Lombardi <johann@whamcloud.com>
Signed-off-by: Aditya Kali <adityakali@google.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2012-07-22 20:21:31 -04:00
|
|
|
|
2016-01-08 16:01:21 -05:00
|
|
|
|
2019-07-22 09:26:24 -07:00
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2015-10-17 16:18:43 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:05 +02:00
|
|
|
|
|
|
|
|
|
2017-02-05 01:27:48 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2017-02-05 01:28:48 -05:00
|
|
|
|
2020-05-28 07:59:58 -07:00
|
|
|
|
2017-02-05 01:28:48 -05:00
|
|
|
|
2023-06-16 18:50:49 +02:00
|
|
|
|
2017-02-05 01:28:48 -05:00
|
|
|
|
2023-06-16 18:50:49 +02:00
|
|
|
|
2017-02-05 01:28:48 -05:00
|
|
|
|
2017-02-05 01:27:48 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2016-01-08 16:01:21 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-10-09 23:53:47 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-08-01 23:14:20 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2009-01-03 20:27:38 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-03-12 12:18:34 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
ext4: fix use-after-free in ext4_search_dir
We got issue as follows:
EXT4-fs (loop0): mounted filesystem without journal. Opts: ,errors=continue
==================================================================
BUG: KASAN: use-after-free in ext4_search_dir fs/ext4/namei.c:1394 [inline]
BUG: KASAN: use-after-free in search_dirblock fs/ext4/namei.c:1199 [inline]
BUG: KASAN: use-after-free in __ext4_find_entry+0xdca/0x1210 fs/ext4/namei.c:1553
Read of size 1 at addr ffff8881317c3005 by task syz-executor117/2331
CPU: 1 PID: 2331 Comm: syz-executor117 Not tainted 5.10.0+ #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.14.0-0-g155821a1990b-prebuilt.qemu.org 04/01/2014
Call Trace:
__dump_stack lib/dump_stack.c:83 [inline]
dump_stack+0x144/0x187 lib/dump_stack.c:124
print_address_description+0x7d/0x630 mm/kasan/report.c:387
__kasan_report+0x132/0x190 mm/kasan/report.c:547
kasan_report+0x47/0x60 mm/kasan/report.c:564
ext4_search_dir fs/ext4/namei.c:1394 [inline]
search_dirblock fs/ext4/namei.c:1199 [inline]
__ext4_find_entry+0xdca/0x1210 fs/ext4/namei.c:1553
ext4_lookup_entry fs/ext4/namei.c:1622 [inline]
ext4_lookup+0xb8/0x3a0 fs/ext4/namei.c:1690
__lookup_hash+0xc5/0x190 fs/namei.c:1451
do_rmdir+0x19e/0x310 fs/namei.c:3760
do_syscall_64+0x33/0x40 arch/x86/entry/common.c:46
entry_SYSCALL_64_after_hwframe+0x44/0xa9
RIP: 0033:0x445e59
Code: 4d c7 fb ff c3 66 2e 0f 1f 84 00 00 00 00 00 66 90 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 0f 83 1b c7 fb ff c3 66 2e 0f 1f 84 00 00 00 00
RSP: 002b:00007fff2277fac8 EFLAGS: 00000246 ORIG_RAX: 0000000000000054
RAX: ffffffffffffffda RBX: 0000000000400280 RCX: 0000000000445e59
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 00000000200000c0
RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000002
R10: 00007fff2277f990 R11: 0000000000000246 R12: 0000000000000000
R13: 431bde82d7b634db R14: 0000000000000000 R15: 0000000000000000
The buggy address belongs to the page:
page:0000000048cd3304 refcount:0 mapcount:0 mapping:0000000000000000 index:0x1 pfn:0x1317c3
flags: 0x200000000000000()
raw: 0200000000000000 ffffea0004526588 ffffea0004528088 0000000000000000
raw: 0000000000000001 0000000000000000 00000000ffffffff 0000000000000000
page dumped because: kasan: bad access detected
Memory state around the buggy address:
ffff8881317c2f00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ffff8881317c2f80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
>ffff8881317c3000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
^
ffff8881317c3080: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
ffff8881317c3100: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
==================================================================
ext4_search_dir:
...
de = (struct ext4_dir_entry_2 *)search_buf;
dlimit = search_buf + buf_size;
while ((char *) de < dlimit) {
...
if ((char *) de + de->name_len <= dlimit &&
ext4_match(dir, fname, de)) {
...
}
...
de_len = ext4_rec_len_from_disk(de->rec_len, dir->i_sb->s_blocksize);
if (de_len <= 0)
return -1;
offset += de_len;
de = (struct ext4_dir_entry_2 *) ((char *) de + de_len);
}
Assume:
de=0xffff8881317c2fff
dlimit=0x0xffff8881317c3000
If read 'de->name_len' which address is 0xffff8881317c3005, obviously is
out of range, then will trigger use-after-free.
To solve this issue, 'dlimit' must reserve 8 bytes, as we will read
'de->name_len' to judge if '(char *) de + de->name_len' out of range.
Signed-off-by: Ye Bin <yebin10@huawei.com>
Reviewed-by: Jan Kara <jack@suse.cz>
Link: https://lore.kernel.org/r/20220324064816.1209985-1-yebin10@huawei.com
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
Cc: stable@kernel.org
2022-03-24 14:48:16 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2021-03-19 07:34:13 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-05-22 16:07:58 +01:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2021-03-19 07:34:13 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-06-06 13:43:16 +08:00
|
|
|
|
2021-03-19 07:34:13 +00:00
|
|
|
|
2024-06-06 13:43:16 +08:00
|
|
|
|
2021-03-19 07:34:13 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-04-29 18:23:10 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-12-10 14:05:59 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2012-04-29 18:23:10 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
|
|
|
|
|
2021-03-19 07:34:13 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-08-05 01:46:37 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-04-01 15:29:48 +03:00
|
|
|
|
2010-08-05 01:46:37 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-12-28 07:32:52 +00:00
|
|
|
|
2016-04-01 15:29:48 +03:00
|
|
|
|
2010-08-05 01:46:37 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-10-17 16:18:43 -04:00
|
|
|
|
2010-05-16 22:00:00 -04:00
|
|
|
|
2017-08-05 19:47:34 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2018-01-07 23:36:19 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-10-28 13:21:44 -04:00
|
|
|
|
|
|
|
|
|
2021-03-19 07:34:13 +00:00
|
|
|
|
2024-08-27 16:16:36 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2012-04-29 18:27:10 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-12-01 17:08:41 -08:00
|
|
|
|
2012-04-29 18:27:10 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-03-18 22:44:40 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-05-18 13:14:47 -04:00
|
|
|
|
|
|
|
|
|
2016-07-10 14:01:03 -04:00
|
|
|
|
2015-05-18 13:14:47 -04:00
|
|
|
|
2018-12-12 15:20:12 +05:30
|
|
|
|
2016-07-10 14:01:03 -04:00
|
|
|
|
2015-05-18 13:14:47 -04:00
|
|
|
|
2022-01-18 07:56:14 +01:00
|
|
|
|
2024-06-06 10:33:47 +03:00
|
|
|
|
2019-06-19 23:45:09 -04:00
|
|
|
|
2015-05-18 13:14:47 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-03-19 07:34:13 +00:00
|
|
|
|
2015-05-18 13:14:47 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2017-06-22 11:44:55 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2017-06-22 11:31:25 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-08-30 15:04:52 +02:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-08-23 17:47:19 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2017-06-21 21:09:57 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: add support for lazy inode table initialization
When the lazy_itable_init extended option is passed to mke2fs, it
considerably speeds up filesystem creation because inode tables are
not zeroed out. The fact that parts of the inode table are
uninitialized is not a problem so long as the block group descriptors,
which contain information regarding how much of the inode table has
been initialized, has not been corrupted However, if the block group
checksums are not valid, e2fsck must scan the entire inode table, and
the the old, uninitialized data could potentially cause e2fsck to
report false problems.
Hence, it is important for the inode tables to be initialized as soon
as possble. This commit adds this feature so that mke2fs can safely
use the lazy inode table initialization feature to speed up formatting
file systems.
This is done via a new new kernel thread called ext4lazyinit, which is
created on demand and destroyed, when it is no longer needed. There
is only one thread for all ext4 filesystems in the system. When the
first filesystem with inititable mount option is mounted, ext4lazyinit
thread is created, then the filesystem can register its request in the
request list.
This thread then walks through the list of requests picking up
scheduled requests and invoking ext4_init_inode_table(). Next schedule
time for the request is computed by multiplying the time it took to
zero out last inode table with wait multiplier, which can be set with
the (init_itable=n) mount option (default is 10). We are doing
this so we do not take the whole I/O bandwidth. When the thread is no
longer necessary (request list is empty) it frees the appropriate
structures and exits (and can be created later later by another
filesystem).
We do not disturb regular inode allocations in any way, it just do not
care whether the inode table is, or is not zeroed. But when zeroing, we
have to skip used inodes, obviously. Also we should prevent new inode
allocations from the group, while zeroing is on the way. For that we
take write alloc_sem lock in ext4_init_inode_table() and read alloc_sem
in the ext4_claim_inode, so when we are unlucky and allocator hits the
group which is currently being zeroed, it just has to wait.
This can be suppresed using the mount option no_init_itable.
Signed-off-by: Lukas Czerner <lczerner@redhat.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2010-10-27 21:30:05 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-07-17 00:14:40 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: add support for lazy inode table initialization
When the lazy_itable_init extended option is passed to mke2fs, it
considerably speeds up filesystem creation because inode tables are
not zeroed out. The fact that parts of the inode table are
uninitialized is not a problem so long as the block group descriptors,
which contain information regarding how much of the inode table has
been initialized, has not been corrupted However, if the block group
checksums are not valid, e2fsck must scan the entire inode table, and
the the old, uninitialized data could potentially cause e2fsck to
report false problems.
Hence, it is important for the inode tables to be initialized as soon
as possble. This commit adds this feature so that mke2fs can safely
use the lazy inode table initialization feature to speed up formatting
file systems.
This is done via a new new kernel thread called ext4lazyinit, which is
created on demand and destroyed, when it is no longer needed. There
is only one thread for all ext4 filesystems in the system. When the
first filesystem with inititable mount option is mounted, ext4lazyinit
thread is created, then the filesystem can register its request in the
request list.
This thread then walks through the list of requests picking up
scheduled requests and invoking ext4_init_inode_table(). Next schedule
time for the request is computed by multiplying the time it took to
zero out last inode table with wait multiplier, which can be set with
the (init_itable=n) mount option (default is 10). We are doing
this so we do not take the whole I/O bandwidth. When the thread is no
longer necessary (request list is empty) it frees the appropriate
structures and exits (and can be created later later by another
filesystem).
We do not disturb regular inode allocations in any way, it just do not
care whether the inode table is, or is not zeroed. But when zeroing, we
have to skip used inodes, obviously. Also we should prevent new inode
allocations from the group, while zeroing is on the way. For that we
take write alloc_sem lock in ext4_init_inode_table() and read alloc_sem
in the ext4_claim_inode, so when we are unlucky and allocator hits the
group which is currently being zeroed, it just has to wait.
This can be suppresed using the mount option no_init_itable.
Signed-off-by: Lukas Czerner <lczerner@redhat.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2010-10-27 21:30:05 -04:00
|
|
|
|
|
|
|
|
|
2020-07-17 00:14:40 -04:00
|
|
|
|
|
|
|
|
|
ext4: add support for lazy inode table initialization
When the lazy_itable_init extended option is passed to mke2fs, it
considerably speeds up filesystem creation because inode tables are
not zeroed out. The fact that parts of the inode table are
uninitialized is not a problem so long as the block group descriptors,
which contain information regarding how much of the inode table has
been initialized, has not been corrupted However, if the block group
checksums are not valid, e2fsck must scan the entire inode table, and
the the old, uninitialized data could potentially cause e2fsck to
report false problems.
Hence, it is important for the inode tables to be initialized as soon
as possble. This commit adds this feature so that mke2fs can safely
use the lazy inode table initialization feature to speed up formatting
file systems.
This is done via a new new kernel thread called ext4lazyinit, which is
created on demand and destroyed, when it is no longer needed. There
is only one thread for all ext4 filesystems in the system. When the
first filesystem with inititable mount option is mounted, ext4lazyinit
thread is created, then the filesystem can register its request in the
request list.
This thread then walks through the list of requests picking up
scheduled requests and invoking ext4_init_inode_table(). Next schedule
time for the request is computed by multiplying the time it took to
zero out last inode table with wait multiplier, which can be set with
the (init_itable=n) mount option (default is 10). We are doing
this so we do not take the whole I/O bandwidth. When the thread is no
longer necessary (request list is empty) it frees the appropriate
structures and exits (and can be created later later by another
filesystem).
We do not disturb regular inode allocations in any way, it just do not
care whether the inode table is, or is not zeroed. But when zeroing, we
have to skip used inodes, obviously. Also we should prevent new inode
allocations from the group, while zeroing is on the way. For that we
take write alloc_sem lock in ext4_init_inode_table() and read alloc_sem
in the ext4_claim_inode, so when we are unlucky and allocator hits the
group which is currently being zeroed, it just has to wait.
This can be suppresed using the mount option no_init_itable.
Signed-off-by: Lukas Czerner <lczerner@redhat.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2010-10-27 21:30:05 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-27 21:30:05 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-05-24 18:31:25 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-04-29 18:23:10 -04:00
|
|
|
|
|
|
|
|
|
2011-05-24 18:31:25 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2007-05-24 13:04:54 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2008-11-05 00:14:04 -05:00
|
|
|
|
2012-06-30 19:14:57 -04:00
|
|
|
|
2023-02-22 04:30:25 +08:00
|
|
|
|
2012-04-29 18:33:10 -04:00
|
|
|
|
2024-08-20 21:22:32 +08:00
|
|
|
|
2023-02-22 04:30:24 +08:00
|
|
|
|
2012-04-29 18:33:10 -04:00
|
|
|
|
2024-08-20 21:22:32 +08:00
|
|
|
|
2023-02-22 04:30:27 +08:00
|
|
|
|
2012-04-29 18:35:10 -04:00
|
|
|
|
2012-10-22 00:34:32 -04:00
|
|
|
|
2023-02-22 04:30:26 +08:00
|
|
|
|
2012-04-29 18:35:10 -04:00
|
|
|
|
2012-10-22 00:34:32 -04:00
|
|
|
|
2008-11-05 00:14:04 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2013-04-03 23:32:34 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-07-11 19:27:31 -04:00
|
|
|
|
2011-05-25 07:41:26 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-09-09 19:14:51 -04:00
|
|
|
|
|
|
|
|
|
2011-09-09 19:10:51 -04:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
ext4: allow ext4_get_group_info() to fail
Previously, ext4_get_group_info() would treat an invalid group number
as BUG(), since in theory it should never happen. However, if a
malicious attaker (or fuzzer) modifies the superblock via the block
device while it is the file system is mounted, it is possible for
s_first_data_block to get set to a very large number. In that case,
when calculating the block group of some block number (such as the
starting block of a preallocation region), could result in an
underflow and very large block group number. Then the BUG_ON check in
ext4_get_group_info() would fire, resutling in a denial of service
attack that can be triggered by root or someone with write access to
the block device.
For a quality of implementation perspective, it's best that even if
the system administrator does something that they shouldn't, that it
will not trigger a BUG. So instead of BUG'ing, ext4_get_group_info()
will call ext4_error and return NULL. We also add fallback code in
all of the callers of ext4_get_group_info() that it might NULL.
Also, since ext4_get_group_info() was already borderline to be an
inline function, un-inline it. The results in a next reduction of the
compiled text size of ext4 by roughly 2k.
Cc: stable@kernel.org
Link: https://lore.kernel.org/r/20230430154311.579720-2-tytso@mit.edu
Reported-by: syzbot+e2efa3efc15a1c9e95c3@syzkaller.appspotmail.com
Link: https://syzkaller.appspot.com/bug?id=69b28112e098b070f639efb356393af3ffec4220
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
Reviewed-by: Jan Kara <jack@suse.cz>
2023-04-29 00:06:28 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2012-02-20 17:52:46 -05:00
|
|
|
|
|
|
|
|
|
2020-04-21 10:54:07 +03:00
|
|
|
|
|
|
|
|
|
2012-02-20 17:52:46 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-09-09 19:12:51 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-06-28 10:01:31 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2022-01-18 07:56:14 +01:00
|
|
|
|
2021-03-19 07:34:14 +00:00
|
|
|
|
2024-06-06 10:33:52 +03:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-06-19 23:45:09 -04:00
|
|
|
|
|
|
|
|
|
2022-05-15 12:07:46 +05:30
|
|
|
|
2018-12-12 15:20:12 +05:30
|
|
|
|
2022-05-15 12:07:46 +05:30
|
|
|
|
|
|
|
|
|
2022-05-15 12:07:47 +05:30
|
|
|
|
|
|
|
|
|
fscrypt: fix race where ->lookup() marks plaintext dentry as ciphertext
->lookup() in an encrypted directory begins as follows:
1. fscrypt_prepare_lookup():
a. Try to load the directory's encryption key.
b. If the key is unavailable, mark the dentry as a ciphertext name
via d_flags.
2. fscrypt_setup_filename():
a. Try to load the directory's encryption key.
b. If the key is available, encrypt the name (treated as a plaintext
name) to get the on-disk name. Otherwise decode the name
(treated as a ciphertext name) to get the on-disk name.
But if the key is concurrently added, it may be found at (2a) but not at
(1a). In this case, the dentry will be wrongly marked as a ciphertext
name even though it was actually treated as plaintext.
This will cause the dentry to be wrongly invalidated on the next lookup,
potentially causing problems. For example, if the racy ->lookup() was
part of sys_mount(), then the new mount will be detached when anything
tries to access it. This is despite the mountpoint having a plaintext
path, which should remain valid now that the key was added.
Of course, this is only possible if there's a userspace race. Still,
the additional kernel-side race is confusing and unexpected.
Close the kernel-side race by changing fscrypt_prepare_lookup() to also
set the on-disk filename (step 2b), consistent with the d_flags update.
Fixes: 28b4c263961c ("ext4 crypto: revalidate dentry after adding or removing the key")
Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2019-03-20 11:39:13 -07:00
|
|
|
|
2022-05-15 12:07:47 +05:30
|
|
|
|
|
|
|
|
|
fscrypt: fix race where ->lookup() marks plaintext dentry as ciphertext
->lookup() in an encrypted directory begins as follows:
1. fscrypt_prepare_lookup():
a. Try to load the directory's encryption key.
b. If the key is unavailable, mark the dentry as a ciphertext name
via d_flags.
2. fscrypt_setup_filename():
a. Try to load the directory's encryption key.
b. If the key is available, encrypt the name (treated as a plaintext
name) to get the on-disk name. Otherwise decode the name
(treated as a ciphertext name) to get the on-disk name.
But if the key is concurrently added, it may be found at (2a) but not at
(1a). In this case, the dentry will be wrongly marked as a ciphertext
name even though it was actually treated as plaintext.
This will cause the dentry to be wrongly invalidated on the next lookup,
potentially causing problems. For example, if the racy ->lookup() was
part of sys_mount(), then the new mount will be detached when anything
tries to access it. This is despite the mountpoint having a plaintext
path, which should remain valid now that the key was added.
Of course, this is only possible if there's a userspace race. Still,
the additional kernel-side race is confusing and unexpected.
Close the kernel-side race by changing fscrypt_prepare_lookup() to also
set the on-disk filename (step 2b), consistent with the d_flags update.
Fixes: 28b4c263961c ("ext4 crypto: revalidate dentry after adding or removing the key")
Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2019-03-20 11:39:13 -07:00
|
|
|
|
2022-05-15 12:07:47 +05:30
|
|
|
|
2016-07-10 14:01:03 -04:00
|
|
|
|
2022-05-15 12:07:48 +05:30
|
|
|
|
|
|
|
|
|
fscrypt: fix race where ->lookup() marks plaintext dentry as ciphertext
->lookup() in an encrypted directory begins as follows:
1. fscrypt_prepare_lookup():
a. Try to load the directory's encryption key.
b. If the key is unavailable, mark the dentry as a ciphertext name
via d_flags.
2. fscrypt_setup_filename():
a. Try to load the directory's encryption key.
b. If the key is available, encrypt the name (treated as a plaintext
name) to get the on-disk name. Otherwise decode the name
(treated as a ciphertext name) to get the on-disk name.
But if the key is concurrently added, it may be found at (2a) but not at
(1a). In this case, the dentry will be wrongly marked as a ciphertext
name even though it was actually treated as plaintext.
This will cause the dentry to be wrongly invalidated on the next lookup,
potentially causing problems. For example, if the racy ->lookup() was
part of sys_mount(), then the new mount will be detached when anything
tries to access it. This is despite the mountpoint having a plaintext
path, which should remain valid now that the key was added.
Of course, this is only possible if there's a userspace race. Still,
the additional kernel-side race is confusing and unexpected.
Close the kernel-side race by changing fscrypt_prepare_lookup() to also
set the on-disk filename (step 2b), consistent with the d_flags update.
Fixes: 28b4c263961c ("ext4 crypto: revalidate dentry after adding or removing the key")
Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2019-03-20 11:39:13 -07:00
|
|
|
|
2016-07-10 14:01:03 -04:00
|
|
|
|
fscrypt: fix race where ->lookup() marks plaintext dentry as ciphertext
->lookup() in an encrypted directory begins as follows:
1. fscrypt_prepare_lookup():
a. Try to load the directory's encryption key.
b. If the key is unavailable, mark the dentry as a ciphertext name
via d_flags.
2. fscrypt_setup_filename():
a. Try to load the directory's encryption key.
b. If the key is available, encrypt the name (treated as a plaintext
name) to get the on-disk name. Otherwise decode the name
(treated as a ciphertext name) to get the on-disk name.
But if the key is concurrently added, it may be found at (2a) but not at
(1a). In this case, the dentry will be wrongly marked as a ciphertext
name even though it was actually treated as plaintext.
This will cause the dentry to be wrongly invalidated on the next lookup,
potentially causing problems. For example, if the racy ->lookup() was
part of sys_mount(), then the new mount will be detached when anything
tries to access it. This is despite the mountpoint having a plaintext
path, which should remain valid now that the key was added.
Of course, this is only possible if there's a userspace race. Still,
the additional kernel-side race is confusing and unexpected.
Close the kernel-side race by changing fscrypt_prepare_lookup() to also
set the on-disk filename (step 2b), consistent with the d_flags update.
Fixes: 28b4c263961c ("ext4 crypto: revalidate dentry after adding or removing the key")
Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2019-03-20 11:39:13 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-05-18 13:14:47 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-06-19 23:45:09 -04:00
|
|
|
|
2024-06-06 10:33:52 +03:00
|
|
|
|
2015-05-18 13:14:47 -04:00
|
|
|
|
2016-07-10 14:01:03 -04:00
|
|
|
|
fscrypt: fix race where ->lookup() marks plaintext dentry as ciphertext
->lookup() in an encrypted directory begins as follows:
1. fscrypt_prepare_lookup():
a. Try to load the directory's encryption key.
b. If the key is unavailable, mark the dentry as a ciphertext name
via d_flags.
2. fscrypt_setup_filename():
a. Try to load the directory's encryption key.
b. If the key is available, encrypt the name (treated as a plaintext
name) to get the on-disk name. Otherwise decode the name
(treated as a ciphertext name) to get the on-disk name.
But if the key is concurrently added, it may be found at (2a) but not at
(1a). In this case, the dentry will be wrongly marked as a ciphertext
name even though it was actually treated as plaintext.
This will cause the dentry to be wrongly invalidated on the next lookup,
potentially causing problems. For example, if the racy ->lookup() was
part of sys_mount(), then the new mount will be detached when anything
tries to access it. This is despite the mountpoint having a plaintext
path, which should remain valid now that the key was added.
Of course, this is only possible if there's a userspace race. Still,
the additional kernel-side race is confusing and unexpected.
Close the kernel-side race by changing fscrypt_prepare_lookup() to also
set the on-disk filename (step 2b), consistent with the d_flags update.
Fixes: 28b4c263961c ("ext4 crypto: revalidate dentry after adding or removing the key")
Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2019-03-20 11:39:13 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-06-19 23:45:09 -04:00
|
|
|
|
|
|
|
|
|
2024-06-06 10:33:52 +03:00
|
|
|
|
2019-06-19 23:45:09 -04:00
|
|
|
|
2022-05-15 12:07:48 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fscrypt: fix race where ->lookup() marks plaintext dentry as ciphertext
->lookup() in an encrypted directory begins as follows:
1. fscrypt_prepare_lookup():
a. Try to load the directory's encryption key.
b. If the key is unavailable, mark the dentry as a ciphertext name
via d_flags.
2. fscrypt_setup_filename():
a. Try to load the directory's encryption key.
b. If the key is available, encrypt the name (treated as a plaintext
name) to get the on-disk name. Otherwise decode the name
(treated as a ciphertext name) to get the on-disk name.
But if the key is concurrently added, it may be found at (2a) but not at
(1a). In this case, the dentry will be wrongly marked as a ciphertext
name even though it was actually treated as plaintext.
This will cause the dentry to be wrongly invalidated on the next lookup,
potentially causing problems. For example, if the racy ->lookup() was
part of sys_mount(), then the new mount will be detached when anything
tries to access it. This is despite the mountpoint having a plaintext
path, which should remain valid now that the key was added.
Of course, this is only possible if there's a userspace race. Still,
the additional kernel-side race is confusing and unexpected.
Close the kernel-side race by changing fscrypt_prepare_lookup() to also
set the on-disk filename (step 2b), consistent with the d_flags update.
Fixes: 28b4c263961c ("ext4 crypto: revalidate dentry after adding or removing the key")
Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2019-03-20 11:39:13 -07:00
|
|
|
|
2015-04-12 00:56:17 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2010-07-27 11:54:40 -04:00
|
|
|
|
2011-01-10 12:10:55 -05:00
|
|
|
|
2010-07-27 11:54:40 -04:00
|
|
|
|
2012-12-10 14:05:58 -05:00
|
|
|
|
|
|
|
|
|
2021-03-19 07:34:13 +00:00
|
|
|
|
2011-01-10 12:10:55 -05:00
|
|
|
|
2021-03-19 07:34:13 +00:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2015-04-12 00:56:26 -04:00
|
|
|
|
|
|
|
|
|
2016-07-10 14:01:03 -04:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2012-12-10 14:05:58 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-05-18 13:14:47 -04:00
|
|
|
|
2012-12-10 14:05:58 -05:00
|
|
|
|
2021-03-19 07:34:13 +00:00
|
|
|
|
2017-04-29 23:27:26 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-12-10 14:05:58 -05:00
|
|
|
|
|
|
|
|
|
2020-11-18 16:30:32 +01:00
|
|
|
|
|
|
|
|
|
2020-02-10 15:43:16 +01:00
|
|
|
|
|
|
|
|
|
2012-12-10 14:05:58 -05:00
|
|
|
|
2020-02-10 15:43:16 +01:00
|
|
|
|
2012-12-10 14:05:58 -05:00
|
|
|
|
2017-04-29 23:47:50 -04:00
|
|
|
|
2012-12-10 14:05:59 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-10-17 16:18:43 -04:00
|
|
|
|
2012-12-10 14:05:59 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-07-28 13:06:26 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2011-07-16 20:44:56 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
ext4: Support case-insensitive file name lookups
This patch implements the actual support for case-insensitive file name
lookups in ext4, based on the feature bit and the encoding stored in the
superblock.
A filesystem that has the casefold feature set is able to configure
directories with the +F (EXT4_CASEFOLD_FL) attribute, enabling lookups
to succeed in that directory in a case-insensitive fashion, i.e: match
a directory entry even if the name used by userspace is not a byte per
byte match with the disk name, but is an equivalent case-insensitive
version of the Unicode string. This operation is called a
case-insensitive file name lookup.
The feature is configured as an inode attribute applied to directories
and inherited by its children. This attribute can only be enabled on
empty directories for filesystems that support the encoding feature,
thus preventing collision of file names that only differ by case.
* dcache handling:
For a +F directory, Ext4 only stores the first equivalent name dentry
used in the dcache. This is done to prevent unintentional duplication of
dentries in the dcache, while also allowing the VFS code to quickly find
the right entry in the cache despite which equivalent string was used in
a previous lookup, without having to resort to ->lookup().
d_hash() of casefolded directories is implemented as the hash of the
casefolded string, such that we always have a well-known bucket for all
the equivalencies of the same string. d_compare() uses the
utf8_strncasecmp() infrastructure, which handles the comparison of
equivalent, same case, names as well.
For now, negative lookups are not inserted in the dcache, since they
would need to be invalidated anyway, because we can't trust missing file
dentries. This is bad for performance but requires some leveraging of
the vfs layer to fix. We can live without that for now, and so does
everyone else.
* on-disk data:
Despite using a specific version of the name as the internal
representation within the dcache, the name stored and fetched from the
disk is a byte-per-byte match with what the user requested, making this
implementation 'name-preserving'. i.e. no actual information is lost
when writing to storage.
DX is supported by modifying the hashes used in +F directories to make
them case/encoding-aware. The new disk hashes are calculated as the
hash of the full casefolded string, instead of the string directly.
This allows us to efficiently search for file names in the htree without
requiring the user to provide an exact name.
* Dealing with invalid sequences:
By default, when a invalid UTF-8 sequence is identified, ext4 will treat
it as an opaque byte sequence, ignoring the encoding and reverting to
the old behavior for that unique file. This means that case-insensitive
file name lookup will not work only for that file. An optional bit can
be set in the superblock telling the filesystem code and userspace tools
to enforce the encoding. When that optional bit is set, any attempt to
create a file name using an invalid UTF-8 sequence will fail and return
an error to userspace.
* Normalization algorithm:
The UTF-8 algorithms used to compare strings in ext4 is implemented
lives in fs/unicode, and is based on a previous version developed by
SGI. It implements the Canonical decomposition (NFD) algorithm
described by the Unicode specification 12.1, or higher, combined with
the elimination of ignorable code points (NFDi) and full
case-folding (CF) as documented in fs/unicode/utf8_norm.c.
NFD seems to be the best normalization method for EXT4 because:
- It has a lower cost than NFC/NFKC (which requires
decomposing to NFD as an intermediary step)
- It doesn't eliminate important semantic meaning like
compatibility decompositions.
Although:
- This implementation is not completely linguistic accurate, because
different languages have conflicting rules, which would require the
specialization of the filesystem to a given locale, which brings all
sorts of problems for removable media and for users who use more than
one language.
Signed-off-by: Gabriel Krisman Bertazi <krisman@collabora.co.uk>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2019-04-25 14:12:08 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:59 -07:00
|
|
|
|
2023-01-13 12:49:25 +01:00
|
|
|
|
2021-01-21 14:19:57 +01:00
|
|
|
|
2013-02-09 16:27:09 -05:00
|
|
|
|
2017-06-21 21:21:39 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-02-09 16:27:09 -05:00
|
|
|
|
2021-01-21 14:19:57 +01:00
|
|
|
|
2023-01-13 12:49:25 +01:00
|
|
|
|
2021-01-21 14:19:57 +01:00
|
|
|
|
2023-01-13 12:49:25 +01:00
|
|
|
|
2013-02-09 16:27:09 -05:00
|
|
|
|
2023-01-13 12:49:25 +01:00
|
|
|
|
2017-06-21 21:21:39 -04:00
|
|
|
|
2013-02-09 16:27:09 -05:00
|
|
|
|
|
|
|
|
|
2008-09-08 22:25:24 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-27 21:30:15 -04:00
|
|
|
|
ext4: add support for lazy inode table initialization
When the lazy_itable_init extended option is passed to mke2fs, it
considerably speeds up filesystem creation because inode tables are
not zeroed out. The fact that parts of the inode table are
uninitialized is not a problem so long as the block group descriptors,
which contain information regarding how much of the inode table has
been initialized, has not been corrupted However, if the block group
checksums are not valid, e2fsck must scan the entire inode table, and
the the old, uninitialized data could potentially cause e2fsck to
report false problems.
Hence, it is important for the inode tables to be initialized as soon
as possble. This commit adds this feature so that mke2fs can safely
use the lazy inode table initialization feature to speed up formatting
file systems.
This is done via a new new kernel thread called ext4lazyinit, which is
created on demand and destroyed, when it is no longer needed. There
is only one thread for all ext4 filesystems in the system. When the
first filesystem with inititable mount option is mounted, ext4lazyinit
thread is created, then the filesystem can register its request in the
request list.
This thread then walks through the list of requests picking up
scheduled requests and invoking ext4_init_inode_table(). Next schedule
time for the request is computed by multiplying the time it took to
zero out last inode table with wait multiplier, which can be set with
the (init_itable=n) mount option (default is 10). We are doing
this so we do not take the whole I/O bandwidth. When the thread is no
longer necessary (request list is empty) it frees the appropriate
structures and exits (and can be created later later by another
filesystem).
We do not disturb regular inode allocations in any way, it just do not
care whether the inode table is, or is not zeroed. But when zeroing, we
have to skip used inodes, obviously. Also we should prevent new inode
allocations from the group, while zeroing is on the way. For that we
take write alloc_sem lock in ext4_init_inode_table() and read alloc_sem
in the ext4_claim_inode, so when we are unlucky and allocator hits the
group which is currently being zeroed, it just has to wait.
This can be suppresed using the mount option no_init_itable.
Signed-off-by: Lukas Czerner <lczerner@redhat.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2010-10-27 21:30:05 -04:00
|
|
|
|
|
|
|
|
|
2012-02-20 17:52:46 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2020-10-15 13:37:55 -07:00
|
|
|
|
2020-10-15 13:38:01 -07:00
|
|
|
|
2020-10-15 13:37:55 -07:00
|
|
|
|
2020-10-15 13:37:57 -07:00
|
|
|
|
2020-11-05 19:58:53 -08:00
|
|
|
|
2020-10-15 13:37:57 -07:00
|
|
|
|
2020-11-05 19:58:53 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: fix rename whiteout with fast commit
This patch adds rename whiteout support in fast commits. Note that the
whiteout object that gets created is actually char device. Which
imples, the function ext4_inode_journal_mode(struct inode *inode)
would return "JOURNAL_DATA" for this inode. This has a consequence in
fast commit code that it will make creation of the whiteout object a
fast-commit ineligible behavior and thus will fall back to full
commits. With this patch, this can be observed by running fast commits
with rename whiteout and seeing the stats generated by ext4_fc_stats
tracepoint as follows:
ext4_fc_stats: dev 254:32 fc ineligible reasons:
XATTR:0, CROSS_RENAME:0, JOURNAL_FLAG_CHANGE:0, NO_MEM:0, SWAP_BOOT:0,
RESIZE:0, RENAME_DIR:0, FALLOC_RANGE:0, INODE_JOURNAL_DATA:16;
num_commits:6, ineligible: 6, numblks: 3
So in short, this patch guarantees that in case of rename whiteout, we
fall back to full commits.
Amir mentioned that instead of creating a new whiteout object for
every rename, we can create a static whiteout object with irrelevant
nlink. That will make fast commits to not fall back to full
commit. But until this happens, this patch will ensure correctness by
falling back to full commits.
Fixes: 8016e29f4362 ("ext4: fast commit recovery path")
Cc: stable@kernel.org
Signed-off-by: Harshad Shirwadkar <harshadshirwadkar@gmail.com>
Link: https://lore.kernel.org/r/20210316221921.1124955-1-harshadshirwadkar@gmail.com
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2021-03-16 15:19:21 -07:00
|
|
|
|
|
|
|
|
|
2020-11-05 19:58:53 -08:00
|
|
|
|
|
|
|
|
|
2022-01-17 17:36:54 +08:00
|
|
|
|
2020-10-15 13:37:57 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:59 -07:00
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:57 -07:00
|
|
|
|
|
|
|
|
|
2021-12-23 17:44:36 +01:00
|
|
|
|
2022-01-10 11:51:40 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:57 -07:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2018-04-11 11:37:23 +02:00
|
|
|
|
2021-04-01 10:21:28 -07:00
|
|
|
|
2021-04-01 10:21:25 -07:00
|
|
|
|
2012-05-28 14:19:25 -04:00
|
|
|
|
2024-01-05 17:20:58 +08:00
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
|
|
|
|
|
2024-01-05 17:21:01 +08:00
|
|
|
|
2010-10-27 21:30:14 -04:00
|
|
|
|
|
|
|
|
|
2020-07-17 00:14:40 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-11-22 07:44:56 -05:00
|
|
|
|
2009-11-23 07:17:05 -05:00
|
|
|
|
|
|
|
|
|
2012-09-05 01:31:50 -04:00
|
|
|
|
|
|
|
|
|
2009-01-05 21:36:19 -05:00
|
|
|
|
2008-07-11 19:27:31 -04:00
|
|
|
|
2011-07-26 21:46:07 -04:00
|
|
|
|
2011-05-09 10:46:41 -04:00
|
|
|
|
2010-10-27 21:30:12 -04:00
|
|
|
|
2017-06-22 23:54:33 -04:00
|
|
|
|
2020-10-15 13:37:59 -07:00
|
|
|
|
2023-09-29 00:03:56 +08:00
|
|
|
|
2023-06-30 14:29:27 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-27 21:30:12 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2020-10-15 13:37:59 -07:00
|
|
|
|
|
|
|
|
|
2015-04-16 01:55:00 -04:00
|
|
|
|
2014-08-29 20:51:32 -04:00
|
|
|
|
2014-08-29 20:52:15 -04:00
|
|
|
|
2017-08-06 00:07:01 -04:00
|
|
|
|
|
|
|
|
|
2016-03-08 23:08:10 -05:00
|
|
|
|
|
|
|
|
|
2008-10-07 00:46:36 -04:00
|
|
|
|
2016-03-08 23:08:10 -05:00
|
|
|
|
2012-12-10 14:05:57 -05:00
|
|
|
|
|
|
|
|
|
2012-12-10 14:05:51 -05:00
|
|
|
|
2021-08-16 11:57:04 +02:00
|
|
|
|
2012-12-10 14:05:51 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:04 +02:00
|
|
|
|
2012-12-10 14:05:51 -05:00
|
|
|
|
2021-08-16 11:57:04 +02:00
|
|
|
|
2012-12-10 14:05:51 -05:00
|
|
|
|
2012-12-10 14:05:57 -05:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
ext4: avoid declaring fs inconsistent due to invalid file handles
If we receive a file handle, either from NFS or open_by_handle_at(2),
and it points at an inode which has not been initialized, and the file
system has metadata checksums enabled, we shouldn't try to get the
inode, discover the checksum is invalid, and then declare the file
system as being inconsistent.
This can be reproduced by creating a test file system via "mke2fs -t
ext4 -O metadata_csum /tmp/foo.img 8M", mounting it, cd'ing into that
directory, and then running the following program.
#define _GNU_SOURCE
#include <fcntl.h>
struct handle {
struct file_handle fh;
unsigned char fid[MAX_HANDLE_SZ];
};
int main(int argc, char **argv)
{
struct handle h = {{8, 1 }, { 12, }};
open_by_handle_at(AT_FDCWD, &h.fh, O_RDONLY);
return 0;
}
Google-Bug-Id: 120690101
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
Cc: stable@kernel.org
2018-12-19 12:29:13 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2022-10-26 12:23:09 +08:00
|
|
|
|
2023-05-23 23:49:48 -04:00
|
|
|
|
|
|
|
|
|
ext4: avoid declaring fs inconsistent due to invalid file handles
If we receive a file handle, either from NFS or open_by_handle_at(2),
and it points at an inode which has not been initialized, and the file
system has metadata checksums enabled, we shouldn't try to get the
inode, discover the checksum is invalid, and then declare the file
system as being inconsistent.
This can be reproduced by creating a test file system via "mke2fs -t
ext4 -O metadata_csum /tmp/foo.img 8M", mounting it, cd'ing into that
directory, and then running the following program.
#define _GNU_SOURCE
#include <fcntl.h>
struct handle {
struct file_handle fh;
unsigned char fid[MAX_HANDLE_SZ];
};
int main(int argc, char **argv)
{
struct handle h = {{8, 1 }, { 12, }};
open_by_handle_at(AT_FDCWD, &h.fh, O_RDONLY);
return 0;
}
Google-Bug-Id: 120690101
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
Cc: stable@kernel.org
2018-12-19 12:29:13 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-03-05 09:21:37 +01:00
|
|
|
|
2023-01-13 12:49:11 +01:00
|
|
|
|
2021-01-21 14:19:43 +01:00
|
|
|
|
2022-08-26 23:58:47 -07:00
|
|
|
|
2023-01-13 12:49:12 +01:00
|
|
|
|
2021-01-21 14:19:43 +01:00
|
|
|
|
2010-06-07 13:16:22 -04:00
|
|
|
|
|
|
|
|
|
2023-01-13 12:49:12 +01:00
|
|
|
|
2021-01-21 14:19:43 +01:00
|
|
|
|
2011-05-27 06:53:02 -04:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:59 -07:00
|
|
|
|
|
|
|
|
|
2013-08-16 21:19:41 -04:00
|
|
|
|
2008-07-11 19:27:31 -04:00
|
|
|
|
2016-11-13 22:02:26 -05:00
|
|
|
|
2018-07-29 17:00:22 -04:00
|
|
|
|
2022-03-08 10:50:43 -08:00
|
|
|
|
2020-05-28 07:59:59 -07:00
|
|
|
|
2009-02-26 01:04:07 -05:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:21:03 -07:00
|
|
|
|
2022-12-07 12:27:11 +01:00
|
|
|
|
2008-08-19 22:16:03 -04:00
|
|
|
|
2013-05-27 23:32:35 -04:00
|
|
|
|
|
|
|
|
|
2018-10-02 22:20:50 -04:00
|
|
|
|
2009-12-14 15:21:14 +03:00
|
|
|
|
2016-01-08 16:01:21 -05:00
|
|
|
|
2018-10-01 14:33:24 -04:00
|
|
|
|
2010-01-25 04:00:31 -05:00
|
|
|
|
|
|
|
|
|
2015-12-07 15:09:35 -05:00
|
|
|
|
|
|
|
|
|
2011-06-27 19:40:50 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-06-04 12:56:55 -04:00
|
|
|
|
2013-04-03 12:47:17 -04:00
|
|
|
|
2014-07-15 06:03:38 -04:00
|
|
|
|
|
|
|
|
|
2011-06-27 16:36:31 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2008-04-29 22:03:54 -04:00
|
|
|
|
2008-09-08 22:25:24 -04:00
|
|
|
|
2023-01-13 12:49:21 +01:00
|
|
|
|
2021-04-07 14:36:43 +02:00
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:59 -07:00
|
|
|
|
2022-06-29 00:00:26 -04:00
|
|
|
|
2023-06-01 11:44:57 +02:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2008-01-28 23:58:26 -05:00
|
|
|
|
2008-09-13 12:52:26 -04:00
|
|
|
|
2013-04-10 23:32:52 -04:00
|
|
|
|
2009-02-14 23:01:36 -05:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2020-10-15 13:37:59 -07:00
|
|
|
|
|
|
|
|
|
2019-06-21 15:49:26 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2015-05-18 13:14:47 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-08-10 17:07:05 +09:00
|
|
|
|
2012-12-10 14:06:00 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-07-10 14:01:03 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2020-02-15 16:40:37 -05:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-01-04 17:09:44 -05:00
|
|
|
|
2021-12-13 14:56:18 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2018-11-25 17:20:31 -05:00
|
|
|
|
2022-07-14 11:07:17 -07:00
|
|
|
|
2020-09-24 15:33:37 +08:00
|
|
|
|
|
|
|
|
|
2022-07-14 11:07:17 -07:00
|
|
|
|
2024-09-06 17:17:46 +08:00
|
|
|
|
2022-07-14 11:07:17 -07:00
|
|
|
|
2024-09-06 17:17:46 +08:00
|
|
|
|
2022-07-14 11:07:17 -07:00
|
|
|
|
2020-09-24 15:33:35 +08:00
|
|
|
|
2015-09-23 12:46:17 -04:00
|
|
|
|
2012-07-09 16:27:05 -04:00
|
|
|
|
2021-12-13 14:56:18 +01:00
|
|
|
|
|
|
|
|
|
2012-10-10 01:06:58 -04:00
|
|
|
|
2012-09-05 01:29:50 -04:00
|
|
|
|
|
|
|
|
|
2013-02-08 13:00:31 -05:00
|
|
|
|
|
|
|
|
|
2018-05-12 11:39:40 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-08-03 00:28:39 +08:00
|
|
|
|
|
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
2020-11-27 12:33:57 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-03-28 19:33:43 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-10-31 17:11:33 -07:00
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
2011-10-31 17:11:33 -07:00
|
|
|
|
2010-07-27 11:56:40 -04:00
|
|
|
|
|
|
|
|
|
2011-10-31 17:11:33 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-06-15 14:50:26 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-10-31 17:11:33 -07:00
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
2011-05-24 18:31:25 -04:00
|
|
|
|
|
|
|
|
|
2011-10-31 17:11:33 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
2015-06-15 14:50:26 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-03-28 19:33:43 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-06-15 14:50:26 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-11-27 12:33:57 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-03-28 19:33:43 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-11-27 12:33:57 +01:00
|
|
|
|
|
|
|
|
|
2020-03-28 19:33:43 -04:00
|
|
|
|
2020-11-27 12:33:57 +01:00
|
|
|
|
|
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
|
|
|
|
|
2015-06-15 14:50:26 -04:00
|
|
|
|
|
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-03-28 19:33:43 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-11-27 12:33:57 +01:00
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
2020-03-28 19:33:43 -04:00
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
|
|
|
|
|
2020-11-27 12:33:57 +01:00
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-06-15 14:50:26 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-07-01 08:12:37 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:21:15 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-09-09 19:08:51 -04:00
|
|
|
|
|
|
|
|
|
2009-01-05 22:20:24 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:21:15 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-09-09 19:08:51 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-01-05 22:20:24 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-04-29 18:45:10 -04:00
|
|
|
|
2009-05-01 19:44:44 -04:00
|
|
|
|
2012-04-29 18:45:10 -04:00
|
|
|
|
|
|
|
|
|
2013-01-13 08:41:45 -05:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
2014-10-13 03:36:16 -04:00
|
|
|
|
|
|
|
|
|
2024-12-01 17:08:41 -08:00
|
|
|
|
2014-10-13 03:36:16 -04:00
|
|
|
|
2017-06-22 11:44:55 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-08-25 17:00:16 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
|
|
|
|
|
2020-08-25 17:00:16 +02:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-08-25 17:00:16 +02:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-08-25 17:00:16 +02:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-10-16 18:38:25 -04:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2017-06-21 21:09:57 -04:00
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2017-06-21 21:09:57 -04:00
|
|
|
|
|
|
|
|
|
2009-01-17 18:41:37 -05:00
|
|
|
|
|
|
|
|
|
2017-06-21 21:09:57 -04:00
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2006-10-11 01:21:10 -07:00
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2009-05-01 08:50:38 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-01-29 00:19:52 -05:00
|
|
|
|
2008-07-11 19:27:31 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2010-07-27 11:56:40 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2008-10-09 10:56:23 -04:00
|
|
|
|
2011-09-09 19:16:51 -04:00
|
|
|
|
|
|
|
|
|
2009-01-06 14:41:04 -08:00
|
|
|
|
2008-10-09 10:56:23 -04:00
|
|
|
|
2011-09-09 19:16:51 -04:00
|
|
|
|
2008-10-09 10:56:23 -04:00
|
|
|
|
2011-09-09 19:16:51 -04:00
|
|
|
|
2008-10-09 10:56:23 -04:00
|
|
|
|
|
|
|
|
|
2022-01-21 15:06:11 +08:00
|
|
|
|
2008-09-13 13:06:18 -04:00
|
|
|
|
|
|
|
|
|
2013-08-17 10:09:31 -04:00
|
|
|
|
2016-01-22 15:40:57 -05:00
|
|
|
|
2008-09-13 13:06:18 -04:00
|
|
|
|
|
|
|
|
|
2020-02-07 09:29:11 -05:00
|
|
|
|
2008-09-13 13:06:18 -04:00
|
|
|
|
2013-08-17 10:09:31 -04:00
|
|
|
|
|
|
|
|
|
2022-01-21 15:06:11 +08:00
|
|
|
|
2014-08-23 17:48:28 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-12-07 14:34:49 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-01-05 22:19:52 -05:00
|
|
|
|
|
|
|
|
|
2020-09-28 19:36:35 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-01-05 22:19:52 -05:00
|
|
|
|
2009-08-25 22:36:45 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2022-09-08 11:21:28 +02:00
|
|
|
|
|
|
|
|
|
2010-05-16 15:00:00 -04:00
|
|
|
|
ext4: improve cr 0 / cr 1 group scanning
Instead of traversing through groups linearly, scan groups in specific
orders at cr 0 and cr 1. At cr 0, we want to find groups that have the
largest free order >= the order of the request. So, with this patch,
we maintain lists for each possible order and insert each group into a
list based on the largest free order in its buddy bitmap. During cr 0
allocation, we traverse these lists in the increasing order of largest
free orders. This allows us to find a group with the best available cr
0 match in constant time. If nothing can be found, we fallback to cr 1
immediately.
At CR1, the story is slightly different. We want to traverse in the
order of increasing average fragment size. For CR1, we maintain a rb
tree of groupinfos which is sorted by average fragment size. Instead
of traversing linearly, at CR1, we traverse in the order of increasing
average fragment size, starting at the most optimal group. This brings
down cr 1 search complexity to log(num groups).
For cr >= 2, we just perform the linear search as before. Also, in
case of lock contention, we intermittently fallback to linear search
even in CR 0 and CR 1 cases. This allows us to proceed during the
allocation path even in case of high contention.
There is an opportunity to do optimization at CR2 too. That's because
at CR2 we only consider groups where bb_free counter (number of free
blocks) is greater than the request extent size. That's left as future
work.
All the changes introduced in this patch are protected under a new
mount option "mb_optimize_scan".
With this patchset, following experiment was performed:
Created a highly fragmented disk of size 65TB. The disk had no
contiguous 2M regions. Following command was run consecutively for 3
times:
time dd if=/dev/urandom of=file bs=2M count=10
Here are the results with and without cr 0/1 optimizations introduced
in this patch:
|---------+------------------------------+---------------------------|
| | Without CR 0/1 Optimizations | With CR 0/1 Optimizations |
|---------+------------------------------+---------------------------|
| 1st run | 5m1.871s | 2m47.642s |
| 2nd run | 2m28.390s | 0m0.611s |
| 3rd run | 2m26.530s | 0m1.255s |
|---------+------------------------------+---------------------------|
Signed-off-by: Harshad Shirwadkar <harshadshirwadkar@gmail.com>
Reported-by: kernel test robot <lkp@intel.com>
Reported-by: Dan Carpenter <dan.carpenter@oracle.com>
Reviewed-by: Andreas Dilger <adilger@dilger.ca>
Link: https://lore.kernel.org/r/20210401172129.189766-6-harshadshirwadkar@gmail.com
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2021-04-01 10:21:27 -07:00
|
|
|
|
2009-01-05 22:19:52 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2022-09-08 11:21:28 +02:00
|
|
|
|
ext4: improve cr 0 / cr 1 group scanning
Instead of traversing through groups linearly, scan groups in specific
orders at cr 0 and cr 1. At cr 0, we want to find groups that have the
largest free order >= the order of the request. So, with this patch,
we maintain lists for each possible order and insert each group into a
list based on the largest free order in its buddy bitmap. During cr 0
allocation, we traverse these lists in the increasing order of largest
free orders. This allows us to find a group with the best available cr
0 match in constant time. If nothing can be found, we fallback to cr 1
immediately.
At CR1, the story is slightly different. We want to traverse in the
order of increasing average fragment size. For CR1, we maintain a rb
tree of groupinfos which is sorted by average fragment size. Instead
of traversing linearly, at CR1, we traverse in the order of increasing
average fragment size, starting at the most optimal group. This brings
down cr 1 search complexity to log(num groups).
For cr >= 2, we just perform the linear search as before. Also, in
case of lock contention, we intermittently fallback to linear search
even in CR 0 and CR 1 cases. This allows us to proceed during the
allocation path even in case of high contention.
There is an opportunity to do optimization at CR2 too. That's because
at CR2 we only consider groups where bb_free counter (number of free
blocks) is greater than the request extent size. That's left as future
work.
All the changes introduced in this patch are protected under a new
mount option "mb_optimize_scan".
With this patchset, following experiment was performed:
Created a highly fragmented disk of size 65TB. The disk had no
contiguous 2M regions. Following command was run consecutively for 3
times:
time dd if=/dev/urandom of=file bs=2M count=10
Here are the results with and without cr 0/1 optimizations introduced
in this patch:
|---------+------------------------------+---------------------------|
| | Without CR 0/1 Optimizations | With CR 0/1 Optimizations |
|---------+------------------------------+---------------------------|
| 1st run | 5m1.871s | 2m47.642s |
| 2nd run | 2m28.390s | 0m0.611s |
| 3rd run | 2m26.530s | 0m1.255s |
|---------+------------------------------+---------------------------|
Signed-off-by: Harshad Shirwadkar <harshadshirwadkar@gmail.com>
Reported-by: kernel test robot <lkp@intel.com>
Reported-by: Dan Carpenter <dan.carpenter@oracle.com>
Reviewed-by: Andreas Dilger <adilger@dilger.ca>
Link: https://lore.kernel.org/r/20210401172129.189766-6-harshadshirwadkar@gmail.com
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2021-04-01 10:21:27 -07:00
|
|
|
|
2009-08-25 22:36:45 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-01-05 22:19:52 -05:00
|
|
|
|
|
|
|
|
|
2011-07-11 00:03:38 -04:00
|
|
|
|
|
|
|
|
|
ext4: mark block group as corrupt on block bitmap error
When we notice a block-bitmap corruption (because of device failure or
something else), we should mark this group as corrupt and prevent
further block allocations/deallocations from it. Currently, we end up
generating one error message for every block in the bitmap. This
potentially could make the system unstable as noticed in some
bugs. With this patch, the error will be printed only the first time
and mark the entire block group as corrupted. This prevents future
access allocations/deallocations from it.
Also tested by corrupting the block
bitmap and forcefully introducing the mb_free_blocks error:
(1) create a largefile (2Gb)
$ dd if=/dev/zero of=largefile oflag=direct bs=10485760 count=200
(2) umount filesystem. use dumpe2fs to see which block-bitmaps
are in use by largefile and note their block numbers
(3) use dd to zero-out the used block bitmaps
$ dd if=/dev/zero of=/dev/hdc4 bs=4096 seek=14 count=8 oflag=direct
(4) mount the FS and delete the largefile.
(5) recreate the largefile. verify that the new largefile does not
get any blocks from the groups marked as bad.
Without the patch, we will see mb_free_blocks error for each bit in
each zero'ed out bitmap at (4). With the patch, we only see the error
once per blockgroup:
[ 309.706803] EXT4-fs error (device sdb4): ext4_mb_generate_buddy:735: group 15: 32768 clusters in bitmap, 0 in gd. blk grp corrupted.
[ 309.720824] EXT4-fs error (device sdb4): ext4_mb_generate_buddy:735: group 14: 32768 clusters in bitmap, 0 in gd. blk grp corrupted.
[ 309.732858] EXT4-fs error (device sdb4) in ext4_free_blocks:4802: IO failure
[ 309.748321] EXT4-fs error (device sdb4): ext4_mb_generate_buddy:735: group 13: 32768 clusters in bitmap, 0 in gd. blk grp corrupted.
[ 309.760331] EXT4-fs error (device sdb4) in ext4_free_blocks:4802: IO failure
[ 309.769695] EXT4-fs error (device sdb4): ext4_mb_generate_buddy:735: group 12: 32768 clusters in bitmap, 0 in gd. blk grp corrupted.
[ 309.781721] EXT4-fs error (device sdb4) in ext4_free_blocks:4802: IO failure
[ 309.798166] EXT4-fs error (device sdb4): ext4_mb_generate_buddy:735: group 11: 32768 clusters in bitmap, 0 in gd. blk grp corrupted.
[ 309.810184] EXT4-fs error (device sdb4) in ext4_free_blocks:4802: IO failure
[ 309.819532] EXT4-fs error (device sdb4): ext4_mb_generate_buddy:735: group 10: 32768 clusters in bitmap, 0 in gd. blk grp corrupted.
Google-Bug-Id: 7258357
[darrick.wong@oracle.com]
Further modifications (by Darrick) to make more obvious that this corruption
bit applies to blocks only. Set the corruption flag if the block group bitmap
verification fails.
Original-author: Aditya Kali <adityakali@google.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2013-08-28 17:35:51 -04:00
|
|
|
|
2013-08-28 18:32:58 -04:00
|
|
|
|
2018-05-12 11:39:40 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-04-21 10:54:07 +03:00
|
|
|
|
2009-01-05 22:19:52 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: mark block group as corrupt on block bitmap error
When we notice a block-bitmap corruption (because of device failure or
something else), we should mark this group as corrupt and prevent
further block allocations/deallocations from it. Currently, we end up
generating one error message for every block in the bitmap. This
potentially could make the system unstable as noticed in some
bugs. With this patch, the error will be printed only the first time
and mark the entire block group as corrupted. This prevents future
access allocations/deallocations from it.
Also tested by corrupting the block
bitmap and forcefully introducing the mb_free_blocks error:
(1) create a largefile (2Gb)
$ dd if=/dev/zero of=largefile oflag=direct bs=10485760 count=200
(2) umount filesystem. use dumpe2fs to see which block-bitmaps
are in use by largefile and note their block numbers
(3) use dd to zero-out the used block bitmaps
$ dd if=/dev/zero of=/dev/hdc4 bs=4096 seek=14 count=8 oflag=direct
(4) mount the FS and delete the largefile.
(5) recreate the largefile. verify that the new largefile does not
get any blocks from the groups marked as bad.
Without the patch, we will see mb_free_blocks error for each bit in
each zero'ed out bitmap at (4). With the patch, we only see the error
once per blockgroup:
[ 309.706803] EXT4-fs error (device sdb4): ext4_mb_generate_buddy:735: group 15: 32768 clusters in bitmap, 0 in gd. blk grp corrupted.
[ 309.720824] EXT4-fs error (device sdb4): ext4_mb_generate_buddy:735: group 14: 32768 clusters in bitmap, 0 in gd. blk grp corrupted.
[ 309.732858] EXT4-fs error (device sdb4) in ext4_free_blocks:4802: IO failure
[ 309.748321] EXT4-fs error (device sdb4): ext4_mb_generate_buddy:735: group 13: 32768 clusters in bitmap, 0 in gd. blk grp corrupted.
[ 309.760331] EXT4-fs error (device sdb4) in ext4_free_blocks:4802: IO failure
[ 309.769695] EXT4-fs error (device sdb4): ext4_mb_generate_buddy:735: group 12: 32768 clusters in bitmap, 0 in gd. blk grp corrupted.
[ 309.781721] EXT4-fs error (device sdb4) in ext4_free_blocks:4802: IO failure
[ 309.798166] EXT4-fs error (device sdb4): ext4_mb_generate_buddy:735: group 11: 32768 clusters in bitmap, 0 in gd. blk grp corrupted.
[ 309.810184] EXT4-fs error (device sdb4) in ext4_free_blocks:4802: IO failure
[ 309.819532] EXT4-fs error (device sdb4): ext4_mb_generate_buddy:735: group 10: 32768 clusters in bitmap, 0 in gd. blk grp corrupted.
Google-Bug-Id: 7258357
[darrick.wong@oracle.com]
Further modifications (by Darrick) to make more obvious that this corruption
bit applies to blocks only. Set the corruption flag if the block group bitmap
verification fails.
Original-author: Aditya Kali <adityakali@google.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2013-08-28 17:35:51 -04:00
|
|
|
|
|
|
|
|
|
2013-08-28 18:32:58 -04:00
|
|
|
|
|
|
|
|
|
2009-01-05 22:19:52 -05:00
|
|
|
|
2011-07-11 00:03:38 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-04-21 10:54:07 +03:00
|
|
|
|
|
|
|
|
|
2011-07-11 00:03:38 -04:00
|
|
|
|
ext4: Avoid group preallocation for closed files
Currently the group preallocation code tries to find a large (512)
free block from which to do per-cpu group allocation for small files.
The problem with this scheme is that it leaves the filesystem horribly
fragmented. In the worst case, if the filesystem is unmounted and
remounted (after a system shutdown, for example) we forget the fact
that wee were using a particular (now-partially filled) 512 block
extent. So the next time we try to allocate space for a small file,
we will find *another* completely free 512 block chunk to allocate
small files. Given that there are 32,768 blocks in a block group,
after 64 iterations of "mount, write one 4k file in a directory,
unmount", the block group will have 64 files, each separated by 511
blocks, and the block group will no longer have any free 512
completely free chunks of blocks for group preallocation space.
So if we try to allocate blocks for a file that has been closed, such
that we know the final size of the file, and the filesystem is not
busy, avoid using group preallocation.
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2009-09-18 13:34:02 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-02 20:35:09 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: Avoid group preallocation for closed files
Currently the group preallocation code tries to find a large (512)
free block from which to do per-cpu group allocation for small files.
The problem with this scheme is that it leaves the filesystem horribly
fragmented. In the worst case, if the filesystem is unmounted and
remounted (after a system shutdown, for example) we forget the fact
that wee were using a particular (now-partially filled) 512 block
extent. So the next time we try to allocate space for a small file,
we will find *another* completely free 512 block chunk to allocate
small files. Given that there are 32,768 blocks in a block group,
after 64 iterations of "mount, write one 4k file in a directory,
unmount", the block group will have 64 files, each separated by 511
blocks, and the block group will no longer have any free 512
completely free chunks of blocks for group preallocation space.
So if we try to allocate blocks for a file that has been closed, such
that we know the final size of the file, and the filesystem is not
busy, avoid using group preallocation.
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2009-09-18 13:34:02 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-01-05 22:19:52 -05:00
|
|
|
|
|
|
|
|
|
ext4: Avoid group preallocation for closed files
Currently the group preallocation code tries to find a large (512)
free block from which to do per-cpu group allocation for small files.
The problem with this scheme is that it leaves the filesystem horribly
fragmented. In the worst case, if the filesystem is unmounted and
remounted (after a system shutdown, for example) we forget the fact
that wee were using a particular (now-partially filled) 512 block
extent. So the next time we try to allocate space for a small file,
we will find *another* completely free 512 block chunk to allocate
small files. Given that there are 32,768 blocks in a block group,
after 64 iterations of "mount, write one 4k file in a directory,
unmount", the block group will have 64 files, each separated by 511
blocks, and the block group will no longer have any free 512
completely free chunks of blocks for group preallocation space.
So if we try to allocate blocks for a file that has been closed, such
that we know the final size of the file, and the filesystem is not
busy, avoid using group preallocation.
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2009-09-18 13:34:02 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-01-05 22:19:52 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-02 20:35:09 -04:00
|
|
|
|
2009-01-05 22:19:52 -05:00
|
|
|
|
|
|
|
|
|
2020-10-22 06:20:59 +03:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:05 +02:00
|
|
|
|
2020-10-22 06:20:59 +03:00
|
|
|
|
|
|
|
|
|
2011-06-27 19:16:02 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2007-02-12 00:55:40 -08:00
|
|
|
|
2006-10-11 01:20:53 -07:00
|
|
|
|
2010-10-27 21:30:06 -04:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2013-02-09 15:23:03 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-03-24 18:01:09 +00:00
|
|
|
|
2013-02-09 15:23:03 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-07-15 14:24:01 -04:00
|
|
|
|
2023-05-15 16:10:44 +05:30
|
|
|
|
|
|
|
|
|
2013-02-09 15:23:03 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-07-15 14:24:01 -04:00
|
|
|
|
2013-02-09 15:23:03 -05:00
|
|
|
|
2015-05-18 13:14:47 -04:00
|
|
|
|
|
|
|
|
|
2016-01-08 16:00:31 -05:00
|
|
|
|
2013-02-09 15:23:03 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-05-17 16:08:53 -04:00
|
|
|
|
2013-02-09 15:23:03 -05:00
|
|
|
|
2019-06-21 21:57:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-02-09 15:23:03 -05:00
|
|
|
|
2015-05-18 13:14:47 -04:00
|
|
|
|
2013-02-09 15:23:03 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-07-10 14:01:03 -04:00
|
|
|
|
2013-02-09 15:23:03 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2022-06-30 17:01:00 +08:00
|
|
|
|
2017-10-01 17:57:54 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2017-01-22 19:35:49 -05:00
|
|
|
|
2013-02-09 15:23:03 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-07-15 10:10:04 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2007-02-12 00:55:40 -08:00
|
|
|
|
|
|
|
|
|
2009-05-01 13:49:15 -04:00
|
|
|
|
2012-12-10 14:05:57 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-06-21 16:31:47 -04:00
|
|
|
|
|
|
|
|
|
2019-06-21 15:49:26 -04:00
|
|
|
|
|
|
|
|
|
2022-11-06 14:48:36 -08:00
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:59 -07:00
|
|
|
|
|
|
|
|
|
ext4: Support case-insensitive file name lookups
This patch implements the actual support for case-insensitive file name
lookups in ext4, based on the feature bit and the encoding stored in the
superblock.
A filesystem that has the casefold feature set is able to configure
directories with the +F (EXT4_CASEFOLD_FL) attribute, enabling lookups
to succeed in that directory in a case-insensitive fashion, i.e: match
a directory entry even if the name used by userspace is not a byte per
byte match with the disk name, but is an equivalent case-insensitive
version of the Unicode string. This operation is called a
case-insensitive file name lookup.
The feature is configured as an inode attribute applied to directories
and inherited by its children. This attribute can only be enabled on
empty directories for filesystems that support the encoding feature,
thus preventing collision of file names that only differ by case.
* dcache handling:
For a +F directory, Ext4 only stores the first equivalent name dentry
used in the dcache. This is done to prevent unintentional duplication of
dentries in the dcache, while also allowing the VFS code to quickly find
the right entry in the cache despite which equivalent string was used in
a previous lookup, without having to resort to ->lookup().
d_hash() of casefolded directories is implemented as the hash of the
casefolded string, such that we always have a well-known bucket for all
the equivalencies of the same string. d_compare() uses the
utf8_strncasecmp() infrastructure, which handles the comparison of
equivalent, same case, names as well.
For now, negative lookups are not inserted in the dcache, since they
would need to be invalidated anyway, because we can't trust missing file
dentries. This is bad for performance but requires some leveraging of
the vfs layer to fix. We can live without that for now, and so does
everyone else.
* on-disk data:
Despite using a specific version of the name as the internal
representation within the dcache, the name stored and fetched from the
disk is a byte-per-byte match with what the user requested, making this
implementation 'name-preserving'. i.e. no actual information is lost
when writing to storage.
DX is supported by modifying the hashes used in +F directories to make
them case/encoding-aware. The new disk hashes are calculated as the
hash of the full casefolded string, instead of the string directly.
This allows us to efficiently search for file names in the htree without
requiring the user to provide an exact name.
* Dealing with invalid sequences:
By default, when a invalid UTF-8 sequence is identified, ext4 will treat
it as an opaque byte sequence, ignoring the encoding and reverting to
the old behavior for that unique file. This means that case-insensitive
file name lookup will not work only for that file. An optional bit can
be set in the superblock telling the filesystem code and userspace tools
to enforce the encoding. When that optional bit is set, any attempt to
create a file name using an invalid UTF-8 sequence will fail and return
an error to userspace.
* Normalization algorithm:
The UTF-8 algorithms used to compare strings in ext4 is implemented
lives in fs/unicode, and is based on a previous version developed by
SGI. It implements the Canonical decomposition (NFD) algorithm
described by the Unicode specification 12.1, or higher, combined with
the elimination of ignorable code points (NFDi) and full
case-folding (CF) as documented in fs/unicode/utf8_norm.c.
NFD seems to be the best normalization method for EXT4 because:
- It has a lower cost than NFC/NFKC (which requires
decomposing to NFD as an intermediary step)
- It doesn't eliminate important semantic meaning like
compatibility decompositions.
Although:
- This implementation is not completely linguistic accurate, because
different languages have conflicting rules, which would require the
specialization of the filesystem to a given locale, which brings all
sorts of problems for removable media and for users who use more than
one language.
Signed-off-by: Gabriel Krisman Bertazi <krisman@collabora.co.uk>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2019-04-25 14:12:08 -04:00
|
|
|
|
2013-04-19 17:53:09 -04:00
|
|
|
|
2017-08-05 19:00:31 -04:00
|
|
|
|
2013-04-19 17:53:09 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-10-17 16:18:43 -04:00
|
|
|
|
2013-04-19 17:53:09 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-04-08 00:00:32 -04:00
|
|
|
|
2020-06-01 21:47:20 -07:00
|
|
|
|
2023-03-24 18:01:23 +00:00
|
|
|
|
2019-07-22 09:26:24 -07:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2015-04-27 17:51:30 -04:00
|
|
|
|
2007-02-12 00:55:40 -08:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
2015-09-23 12:44:17 -04:00
|
|
|
|
2021-06-11 14:02:08 +00:00
|
|
|
|
2015-09-23 12:44:17 -04:00
|
|
|
|
2015-09-23 12:46:17 -04:00
|
|
|
|
2015-09-23 12:44:17 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-17 15:38:01 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-27 21:30:14 -04:00
|
|
|
|
|
|
|
|
|
2020-07-28 15:04:34 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-06-27 19:16:02 -04:00
|
|
|
|
|
|
|
|
|
2022-02-16 12:32:49 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-17 15:38:01 -04:00
|
|
|
|
2006-10-11 01:21:03 -07:00
|
|
|
|
2012-11-28 13:03:30 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-08-28 14:47:06 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-04-26 18:34:37 -07:00
|
|
|
|
2013-06-04 13:01:11 -04:00
|
|
|
|
2010-05-16 19:00:00 -04:00
|
|
|
|
|
|
|
|
|
2016-11-13 22:02:28 -05:00
|
|
|
|
2013-04-03 12:45:17 -04:00
|
|
|
|
|
|
|
|
|
2006-10-11 01:21:03 -07:00
|
|
|
|
|
|
|
|
|
2011-01-14 13:07:43 +01:00
|
|
|
|
2007-07-17 21:42:41 -04:00
|
|
|
|
2013-06-04 13:21:11 -04:00
|
|
|
|
|
|
|
|
|
2019-10-16 13:07:08 +05:30
|
|
|
|
|
|
|
|
|
2010-05-16 19:00:00 -04:00
|
|
|
|
|
|
|
|
|
2012-11-28 13:03:30 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-08-22 10:35:34 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-09-01 14:43:09 -04:00
|
|
|
|
2024-08-22 10:35:31 +08:00
|
|
|
|
2014-09-01 14:43:09 -04:00
|
|
|
|
2022-09-24 10:12:11 +08:00
|
|
|
|
2012-11-28 13:03:30 -05:00
|
|
|
|
2014-08-30 23:52:19 -04:00
|
|
|
|
2008-11-22 15:04:59 -05:00
|
|
|
|
|
|
|
|
|
2019-08-11 16:32:41 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-08-16 22:05:14 -04:00
|
|
|
|
2014-08-30 23:52:19 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2018-10-01 14:19:37 -04:00
|
|
|
|
2019-11-05 17:44:16 +01:00
|
|
|
|
2019-11-05 17:44:29 +01:00
|
|
|
|
|
|
|
|
|
2020-10-15 13:37:59 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-11-05 17:44:16 +01:00
|
|
|
|
2012-11-28 13:03:30 -05:00
|
|
|
|
2009-06-17 19:24:03 -04:00
|
|
|
|
2013-04-08 12:54:05 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-06-17 19:24:03 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-27 21:30:10 -04:00
|
|
|
|
2010-10-27 21:30:14 -04:00
|
|
|
|
|
|
|
|
|
2010-10-27 21:30:10 -04:00
|
|
|
|
2013-06-04 11:58:58 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2013-06-04 14:21:02 -04:00
|
|
|
|
2010-10-27 21:30:10 -04:00
|
|
|
|
2023-03-24 18:01:08 +00:00
|
|
|
|
|
|
|
|
|
2019-10-16 13:07:10 +05:30
|
|
|
|
|
|
|
|
|
2008-11-22 15:04:59 -05:00
|
|
|
|
2011-05-24 18:31:25 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-04-30 21:50:46 +03:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-07-22 09:26:24 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:05 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:06 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-08-16 11:57:05 +02:00
|
|
|
|
2009-01-05 21:49:55 -05:00
|
|
|
|
2012-09-19 21:48:00 -04:00
|
|
|
|
2009-01-05 21:49:55 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ext4: serialize unaligned asynchronous DIO
ext4 has a data corruption case when doing non-block-aligned
asynchronous direct IO into a sparse file, as demonstrated
by xfstest 240.
The root cause is that while ext4 preallocates space in the
hole, mappings of that space still look "new" and
dio_zero_block() will zero out the unwritten portions. When
more than one AIO thread is going, they both find this "new"
block and race to zero out their portion; this is uncoordinated
and causes data corruption.
Dave Chinner fixed this for xfs by simply serializing all
unaligned asynchronous direct IO. I've done the same here.
The difference is that we only wait on conversions, not all IO.
This is a very big hammer, and I'm not very pleased with
stuffing this into ext4_file_write(). But since ext4 is
DIO_LOCKING, we need to serialize it at this high level.
I tried to move this into ext4_ext_direct_IO, but by then
we have the i_mutex already, and we will wait on the
work queue to do conversions - which must also take the
i_mutex. So that won't work.
This was originally exposed by qemu-kvm installing to
a raw disk image with a normal sector-63 alignment. I've
tested a backport of this patch with qemu, and it does
avoid the corruption. It is also quite a lot slower
(14 min for package installs, vs. 8 min for well-aligned)
but I'll take slow correctness over fast corruption any day.
Mingming suggested that we can track outstanding
conversions, and wait on those so that non-sparse
files won't be affected, and I've implemented that here;
unaligned AIO to nonsparse files won't take a perf hit.
[tytso@mit.edu: Keep the mutex as a hashed array instead
of bloating the ext4 inode]
[tytso@mit.edu: Fix up namespace issues so that global
variables are protected with an "ext4_" prefix.]
Signed-off-by: Eric Sandeen <sandeen@redhat.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
2011-02-12 08:17:34 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-26 21:35:44 -04:00
|
|
|
|
2022-06-29 00:00:26 -04:00
|
|
|
|
2011-07-26 21:35:44 -04:00
|
|
|
|
2016-02-29 08:36:38 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2017-01-27 23:20:26 -08:00
|
|
|
|
2019-12-18 18:44:33 +01:00
|
|
|
|
2019-11-05 23:03:31 +11:00
|
|
|
|
2016-11-20 17:36:06 -05:00
|
|
|
|
2019-08-22 23:00:32 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-04-26 14:29:47 +08:00
|
|
|
|
2019-08-22 23:00:32 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-11-04 16:22:57 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-08-30 13:37:38 +08:00
|
|
|
|
|
|
|
|
|
2024-08-30 13:37:37 +08:00
|
|
|
|
2006-10-11 01:20:50 -07:00
|
|
|
|
|
|
|
|
|
2015-10-17 16:16:04 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-04-29 18:13:32 -04:00
|
|
|
|
