Gh0st
/
linux
mirror of https://github.com/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
linux/fs/ext4/move_extent.c

655 lines
18 KiB
C
Raw Blame History

// SPDX-License-Identifier: LGPL-2.1
/*
* Copyright (c) 2008,2009 NEC Software Tohoku, Ltd.
* Written by Takashi Sato <t-sato@yk.jp.nec.com>
* Akira Fujita <a-fujita@rs.jp.nec.com>
*/
#include <linux/fs.h>
#include <linux/quotaops.h>
#include <linux/slab.h>
#include <linux/sched/mm.h>
#include "ext4_jbd2.h"
#include "ext4.h"
#include "ext4_extents.h"
#include <trace/events/ext4.h>
struct mext_data {
struct inode *orig_inode; /* Origin file inode */
struct inode *donor_inode; /* Donor file inode */
struct ext4_map_blocks orig_map;/* Origin file's move mapping */
ext4_lblk_t donor_lblk; /* Start block of the donor file */
};
/**
* ext4_double_down_write_data_sem() - write lock two inodes's i_data_sem
* @first: inode to be locked
* @second: inode to be locked
*
* Acquire write lock of i_data_sem of the two inodes
*/
void
ext4_double_down_write_data_sem(struct inode *first, struct inode *second)
{
if (first < second) {
down_write(&EXT4_I(first)->i_data_sem);
down_write_nested(&EXT4_I(second)->i_data_sem, I_DATA_SEM_OTHER);
} else {
down_write(&EXT4_I(second)->i_data_sem);
down_write_nested(&EXT4_I(first)->i_data_sem, I_DATA_SEM_OTHER);
}
}
/**
* ext4_double_up_write_data_sem - Release two inodes' write lock of i_data_sem
*
* @orig_inode: original inode structure to be released its lock first
* @donor_inode: donor inode structure to be released its lock second
* Release write lock of i_data_sem of two inodes (orig and donor).
*/
void
ext4_double_up_write_data_sem(struct inode *orig_inode,
struct inode *donor_inode)
{
up_write(&EXT4_I(orig_inode)->i_data_sem);
up_write(&EXT4_I(donor_inode)->i_data_sem);
}
/* Grab and lock folio on both @inode1 and @inode2 by inode order. */
static int mext_folio_double_lock(struct inode *inode1, struct inode *inode2,
pgoff_t index1, pgoff_t index2, size_t len,
struct folio *folio[2])
{
struct address_space *mapping[2];
unsigned int flags;
fgf_t fgp_flags = FGP_WRITEBEGIN;
BUG_ON(!inode1 || !inode2);
if (inode1 < inode2) {
mapping[0] = inode1->i_mapping;
mapping[1] = inode2->i_mapping;
} else {
swap(index1, index2);
mapping[0] = inode2->i_mapping;
mapping[1] = inode1->i_mapping;
}
flags = memalloc_nofs_save();
fgp_flags |= fgf_set_order(len);
folio[0] = __filemap_get_folio(mapping[0], index1, fgp_flags,
mapping_gfp_mask(mapping[0]));
if (IS_ERR(folio[0])) {
memalloc_nofs_restore(flags);
return PTR_ERR(folio[0]);
}
folio[1] = __filemap_get_folio(mapping[1], index2, fgp_flags,
mapping_gfp_mask(mapping[1]));
memalloc_nofs_restore(flags);
if (IS_ERR(folio[1])) {
folio_unlock(folio[0]);
folio_put(folio[0]);
return PTR_ERR(folio[1]);
}
/*
* __filemap_get_folio() may not wait on folio's writeback if
* BDI not demand that. But it is reasonable to be very conservative
* here and explicitly wait on folio's writeback
*/
folio_wait_writeback(folio[0]);
folio_wait_writeback(folio[1]);
if (inode1 > inode2)
swap(folio[0], folio[1]);
return 0;
}
static void mext_folio_double_unlock(struct folio *folio[2])
{
folio_unlock(folio[0]);
folio_put(folio[0]);
folio_unlock(folio[1]);
folio_put(folio[1]);
}
/* Force folio buffers uptodate w/o dropping folio's lock */
static int mext_folio_mkuptodate(struct folio *folio, size_t from, size_t to)
{
struct inode *inode = folio->mapping->host;
sector_t block;
struct buffer_head *bh, *head;
unsigned int blocksize, block_start, block_end;
int nr = 0;
bool partial = false;
BUG_ON(!folio_test_locked(folio));
BUG_ON(folio_test_writeback(folio));
if (folio_test_uptodate(folio))
return 0;
blocksize = i_blocksize(inode);
head = folio_buffers(folio);
if (!head)
head = create_empty_buffers(folio, blocksize, 0);
block = folio_pos(folio) >> inode->i_blkbits;
block_end = 0;
bh = head;
do {
block_start = block_end;
block_end = block_start + blocksize;
if (block_end <= from || block_start >= to) {
if (!buffer_uptodate(bh))
partial = true;
continue;
}
if (buffer_uptodate(bh))
continue;
if (!buffer_mapped(bh)) {
int err = ext4_get_block(inode, block, bh, 0);
if (err)
return err;
if (!buffer_mapped(bh)) {
folio_zero_range(folio, block_start, blocksize);
set_buffer_uptodate(bh);
continue;
}
}
lock_buffer(bh);
if (buffer_uptodate(bh)) {
unlock_buffer(bh);
continue;
}
ext4_read_bh_nowait(bh, 0, NULL, false);
nr++;
} while (block++, (bh = bh->b_this_page) != head);
/* No io required */
if (!nr)
goto out;
bh = head;
do {
if (bh_offset(bh) + blocksize <= from)
continue;
if (bh_offset(bh) >= to)
break;
wait_on_buffer(bh);
if (buffer_uptodate(bh))
continue;
return -EIO;
} while ((bh = bh->b_this_page) != head);
out:
if (!partial)
folio_mark_uptodate(folio);
return 0;
}
enum mext_move_type {MEXT_SKIP_EXTENT, MEXT_MOVE_EXTENT, MEXT_COPY_DATA};
/*
* Start to move extent between the origin inode and the donor inode,
* hold one folio for each inode and check the candidate moving extent
* mapping status again.
*/
static int mext_move_begin(struct mext_data *mext, struct folio *folio[2],
enum mext_move_type *move_type)
{
struct inode *orig_inode = mext->orig_inode;
struct inode *donor_inode = mext->donor_inode;
unsigned int blkbits = orig_inode->i_blkbits;
struct ext4_map_blocks donor_map = {0};
loff_t orig_pos, donor_pos;
size_t move_len;
int ret;
orig_pos = ((loff_t)mext->orig_map.m_lblk) << blkbits;
donor_pos = ((loff_t)mext->donor_lblk) << blkbits;
ret = mext_folio_double_lock(orig_inode, donor_inode,
orig_pos >> PAGE_SHIFT, donor_pos >> PAGE_SHIFT,
((size_t)mext->orig_map.m_len) << blkbits, folio);
if (ret)
return ret;
/*
* Check the origin inode's mapping information again under the
* folio lock, as we do not hold the i_data_sem at all times, and
* it may change during the concurrent write-back operation.
*/
if (mext->orig_map.m_seq != READ_ONCE(EXT4_I(orig_inode)->i_es_seq)) {
ret = -ESTALE;
goto error;
}
/* Adjust the moving length according to the length of shorter folio. */
move_len = umin(folio_pos(folio[0]) + folio_size(folio[0]) - orig_pos,
folio_pos(folio[1]) + folio_size(folio[1]) - donor_pos);
move_len >>= blkbits;
if (move_len < mext->orig_map.m_len)
mext->orig_map.m_len = move_len;
donor_map.m_lblk = mext->donor_lblk;
donor_map.m_len = mext->orig_map.m_len;
donor_map.m_flags = 0;
ret = ext4_map_blocks(NULL, donor_inode, &donor_map, 0);
if (ret < 0)
goto error;
/* Adjust the moving length according to the donor mapping length. */
mext->orig_map.m_len = donor_map.m_len;
/* Skip moving if the donor range is a hole or a delalloc extent. */
if (!(donor_map.m_flags & (EXT4_MAP_MAPPED | EXT4_MAP_UNWRITTEN)))
*move_type = MEXT_SKIP_EXTENT;
/* If both mapping ranges are unwritten, no need to copy data. */
else if ((mext->orig_map.m_flags & EXT4_MAP_UNWRITTEN) &&
(donor_map.m_flags & EXT4_MAP_UNWRITTEN))
*move_type = MEXT_MOVE_EXTENT;
else
*move_type = MEXT_COPY_DATA;
return 0;
error:
mext_folio_double_unlock(folio);
return ret;
}
/*
* Re-create the new moved mapping buffers of the original inode and commit
* the entire written range.
*/
static int mext_folio_mkwrite(struct inode *inode, struct folio *folio,
size_t from, size_t to)
{
unsigned int blocksize = i_blocksize(inode);
struct buffer_head *bh, *head;
size_t block_start, block_end;
sector_t block;
int ret;
head = folio_buffers(folio);
if (!head)
head = create_empty_buffers(folio, blocksize, 0);
block = folio_pos(folio) >> inode->i_blkbits;
block_end = 0;
bh = head;
do {
block_start = block_end;
block_end = block_start + blocksize;
if (block_end <= from || block_start >= to)
continue;
ret = ext4_get_block(inode, block, bh, 0);
if (ret)
return ret;
} while (block++, (bh = bh->b_this_page) != head);
block_commit_write(folio, from, to);
return 0;
}
/*
* Save the data in original inode extent blocks and replace one folio size
* aligned original inode extent with one or one partial donor inode extent,
* and then write out the saved data in new original inode blocks. Pass out
* the replaced block count through m_len. Return 0 on success, and an error
* code otherwise.
*/
static int mext_move_extent(struct mext_data *mext, u64 *m_len)
{
struct inode *orig_inode = mext->orig_inode;
struct inode *donor_inode = mext->donor_inode;
struct ext4_map_blocks *orig_map = &mext->orig_map;
unsigned int blkbits = orig_inode->i_blkbits;
struct folio *folio[2] = {NULL, NULL};
loff_t from, length;
enum mext_move_type move_type = 0;
handle_t *handle;
u64 r_len = 0;
unsigned int credits;
int ret, ret2;
*m_len = 0;
trace_ext4_move_extent_enter(orig_inode, orig_map, donor_inode,
mext->donor_lblk);
credits = ext4_chunk_trans_extent(orig_inode, 0) * 2;
handle = ext4_journal_start(orig_inode, EXT4_HT_MOVE_EXTENTS, credits);
if (IS_ERR(handle)) {
ret = PTR_ERR(handle);
goto out;
}
ret = mext_move_begin(mext, folio, &move_type);
if (ret)
goto stop_handle;
if (move_type == MEXT_SKIP_EXTENT)
goto unlock;
/*
* Copy the data. First, read the original inode data into the page
* cache. Then, release the existing mapping relationships and swap
* the extent. Finally, re-establish the new mapping relationships
* and dirty the page cache.
*/
if (move_type == MEXT_COPY_DATA) {
from = offset_in_folio(folio[0],
((loff_t)orig_map->m_lblk) << blkbits);
length = ((loff_t)orig_map->m_len) << blkbits;
ret = mext_folio_mkuptodate(folio[0], from, from + length);
if (ret)
goto unlock;
}
if (!filemap_release_folio(folio[0], 0) ||
!filemap_release_folio(folio[1], 0)) {
ret = -EBUSY;
goto unlock;
}
/* Move extent */
ext4_double_down_write_data_sem(orig_inode, donor_inode);
*m_len = ext4_swap_extents(handle, orig_inode, donor_inode,
orig_map->m_lblk, mext->donor_lblk,
orig_map->m_len, 1, &ret);
ext4_double_up_write_data_sem(orig_inode, donor_inode);
/* A short-length swap cannot occur after a successful swap extent. */
if (WARN_ON_ONCE(!ret && (*m_len != orig_map->m_len)))
ret = -EIO;
if (!(*m_len) || (move_type == MEXT_MOVE_EXTENT))
goto unlock;
/* Copy data */
length = (*m_len) << blkbits;
ret2 = mext_folio_mkwrite(orig_inode, folio[0], from, from + length);
if (ret2) {
if (!ret)
ret = ret2;
goto repair_branches;
}
/*
* Even in case of data=writeback it is reasonable to pin
* inode to transaction, to prevent unexpected data loss.
*/
ret2 = ext4_jbd2_inode_add_write(handle, orig_inode,
((loff_t)orig_map->m_lblk) << blkbits, length);
if (!ret)
ret = ret2;
unlock:
mext_folio_double_unlock(folio);
stop_handle:
ext4_journal_stop(handle);
out:
trace_ext4_move_extent_exit(orig_inode, orig_map->m_lblk, donor_inode,
mext->donor_lblk, orig_map->m_len, *m_len,
move_type, ret);
return ret;
repair_branches:
ret2 = 0;
r_len = ext4_swap_extents(handle, donor_inode, orig_inode,
mext->donor_lblk, orig_map->m_lblk,
*m_len, 0, &ret2);
if (ret2 || r_len != *m_len) {
ext4_error_inode_block(orig_inode, (sector_t)(orig_map->m_lblk),
EIO, "Unable to copy data block, data will be lost!");
ret = -EIO;
}
*m_len = 0;
goto unlock;
}
/*
* Check the validity of the basic filesystem environment and the
* inodes' support status.
*/
static int mext_check_validity(struct inode *orig_inode,
struct inode *donor_inode)
{
struct super_block *sb = orig_inode->i_sb;
/* origin and donor should be different inodes */
if (orig_inode == donor_inode) {
ext4_debug("ext4 move extent: The argument files should not be same inode [ino:orig %lu, donor %lu]\n",
orig_inode->i_ino, donor_inode->i_ino);
return -EINVAL;
}
/* origin and donor should belone to the same filesystem */
if (orig_inode->i_sb != donor_inode->i_sb) {
ext4_debug("ext4 move extent: The argument files should be in same FS [ino:orig %lu, donor %lu]\n",
orig_inode->i_ino, donor_inode->i_ino);
return -EINVAL;
}
/* Regular file check */
if (!S_ISREG(orig_inode->i_mode) || !S_ISREG(donor_inode->i_mode)) {
ext4_debug("ext4 move extent: The argument files should be regular file [ino:orig %lu, donor %lu]\n",
orig_inode->i_ino, donor_inode->i_ino);
return -EINVAL;
}
if (ext4_has_feature_bigalloc(sb)) {
ext4_msg(sb, KERN_ERR,
"Online defrag not supported with bigalloc");
return -EOPNOTSUPP;
}
if (IS_DAX(orig_inode)) {
ext4_msg(sb, KERN_ERR,
"Online defrag not supported with DAX");
return -EOPNOTSUPP;
}
/*
* TODO: it's not obvious how to swap blocks for inodes with full
* journaling enabled.
*/
if (ext4_should_journal_data(orig_inode) ||
ext4_should_journal_data(donor_inode)) {
ext4_msg(sb, KERN_ERR,
"Online defrag not supported with data journaling");
return -EOPNOTSUPP;
}
if (IS_ENCRYPTED(orig_inode) || IS_ENCRYPTED(donor_inode)) {
ext4_msg(sb, KERN_ERR,
"Online defrag not supported for encrypted files");
return -EOPNOTSUPP;
}
/* Ext4 move extent supports only extent based file */
if (!(ext4_test_inode_flag(orig_inode, EXT4_INODE_EXTENTS)) ||
!(ext4_test_inode_flag(donor_inode, EXT4_INODE_EXTENTS))) {
ext4_msg(sb, KERN_ERR,
"Online defrag not supported for non-extent files");
return -EOPNOTSUPP;
}
if (donor_inode->i_mode & (S_ISUID|S_ISGID)) {
ext4_debug("ext4 move extent: suid or sgid is set to donor file [ino:orig %lu, donor %lu]\n",
orig_inode->i_ino, donor_inode->i_ino);
return -EINVAL;
}
if (IS_IMMUTABLE(donor_inode) || IS_APPEND(donor_inode)) {
ext4_debug("ext4 move extent: donor should not be immutable or append file [ino:orig %lu, donor %lu]\n",
orig_inode->i_ino, donor_inode->i_ino);
return -EPERM;
}
/* Ext4 move extent does not support swap files */
if (IS_SWAPFILE(orig_inode) || IS_SWAPFILE(donor_inode)) {
ext4_debug("ext4 move extent: The argument files should not be swap files [ino:orig %lu, donor %lu]\n",
orig_inode->i_ino, donor_inode->i_ino);
return -ETXTBSY;
}
if (ext4_is_quota_file(orig_inode) || ext4_is_quota_file(donor_inode)) {
ext4_debug("ext4 move extent: The argument files should not be quota files [ino:orig %lu, donor %lu]\n",
orig_inode->i_ino, donor_inode->i_ino);
return -EOPNOTSUPP;
}
if ((!orig_inode->i_size) || (!donor_inode->i_size)) {
ext4_debug("ext4 move extent: File size is 0 byte\n");
return -EINVAL;
}
return 0;
}
/*
* Check the moving range of ext4_move_extents() whether the files can be
* exchanged with each other, and adjust the length to fit within the file
* size. Return 0 on success, or a negative error value on failure.
*/
static int mext_check_adjust_range(struct inode *orig_inode,
struct inode *donor_inode, __u64 orig_start,
__u64 donor_start, __u64 *len)
{
__u64 orig_eof, donor_eof;
/* Start offset should be same */
if ((orig_start & ~(PAGE_MASK >> orig_inode->i_blkbits)) !=
(donor_start & ~(PAGE_MASK >> orig_inode->i_blkbits))) {
ext4_debug("ext4 move extent: orig and donor's start offsets are not aligned [ino:orig %lu, donor %lu]\n",
orig_inode->i_ino, donor_inode->i_ino);
return -EINVAL;
}
if ((orig_start >= EXT_MAX_BLOCKS) ||
(donor_start >= EXT_MAX_BLOCKS) ||
(*len > EXT_MAX_BLOCKS) ||
(donor_start + *len >= EXT_MAX_BLOCKS) ||
(orig_start + *len >= EXT_MAX_BLOCKS)) {
ext4_debug("ext4 move extent: Can't handle over [%u] blocks [ino:orig %lu, donor %lu]\n",
EXT_MAX_BLOCKS,
orig_inode->i_ino, donor_inode->i_ino);
return -EINVAL;
}
orig_eof = EXT4_B_TO_LBLK(orig_inode, i_size_read(orig_inode));
donor_eof = EXT4_B_TO_LBLK(donor_inode, i_size_read(donor_inode));
if (orig_eof <= orig_start)
*len = 0;
else if (orig_eof < orig_start + *len - 1)
*len = orig_eof - orig_start;
if (donor_eof <= donor_start)
*len = 0;
else if (donor_eof < donor_start + *len - 1)
*len = donor_eof - donor_start;
if (!*len) {
ext4_debug("ext4 move extent: len should not be 0 [ino:orig %lu, donor %lu]\n",
orig_inode->i_ino, donor_inode->i_ino);
return -EINVAL;
}
return 0;
}
/**
* ext4_move_extents - Exchange the specified range of a file
*
* @o_filp: file structure of the original file
* @d_filp: file structure of the donor file
* @orig_blk: start offset in block for orig
* @donor_blk: start offset in block for donor
* @len: the number of blocks to be moved
* @moved_len: moved block length
*
* This function returns 0 and moved block length is set in moved_len
* if succeed, otherwise returns error value.
*/
int ext4_move_extents(struct file *o_filp, struct file *d_filp, __u64 orig_blk,
__u64 donor_blk, __u64 len, __u64 *moved_len)
{
struct inode *orig_inode = file_inode(o_filp);
struct inode *donor_inode = file_inode(d_filp);
struct mext_data mext;
struct super_block *sb = orig_inode->i_sb;
struct ext4_sb_info *sbi = EXT4_SB(sb);
int retries = 0;
u64 m_len;
int ret;
*moved_len = 0;
/* Protect orig and donor inodes against a truncate */
lock_two_nondirectories(orig_inode, donor_inode);
ret = mext_check_validity(orig_inode, donor_inode);
if (ret)
goto out;
/* Wait for all existing dio workers */
inode_dio_wait(orig_inode);
inode_dio_wait(donor_inode);
/* Check and adjust the specified move_extent range. */
ret = mext_check_adjust_range(orig_inode, donor_inode, orig_blk,
donor_blk, &len);
if (ret)
goto out;
mext.orig_inode = orig_inode;
mext.donor_inode = donor_inode;
while (len) {
mext.orig_map.m_lblk = orig_blk;
mext.orig_map.m_len = len;
mext.orig_map.m_flags = 0;
mext.donor_lblk = donor_blk;
ret = ext4_map_blocks(NULL, orig_inode, &mext.orig_map, 0);
if (ret < 0)
goto out;
/* Skip moving if it is a hole or a delalloc extent. */
if (mext.orig_map.m_flags &
(EXT4_MAP_MAPPED | EXT4_MAP_UNWRITTEN)) {
ret = mext_move_extent(&mext, &m_len);
*moved_len += m_len;
if (!ret)
goto next;
/* Move failed or partially failed. */
if (m_len) {
orig_blk += m_len;
donor_blk += m_len;
len -= m_len;
}
if (ret == -ESTALE)
continue;
if (ret == -ENOSPC &&
ext4_should_retry_alloc(sb, &retries))
continue;
if (ret == -EBUSY &&
sbi->s_journal && retries++ < 4 &&
jbd2_journal_force_commit_nested(sbi->s_journal))
continue;
goto out;
}
next:
orig_blk += mext.orig_map.m_len;
donor_blk += mext.orig_map.m_len;
len -= mext.orig_map.m_len;
retries = 0;
}
out:
if (*moved_len) {
ext4_discard_preallocations(orig_inode);
ext4_discard_preallocations(donor_inode);
}
unlock_two_nondirectories(orig_inode, donor_inode);
return ret;
}
Reference in New Issue View Git Blame Copy Permalink