fast_commit.c

// SPDX-License-Identifier: GPL-2.0

/*
 * fs/ext4/fast_commit.c
 *
 * Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com>
 *
 * Ext4 fast commits routines.
 */
#include "ext4.h"
#include "ext4_jbd2.h"
#include "ext4_extents.h"
#include "mballoc.h"

#include <linux/lockdep.h>
#include <linux/wait_bit.h>
/*
 * Ext4 Fast Commits
 * -----------------
 *
 * Ext4 fast commits implement fine grained journalling for Ext4.
 *
 * Fast commits are organized as a log of tag-length-value (TLV) structs. (See
 * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by
 * TLV during the recovery phase. For the scenarios for which we currently
 * don't have replay code, fast commit falls back to full commits.
 * Fast commits record delta in one of the following three categories.
 *
 * (A) Directory entry updates:
 *
 * - EXT4_FC_TAG_UNLINK		- records directory entry unlink
 * - EXT4_FC_TAG_LINK		- records directory entry link
 * - EXT4_FC_TAG_CREAT		- records inode and directory entry creation
 *
 * (B) File specific data range updates:
 *
 * - EXT4_FC_TAG_ADD_RANGE	- records addition of new blocks to an inode
 * - EXT4_FC_TAG_DEL_RANGE	- records deletion of blocks from an inode
 *
 * (C) Inode metadata (mtime / ctime etc):
 *
 * - EXT4_FC_TAG_INODE		- record the inode that should be replayed
 *				  during recovery. Note that iblocks field is
 *				  not replayed and instead derived during
 *				  replay.
 * Commit Operation
 * ----------------
 * With fast commits, we maintain all the directory entry operations in the
 * order in which they are issued in an in-memory queue. This queue is flushed
 * to disk during the commit operation. We also maintain a list of inodes
 * that need to be committed during a fast commit in another in memory queue of
 * inodes. During the commit operation, we commit in the following order:
 *
 * [1] Prepare all the inodes to write out their data by setting
 *     "EXT4_STATE_FC_FLUSHING_DATA". This ensures that inode cannot be
 *     deleted while it is being flushed.
 * [2] Flush data buffers to disk and clear "EXT4_STATE_FC_FLUSHING_DATA"
 *     state.
 * [3] Lock the journal by calling jbd2_journal_lock_updates(). This ensures
 *     that all the existing handles finish and no new handles can start.
 * [4] Mark all the fast commit eligible inodes as undergoing fast commit by
 *     setting "EXT4_STATE_FC_COMMITTING" state, and snapshot the inode state
 *     needed for log writing.
 * [5] Unlock the journal by calling jbd2_journal_unlock_updates(). This allows
 *     starting of new handles. Updates to inodes being fast committed are
 *     tracked for requeue rather than blocking.
 * [6] Commit all the directory entry updates in the fast commit space.
 * [7] Commit all the changed inodes in the fast commit space.
 * [8] Write tail tag (this tag ensures the atomicity, please read the following
 *     section for more details).
 * [9] Clear "EXT4_STATE_FC_COMMITTING" and wake up waiters in
 *     ext4_fc_cleanup().
 *
 * All the inode updates must be enclosed within jbd2_journal_start()
 * and jbd2_journal_stop() similar to JBD2 journaling.
 *
 * Fast Commit Ineligibility
 * -------------------------
 *
 * Not all operations are supported by fast commits today (e.g extended
 * attributes). Fast commit ineligibility is marked by calling
 * ext4_fc_mark_ineligible(): This makes next fast commit operation to fall back
 * to full commit.
 *
 * Atomicity of commits
 * --------------------
 * In order to guarantee atomicity during the commit operation, fast commit
 * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail
 * tag contains CRC of the contents and TID of the transaction after which
 * this fast commit should be applied. Recovery code replays fast commit
 * logs only if there's at least 1 valid tail present. For every fast commit
 * operation, there is 1 tail. This means, we may end up with multiple tails
 * in the fast commit space. Here's an example:
 *
 * - Create a new file A and remove existing file B
 * - fsync()
 * - Append contents to file A
 * - Truncate file A
 * - fsync()
 *
 * The fast commit space at the end of above operations would look like this:
 *      [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL]
 *             |<---  Fast Commit 1   --->|<---      Fast Commit 2     ---->|
 *
 * Replay code should thus check for all the valid tails in the FC area.
 *
 * Fast Commit Replay Idempotence
 * ------------------------------
 *
 * Fast commits tags are idempotent in nature provided the recovery code follows
 * certain rules. The guiding principle that the commit path follows while
 * committing is that it stores the result of a particular operation instead of
 * storing the procedure.
 *
 * Let's consider this rename operation: 'mv /a /b'. Let's assume dirent '/a'
 * was associated with inode 10. During fast commit, instead of storing this
 * operation as a procedure "rename a to b", we store the resulting file system
 * state as a "series" of outcomes:
 *
 * - Link dirent b to inode 10
 * - Unlink dirent a
 * - Inode <10> with valid refcount
 *
 * Now when recovery code runs, it needs "enforce" this state on the file
 * system. This is what guarantees idempotence of fast commit replay.
 *
 * Let's take an example of a procedure that is not idempotent and see how fast
 * commits make it idempotent. Consider following sequence of operations:
 *
 *     rm A;    mv B A;    read A
 *  (x)     (y)        (z)
 *
 * (x), (y) and (z) are the points at which we can crash. If we store this
 * sequence of operations as is then the replay is not idempotent. Let's say
 * while in replay, we crash at (z). During the second replay, file A (which was
 * actually created as a result of "mv B A" operation) would get deleted. Thus,
 * file named A would be absent when we try to read A. So, this sequence of
 * operations is not idempotent. However, as mentioned above, instead of storing
 * the procedure fast commits store the outcome of each procedure. Thus the fast
 * commit log for above procedure would be as follows:
 *
 * (Let's assume dirent A was linked to inode 10 and dirent B was linked to
 * inode 11 before the replay)
 *
 *    [Unlink A]   [Link A to inode 11]   [Unlink B]   [Inode 11]
 * (w)          (x)                    (y)          (z)
 *
 * If we crash at (z), we will have file A linked to inode 11. During the second
 * replay, we will remove file A (inode 11). But we will create it back and make
 * it point to inode 11. We won't find B, so we'll just skip that step. At this
 * point, the refcount for inode 11 is not reliable, but that gets fixed by the
 * replay of last inode 11 tag. Crashes at points (w), (x) and (y) get handled
 * similarly. Thus, by converting a non-idempotent procedure into a series of
 * idempotent outcomes, fast commits ensured idempotence during the replay.
 *
 * Locking
 * -------
 * sbi->s_fc_lock protects the fast commit inodes queue and the fast commit
 * dentry queue. ei->i_fc_lock protects the fast commit related info in a given
 * inode. Most of the code avoids acquiring both the locks, but if one must do
 * that then sbi->s_fc_lock must be acquired before ei->i_fc_lock.
 *
 * TODOs
 * -----
 *
 * 0) Fast commit replay path hardening: Fast commit replay code should use
 *    journal handles to make sure all the updates it does during the replay
 *    path are atomic. With that if we crash during fast commit replay, after
 *    trying to do recovery again, we will find a file system where fast commit
 *    area is invalid (because new full commit would be found). In order to deal
 *    with that, fast commit replay code should ensure that the "FC_REPLAY"
 *    superblock state is persisted before starting the replay, so that after
 *    the crash, fast commit recovery code can look at that flag and perform
 *    fast commit recovery even if that area is invalidated by later full
 *    commits.
 *
 * 1) Handle more ineligible cases.
 *
 * 2) Change ext4_fc_commit() to lookup logical to physical mapping using extent
 *    status tree. This would get rid of the need to call ext4_fc_track_inode()
 *    before acquiring i_data_sem. To do that we would need to ensure that
 *    modified extents from the extent status tree are not evicted from memory.
 */

#include <trace/events/ext4.h>
static struct kmem_cache *ext4_fc_dentry_cachep;
static struct kmem_cache *ext4_fc_range_cachep;

/*
 * Avoid spending unbounded time/memory snapshotting highly fragmented files
 * under jbd2_journal_lock_updates(). If we exceed this limit, fall back to
 * full commit.
 */
#define EXT4_FC_SNAPSHOT_MAX_INODES	1024
#define EXT4_FC_SNAPSHOT_MAX_RANGES	2048

static inline void ext4_fc_set_snap_err(int *snap_err, int err)
{
	if (snap_err && *snap_err == EXT4_FC_SNAP_ERR_NONE)
		*snap_err = err;
}

static void ext4_fc_free_inode_snap(struct inode *inode);

static inline void ext4_fc_reset_inode(struct inode *inode)
{
	struct ext4_inode_info *ei = EXT4_I(inode);

	ei->i_fc_lblk_start = 0;
	ei->i_fc_lblk_len = 0;
}

void ext4_fc_init_inode(struct inode *inode)
{
	struct ext4_inode_info *ei = EXT4_I(inode);

	ext4_fc_reset_inode(inode);
	ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING);
	ext4_clear_inode_state(inode, EXT4_STATE_FC_REQUEUE);
	INIT_LIST_HEAD(&ei->i_fc_list);
	INIT_LIST_HEAD(&ei->i_fc_dilist);
	ei->i_fc_snap = NULL;
}

static bool ext4_fc_disabled(struct super_block *sb)
{
	return (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
		(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY));
}

static bool ext4_fc_eligible(struct super_block *sb)
{
	return !ext4_fc_disabled(sb) &&
		!(ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE));
}

/*
 * Wait for an inode fast-commit state bit to clear while dropping the
 * fast-commit lock around schedule().
 */
static void ext4_fc_wait_inode_state(struct inode *inode, int bit,
				     int *alloc_ctx)
{
	wait_queue_head_t *wq;
	unsigned long *wait_word = ext4_inode_state_wait_word(inode);
	int wait_bit = ext4_inode_state_wait_bit(bit);

	while (ext4_test_inode_state(inode, bit)) {
		DEFINE_WAIT_BIT(wait, wait_word, wait_bit);

		wq = bit_waitqueue(wait_word, wait_bit);
		prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
		if (ext4_test_inode_state(inode, bit)) {
			ext4_fc_unlock(inode->i_sb, *alloc_ctx);
			schedule();
			*alloc_ctx = ext4_fc_lock(inode->i_sb);
		}
		finish_wait(wq, &wait.wq_entry);
	}
}

static inline void ext4_fc_wake_inode_state(struct inode *inode, int bit)
{
	wake_up_bit(ext4_inode_state_wait_word(inode),
		    ext4_inode_state_wait_bit(bit));
}

static void ext4_fc_snap_stats_update_max(atomic64_t *stat, u64 value)
{
	u64 old = atomic64_read(stat);

	while (value > old) {
		u64 prev = atomic64_cmpxchg(stat, old, value);

		if (prev == old)
			break;
		old = prev;
	}
}

/*
 * Remove inode from fast commit list. If the inode is being committed
 * we wait until inode commit is done.
 */
void ext4_fc_del(struct inode *inode)
{
	struct ext4_inode_info *ei = EXT4_I(inode);
	struct ext4_fc_dentry_update *fc_dentry;
	int alloc_ctx;

	if (ext4_fc_disabled(inode->i_sb))
		return;

	alloc_ctx = ext4_fc_lock(inode->i_sb);
	if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) {
		ext4_fc_free_inode_snap(inode);
		ext4_fc_unlock(inode->i_sb, alloc_ctx);
		return;
	}

	/*
	 * Wait for ongoing fast commit to finish. We cannot remove the inode
	 * from fast commit lists while it is being committed. If we wake from
	 * FC_FLUSHING_DATA, re-check FC_COMMITTING before deleting because the
	 * commit thread sets FC_COMMITTING only after clearing FLUSHING_DATA.
	 */
	for (;;) {
		ext4_fc_wait_inode_state(inode, EXT4_STATE_FC_COMMITTING,
					 &alloc_ctx);

		if (!ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA))
			break;

		ext4_fc_wait_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA,
					 &alloc_ctx);
	}

	ext4_fc_free_inode_snap(inode);
	list_del_init(&ei->i_fc_list);

	/*
	 * Since this inode is getting removed, let's also remove all FC dentry
	 * create references, since it is not needed to log it anyways.
	 */
	if (list_empty(&ei->i_fc_dilist)) {
		ext4_fc_unlock(inode->i_sb, alloc_ctx);
		return;
	}

	fc_dentry = list_first_entry(&ei->i_fc_dilist,
				     struct ext4_fc_dentry_update,
				     fcd_dilist);
	WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT);
	list_del_init(&fc_dentry->fcd_list);
	list_del_init(&fc_dentry->fcd_dilist);

	WARN_ON(!list_empty(&ei->i_fc_dilist));
	ext4_fc_unlock(inode->i_sb, alloc_ctx);

	release_dentry_name_snapshot(&fc_dentry->fcd_name);
	kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);
}

/*
 * Mark file system as fast commit ineligible, and record latest
 * ineligible transaction tid. This means until the recorded
 * transaction, commit operation would result in a full jbd2 commit.
 */
void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle)
{
	struct ext4_sb_info *sbi = EXT4_SB(sb);
	tid_t tid;
	bool has_transaction = true;
	bool is_ineligible;
	int alloc_ctx;

	if (ext4_fc_disabled(sb))
		return;

	if (!IS_ERR_OR_NULL(handle))
		tid = handle->h_transaction->t_tid;
	else {
		read_lock(&sbi->s_journal->j_state_lock);
		if (sbi->s_journal->j_running_transaction)
			tid = sbi->s_journal->j_running_transaction->t_tid;
		else
			has_transaction = false;
		read_unlock(&sbi->s_journal->j_state_lock);
	}
	alloc_ctx = ext4_fc_lock(sb);
	is_ineligible = ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
	if (has_transaction && (!is_ineligible || tid_gt(tid, sbi->s_fc_ineligible_tid)))
		sbi->s_fc_ineligible_tid = tid;
	ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
	ext4_fc_unlock(sb, alloc_ctx);
	WARN_ON(reason >= EXT4_FC_REASON_MAX);
	sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
}

/*
 * Generic fast commit tracking function. If this is the first time this we are
 * called after a full commit, we initialize fast commit fields and then call
 * __fc_track_fn() with update = 0. If we have already been called after a full
 * commit, we pass update = 1. Based on that, the track function can determine
 * if it needs to track a field for the first time or if it needs to just
 * update the previously tracked value.
 *
 * If enqueue is set, this function enqueues the inode in fast commit list.
 */
static int ext4_fc_track_template(
	handle_t *handle, struct inode *inode,
	int (*__fc_track_fn)(handle_t *handle, struct inode *, void *, bool),
	void *args, int enqueue)
{
	bool update = false;
	struct ext4_inode_info *ei = EXT4_I(inode);
	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
	tid_t tid = 0;
	int alloc_ctx;
	int ret;

	tid = handle->h_transaction->t_tid;
	spin_lock(&ei->i_fc_lock);
	if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING))
		ext4_set_inode_state(inode, EXT4_STATE_FC_REQUEUE);
	if (tid == ei->i_sync_tid) {
		update = true;
	} else {
		ext4_fc_reset_inode(inode);
		ei->i_sync_tid = tid;
	}
	ret = __fc_track_fn(handle, inode, args, update);
	spin_unlock(&ei->i_fc_lock);
	if (!enqueue)
		return ret;

	alloc_ctx = ext4_fc_lock(inode->i_sb);
	if (list_empty(&EXT4_I(inode)->i_fc_list))
		list_add_tail(&EXT4_I(inode)->i_fc_list,
				(sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
				 sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ?
				&sbi->s_fc_q[FC_Q_STAGING] :
				&sbi->s_fc_q[FC_Q_MAIN]);
	ext4_fc_unlock(inode->i_sb, alloc_ctx);

	return ret;
}

struct __track_dentry_update_args {
	struct dentry *dentry;
	int op;
};

/* __track_fn for directory entry updates. Called with ei->i_fc_lock. */
static int __track_dentry_update(handle_t *handle, struct inode *inode,
				 void *arg, bool update)
{
	struct ext4_fc_dentry_update *node;
	struct ext4_inode_info *ei = EXT4_I(inode);
	struct __track_dentry_update_args *dentry_update =
		(struct __track_dentry_update_args *)arg;
	struct dentry *dentry = dentry_update->dentry;
	struct inode *dir = dentry->d_parent->d_inode;
	struct super_block *sb = inode->i_sb;
	struct ext4_sb_info *sbi = EXT4_SB(sb);
	int alloc_ctx;

	spin_unlock(&ei->i_fc_lock);

	if (IS_ENCRYPTED(dir)) {
		ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_ENCRYPTED_FILENAME,
					handle);
		spin_lock(&ei->i_fc_lock);
		return -EOPNOTSUPP;
	}

	node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS);
	if (!node) {
		ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, handle);
		spin_lock(&ei->i_fc_lock);
		return -ENOMEM;
	}

	node->fcd_op = dentry_update->op;
	node->fcd_parent = dir->i_ino;
	node->fcd_ino = inode->i_ino;
	take_dentry_name_snapshot(&node->fcd_name, dentry);
	INIT_LIST_HEAD(&node->fcd_dilist);
	INIT_LIST_HEAD(&node->fcd_list);
	alloc_ctx = ext4_fc_lock(sb);
	if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
		sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING)
		list_add_tail(&node->fcd_list,
				&sbi->s_fc_dentry_q[FC_Q_STAGING]);
	else
		list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]);

	/*
	 * This helps us keep a track of all fc_dentry updates which is part of
	 * this ext4 inode. So in case the inode is getting unlinked, before
	 * even we get a chance to fsync, we could remove all fc_dentry
	 * references while evicting the inode in ext4_fc_del().
	 * Also with this, we don't need to loop over all the inodes in
	 * sbi->s_fc_q to get the corresponding inode in
	 * ext4_fc_commit_dentry_updates().
	 */
	if (dentry_update->op == EXT4_FC_TAG_CREAT) {
		WARN_ON(!list_empty(&ei->i_fc_dilist));
		list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist);
	}
	ext4_fc_unlock(sb, alloc_ctx);
	spin_lock(&ei->i_fc_lock);

	return 0;
}

void __ext4_fc_track_unlink(handle_t *handle,
		struct inode *inode, struct dentry *dentry)
{
	struct __track_dentry_update_args args;
	int ret;

	args.dentry = dentry;
	args.op = EXT4_FC_TAG_UNLINK;

	ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
					(void *)&args, 0);
	trace_ext4_fc_track_unlink(handle, inode, dentry, ret);
}

void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry)
{
	struct inode *inode = d_inode(dentry);

	if (ext4_fc_eligible(inode->i_sb))
		__ext4_fc_track_unlink(handle, inode, dentry);
}

void __ext4_fc_track_link(handle_t *handle,
	struct inode *inode, struct dentry *dentry)
{
	struct __track_dentry_update_args args;
	int ret;

	args.dentry = dentry;
	args.op = EXT4_FC_TAG_LINK;

	ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
					(void *)&args, 0);
	trace_ext4_fc_track_link(handle, inode, dentry, ret);
}

void ext4_fc_track_link(handle_t *handle, struct inode *inode,
			struct dentry *dentry)
{
	if (ext4_fc_eligible(inode->i_sb))
		__ext4_fc_track_link(handle, inode, dentry);
}

void __ext4_fc_track_create(handle_t *handle, struct inode *inode,
			  struct dentry *dentry)
{
	struct __track_dentry_update_args args;
	int ret;

	args.dentry = dentry;
	args.op = EXT4_FC_TAG_CREAT;

	ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
					(void *)&args, 0);
	trace_ext4_fc_track_create(handle, inode, dentry, ret);
}

void ext4_fc_track_create(handle_t *handle, struct dentry *dentry)
{
	struct inode *inode = d_inode(dentry);

	if (ext4_fc_eligible(inode->i_sb))
		__ext4_fc_track_create(handle, inode, dentry);
}

/* __track_fn for inode tracking */
static int __track_inode(handle_t *handle, struct inode *inode, void *arg,
			 bool update)
{
	if (update)
		return -EEXIST;

	EXT4_I(inode)->i_fc_lblk_len = 0;

	return 0;
}

void ext4_fc_track_inode(handle_t *handle, struct inode *inode)
{
	int ret;

	if (S_ISDIR(inode->i_mode))
		return;

	if (ext4_should_journal_data(inode)) {
		ext4_fc_mark_ineligible(inode->i_sb,
					EXT4_FC_REASON_INODE_JOURNAL_DATA, handle);
		return;
	}

	if (!ext4_fc_eligible(inode->i_sb))
		return;

	/*
	 * Fast commit snapshots inode state at commit time, so there's no need
	 * to wait for EXT4_STATE_FC_COMMITTING here. If the inode is already
	 * on the commit queue, ext4_fc_cleanup() will requeue it for the new
	 * transaction once the current commit finishes.
	 */

	/*
	 * From this point on, this inode will not be committed either
	 * by fast or full commit as long as the handle is open.
	 */
	ret = ext4_fc_track_template(handle, inode, __track_inode, NULL, 1);
	trace_ext4_fc_track_inode(handle, inode, ret);
}

struct __track_range_args {
	ext4_lblk_t start, end;
};

/* __track_fn for tracking data updates */
static int __track_range(handle_t *handle, struct inode *inode, void *arg,
			 bool update)
{
	struct ext4_inode_info *ei = EXT4_I(inode);
	ext4_lblk_t oldstart;
	struct __track_range_args *__arg =
		(struct __track_range_args *)arg;

	if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) {
		ext4_debug("Special inode %llu being modified\n", inode->i_ino);
		return -ECANCELED;
	}

	oldstart = ei->i_fc_lblk_start;

	if (update && ei->i_fc_lblk_len > 0) {
		ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start);
		ei->i_fc_lblk_len =
			max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) -
				ei->i_fc_lblk_start + 1;
	} else {
		ei->i_fc_lblk_start = __arg->start;
		ei->i_fc_lblk_len = __arg->end - __arg->start + 1;
	}

	return 0;
}

void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start,
			 ext4_lblk_t end)
{
	struct __track_range_args args;
	int ret;

	if (S_ISDIR(inode->i_mode))
		return;

	if (!ext4_fc_eligible(inode->i_sb))
		return;

	if (ext4_has_inline_data(inode)) {
		ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR,
					handle);
		return;
	}

	args.start = start;
	args.end = end;

	ret = ext4_fc_track_template(handle, inode,  __track_range, &args, 1);

	trace_ext4_fc_track_range(handle, inode, start, end, ret);
}

static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail)
{
	blk_opf_t write_flags = JBD2_JOURNAL_REQ_FLAGS;
	struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh;

	/* Add REQ_FUA | REQ_PREFLUSH only its tail */
	if (test_opt(sb, BARRIER) && is_tail)
		write_flags |= REQ_FUA | REQ_PREFLUSH;
	lock_buffer(bh);
	set_buffer_dirty(bh);
	set_buffer_uptodate(bh);
	bh_submit(bh, REQ_OP_WRITE | write_flags, bh_end_write);
	EXT4_SB(sb)->s_fc_bh = NULL;
}

/* Ext4 commit path routines */

/*
 * Allocate len bytes on a fast commit buffer.
 *
 * During the commit time this function is used to manage fast commit
 * block space. We don't split a fast commit log onto different
 * blocks. So this function makes sure that if there's not enough space
 * on the current block, the remaining space in the current block is
 * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case,
 * new block is from jbd2 and CRC is updated to reflect the padding
 * we added.
 */
static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc)
{
	struct ext4_fc_tl tl;
	struct ext4_sb_info *sbi = EXT4_SB(sb);
	struct buffer_head *bh;
	int bsize = sbi->s_journal->j_blocksize;
	int ret, off = sbi->s_fc_bytes % bsize;
	int remaining;
	u8 *dst;

	/*
	 * If 'len' is too long to fit in any block alongside a PAD tlv, then we
	 * cannot fulfill the request.
	 */
	if (len > bsize - EXT4_FC_TAG_BASE_LEN)
		return NULL;

	if (!sbi->s_fc_bh) {
		ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
		if (ret)
			return NULL;
		sbi->s_fc_bh = bh;
	}
	dst = sbi->s_fc_bh->b_data + off;

	/*
	 * Allocate the bytes in the current block if we can do so while still
	 * leaving enough space for a PAD tlv.
	 */
	remaining = bsize - EXT4_FC_TAG_BASE_LEN - off;
	if (len <= remaining) {
		sbi->s_fc_bytes += len;
		return dst;
	}

	/*
	 * Else, terminate the current block with a PAD tlv, then allocate a new
	 * block and allocate the bytes at the start of that new block.
	 */

	tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD);
	tl.fc_len = cpu_to_le16(remaining);
	memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
	memset(dst + EXT4_FC_TAG_BASE_LEN, 0, remaining);
	*crc = ext4_chksum(*crc, sbi->s_fc_bh->b_data, bsize);

	ext4_fc_submit_bh(sb, false);

	ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
	if (ret)
		return NULL;
	sbi->s_fc_bh = bh;
	sbi->s_fc_bytes += bsize - off + len;
	return sbi->s_fc_bh->b_data;
}

/*
 * Complete a fast commit by writing tail tag.
 *
 * Writing tail tag marks the end of a fast commit. In order to guarantee
 * atomicity, after writing tail tag, even if there's space remaining
 * in the block, next commit shouldn't use it. That's why tail tag
 * has the length as that of the remaining space on the block.
 */
static int ext4_fc_write_tail(struct super_block *sb, u32 crc)
{
	struct ext4_sb_info *sbi = EXT4_SB(sb);
	struct ext4_fc_tl tl;
	struct ext4_fc_tail tail;
	int off, bsize = sbi->s_journal->j_blocksize;
	u8 *dst;

	/*
	 * ext4_fc_reserve_space takes care of allocating an extra block if
	 * there's no enough space on this block for accommodating this tail.
	 */
	dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), &crc);
	if (!dst)
		return -ENOSPC;

	off = sbi->s_fc_bytes % bsize;

	tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL);
	tl.fc_len = cpu_to_le16(bsize - off + sizeof(struct ext4_fc_tail));
	sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize);

	memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
	dst += EXT4_FC_TAG_BASE_LEN;
	tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid);
	memcpy(dst, &tail.fc_tid, sizeof(tail.fc_tid));
	dst += sizeof(tail.fc_tid);
	crc = ext4_chksum(crc, sbi->s_fc_bh->b_data,
			  dst - (u8 *)sbi->s_fc_bh->b_data);
	tail.fc_crc = cpu_to_le32(crc);
	memcpy(dst, &tail.fc_crc, sizeof(tail.fc_crc));
	dst += sizeof(tail.fc_crc);
	memset(dst, 0, bsize - off); /* Don't leak uninitialized memory. */

	ext4_fc_submit_bh(sb, true);

	return 0;
}

/*
 * Adds tag, length, value and updates CRC. Returns true if tlv was added.
 * Returns false if there's not enough space.
 */
static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val,
			   u32 *crc)
{
	struct ext4_fc_tl tl;
	u8 *dst;

	dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc);
	if (!dst)
		return false;

	tl.fc_tag = cpu_to_le16(tag);
	tl.fc_len = cpu_to_le16(len);

	memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
	memcpy(dst + EXT4_FC_TAG_BASE_LEN, val, len);

	return true;
}

/* Same as above, but adds dentry tlv. */
static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc,
				   struct ext4_fc_dentry_update *fc_dentry)
{
	struct ext4_fc_dentry_info fcd;
	struct ext4_fc_tl tl;
	int dlen = fc_dentry->fcd_name.name.len;
	u8 *dst = ext4_fc_reserve_space(sb,
			EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc);

	if (!dst)
		return false;

	fcd.fc_parent_ino = cpu_to_le32(fc_dentry->fcd_parent);
	fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino);
	tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op);
	tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen);
	memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
	dst += EXT4_FC_TAG_BASE_LEN;
	memcpy(dst, &fcd, sizeof(fcd));
	dst += sizeof(fcd);
	memcpy(dst, fc_dentry->fcd_name.name.name, dlen);

	return true;
}

struct ext4_fc_range {
	struct list_head list;
	u16 tag;
	ext4_lblk_t lblk;
	ext4_lblk_t len;
	ext4_fsblk_t pblk;
	bool unwritten;
};

struct ext4_fc_inode_snap {
	struct list_head data_list;
	unsigned int inode_len;
	u8 inode_buf[];
};

/*
 * Writes inode in the fast commit space under TLV with tag @tag.
 * Returns 0 on success, error on failure.
 */
static int ext4_fc_write_inode(struct inode *inode, u32 *crc)
{
	struct ext4_inode_info *ei = EXT4_I(inode);
	struct ext4_fc_inode_snap *snap = ei->i_fc_snap;
	struct ext4_fc_snap_stats *stats =
		&EXT4_SB(inode->i_sb)->s_fc_snap_stats;
	struct ext4_fc_inode fc_inode;
	struct ext4_fc_tl tl;
	u8 *dst;
	u8 *src;
	int inode_len;
	int ret;

	if (!snap) {
		atomic64_inc(&stats->snap_fail_no_snap);
		return -ECANCELED;
	}

	src = snap->inode_buf;
	inode_len = snap->inode_len;
	if (!src || inode_len == 0) {
		atomic64_inc(&stats->snap_fail_no_snap);
		return -ECANCELED;
	}

	fc_inode.fc_ino = cpu_to_le32(inode->i_ino);
	tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE);
	tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino));

	ret = -ECANCELED;
	dst = ext4_fc_reserve_space(inode->i_sb,
		EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc);
	if (!dst)
		goto err;

	memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
	dst += EXT4_FC_TAG_BASE_LEN;
	memcpy(dst, &fc_inode, sizeof(fc_inode));
	dst += sizeof(fc_inode);
	memcpy(dst, src, inode_len);
	ret = 0;
err:
	return ret;
}

/*
 * Writes updated data ranges for the inode in question. Updates CRC.
 * Returns 0 on success, error otherwise.
 */
static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc)
{
	struct ext4_inode_info *ei = EXT4_I(inode);
	struct ext4_fc_inode_snap *snap = ei->i_fc_snap;
	struct ext4_fc_snap_stats *stats =
		&EXT4_SB(inode->i_sb)->s_fc_snap_stats;
	struct ext4_fc_add_range fc_ext;
	struct ext4_fc_del_range lrange;
	struct ext4_extent *ex;
	struct ext4_fc_range *range;

	if (!snap) {
		atomic64_inc(&stats->snap_fail_no_snap);
		return -ECANCELED;
	}

	list_for_each_entry(range, &snap->data_list, list) {
		if (range->tag == EXT4_FC_TAG_DEL_RANGE) {
			lrange.fc_ino = cpu_to_le32(inode->i_ino);
			lrange.fc_lblk = cpu_to_le32(range->lblk);
			lrange.fc_len = cpu_to_le32(range->len);
			if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE,
					     sizeof(lrange), (u8 *)&lrange, crc))
				return -ENOSPC;
			continue;
		}

		fc_ext.fc_ino = cpu_to_le32(inode->i_ino);
		ex = (struct ext4_extent *)&fc_ext.fc_ex;
		ex->ee_block = cpu_to_le32(range->lblk);
		ex->ee_len = cpu_to_le16(range->len);
		ext4_ext_store_pblock(ex, range->pblk);
		if (range->unwritten)
			ext4_ext_mark_unwritten(ex);
		else
			ext4_ext_mark_initialized(ex);

		if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE,
				     sizeof(fc_ext), (u8 *)&fc_ext, crc))
			return -ENOSPC;
	}

	return 0;
}

static void ext4_fc_free_ranges(struct list_head *head)
{
	struct ext4_fc_range *range, *range_n;

	list_for_each_entry_safe(range, range_n, head, list) {
		list_del(&range->list);
		kmem_cache_free(ext4_fc_range_cachep, range);
	}
}

static void ext4_fc_free_inode_snap(struct inode *inode)
{
	struct ext4_inode_info *ei = EXT4_I(inode);
	struct ext4_fc_inode_snap *snap = ei->i_fc_snap;

	if (!snap)
		return;

	ext4_fc_free_ranges(&snap->data_list);
	kfree(snap);
	ei->i_fc_snap = NULL;
}

static int ext4_fc_snapshot_inode_data(struct inode *inode,
				       struct list_head *ranges,
				       unsigned int nr_ranges_total,
				       unsigned int *nr_rangesp,
				       int *snap_err)
{
	struct ext4_inode_info *ei = EXT4_I(inode);
	struct ext4_fc_snap_stats *stats =
		&EXT4_SB(inode->i_sb)->s_fc_snap_stats;
	ext4_lblk_t start_lblk, end_lblk, cur_lblk;
	unsigned int nr_ranges = 0;

	spin_lock(&ei->i_fc_lock);
	if (ei->i_fc_lblk_len == 0) {
		spin_unlock(&ei->i_fc_lock);
		if (nr_rangesp)
			*nr_rangesp = 0;
		return 0;
	}
	start_lblk = ei->i_fc_lblk_start;
	end_lblk = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1;