ext4: refactor ext4_file_write_iter

commit aa9714d("Start with shared i_rwsem in case of DIO instead of exclusive")
allowed shared inode lock using in directio overwite case, that can improve
a lot performance for mixed read/write directio workload like database.

This feature was done from upstream when ext4 directio was already switched
to iomap infrastructure, but that had not been supported until uek7, one way
to do it is to backport the iomap feature to older uek, but that introduced
a lot of patches and has high risk especially the change is in io critical
path, it could introuduce corruption issue, so we decides to manually
refactor the directio path to resolve the dependency above patch requires.

For that, we need refactoring ext4_file_write_iter and split out two helpers,
one is ext4_buffered_write_iter, and another is ext4_dio_write_iter.
For the case that directio fallen into buffer io, dio helper will invoke
buffer io helper. That way we can release inode lock dio helper taken before
fallen into buffer io helper.

No function changes.

Orabug: 34405736

Signed-off-by: Junxiao Bi <[email protected]>
Reviewed-by: Darrick J. Wong <[email protected]>
Signed-off-by: Brian Maly <[email protected]>

Author: biger410

fs/ext4/file.c

@@ -216,22 +216,41 @@ ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)
 }
 #endif
 
-static ssize_t
-ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
+static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,
+						struct iov_iter *from)
 {
 	struct inode *inode = file_inode(iocb->ki_filp);
-	int o_direct = iocb->ki_flags & IOCB_DIRECT;
-	int unaligned_aio = 0;
-	int overwrite = 0;
 	ssize_t ret;
 
-	if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
-		return -EIO;
+	if (!inode_trylock(inode)) {
+		if (iocb->ki_flags & IOCB_NOWAIT)
+			return -EAGAIN;
+		inode_lock(inode);
+	}
 
-#ifdef CONFIG_FS_DAX
-	if (IS_DAX(inode))
-		return ext4_dax_write_iter(iocb, from);
-#endif
+	ret = ext4_write_checks(iocb, from);
+	if (ret <= 0)
+		goto out;
+
+	ret = __generic_file_write_iter(iocb, from);
+	inode_unlock(inode);
+
+	if (ret > 0)
+		ret = generic_write_sync(iocb, ret);
+
+	return ret;
+
+out:
+	inode_unlock(inode);
+	return ret;
+}
+
+static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)
+{
+	struct inode *inode = file_inode(iocb->ki_filp);
+	int unaligned_aio = 0;
+	int overwrite = 0;
+	ssize_t ret;
 
 	if (!inode_trylock(inode)) {
 		if (iocb->ki_flags & IOCB_NOWAIT)
@@ -248,7 +267,7 @@ ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	 * of partial blocks of two competing unaligned AIOs can result in data
 	 * corruption.
 	 */
-	if (o_direct && ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) &&
+	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) &&
 	    !is_sync_kiocb(iocb) &&
 	    ext4_unaligned_aio(inode, from, iocb->ki_pos)) {
 		unaligned_aio = 1;
@@ -257,7 +276,7 @@ ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 
 	iocb->private = &overwrite;
 	/* Check whether we do a DIO overwrite or not */
-	if (o_direct && !unaligned_aio) {
+	if (!unaligned_aio) {
 		if (ext4_overwrite_io(inode, iocb->ki_pos, iov_iter_count(from))) {
 			if (ext4_should_dioread_nolock(inode))
 				overwrite = 1;
@@ -287,6 +306,25 @@ ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	return ret;
 }
 
+static ssize_t
+ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
+{
+	struct inode *inode = file_inode(iocb->ki_filp);
+
+	if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
+		return -EIO;
+
+#ifdef CONFIG_FS_DAX
+	if (IS_DAX(inode))
+		return ext4_dax_write_iter(iocb, from);
+#endif
+
+	if (iocb->ki_flags & IOCB_DIRECT)
+		return ext4_dio_write_iter(iocb, from);
+
+	return ext4_buffered_write_iter(iocb, from);
+}
+
 #ifdef CONFIG_FS_DAX
 static vm_fault_t ext4_dax_huge_fault(struct vm_fault *vmf,
 		enum page_entry_size pe_size)