From 7f1a1fbf850f6b73b5c9c82365f01029fb250a1c Mon Sep 17 00:00:00 2001 From: Theodore Ts'o Date: Fri, 24 Sep 2010 10:02:25 -0400 Subject: ext2fs: Add Direct I/O support to the ext2fs library This adds the basic support for Direct I/O to unix_io.c, and adds a new flag EXT_FLAG_DIRECT_IO which can be passed to ext2fs_open() or ext2fs_open2() to request Direct I/O support. Note that device mapper devices in Linux don't support Direct I/O, and in some circumstances using Direct I/O can actually make performance *worse*! Signed-off-by: "Theodore Ts'o" --- lib/ext2fs/closefs.c | 3 + lib/ext2fs/ext2_io.h | 1 + lib/ext2fs/ext2fs.h | 1 + lib/ext2fs/openfs.c | 2 + lib/ext2fs/unix_io.c | 220 ++++++++++++++++++++++++++++++++------------------- 5 files changed, 144 insertions(+), 83 deletions(-) diff --git a/lib/ext2fs/closefs.c b/lib/ext2fs/closefs.c index db6b8a6a..99430d7b 100644 --- a/lib/ext2fs/closefs.c +++ b/lib/ext2fs/closefs.c @@ -145,6 +145,7 @@ static errcode_t write_primary_superblock(ext2_filsys fs, errcode_t retval; if (!fs->io->manager->write_byte || !fs->orig_super) { + fallback: io_channel_set_blksize(fs->io, SUPERBLOCK_OFFSET); retval = io_channel_write_blk(fs->io, 1, -SUPERBLOCK_SIZE, super); @@ -170,6 +171,8 @@ static errcode_t write_primary_superblock(ext2_filsys fs, retval = io_channel_write_byte(fs->io, SUPERBLOCK_OFFSET + (2 * write_idx), size, new_super + write_idx); + if (retval == EXT2_ET_UNIMPLEMENTED) + goto fallback; if (retval) return retval; } diff --git a/lib/ext2fs/ext2_io.h b/lib/ext2fs/ext2_io.h index 5bebbfb3..b917559c 100644 --- a/lib/ext2fs/ext2_io.h +++ b/lib/ext2fs/ext2_io.h @@ -88,6 +88,7 @@ struct struct_io_manager { #define IO_FLAG_RW 0x0001 #define IO_FLAG_EXCLUSIVE 0x0002 +#define IO_FLAG_DIRECT_IO 0x0004 /* * Convenience functions.... diff --git a/lib/ext2fs/ext2fs.h b/lib/ext2fs/ext2fs.h index bc0d7cef..8dac89a9 100644 --- a/lib/ext2fs/ext2fs.h +++ b/lib/ext2fs/ext2fs.h @@ -173,6 +173,7 @@ typedef struct ext2_file *ext2_file_t; #define EXT2_FLAG_EXCLUSIVE 0x4000 #define EXT2_FLAG_SOFTSUPP_FEATURES 0x8000 #define EXT2_FLAG_NOFREE_ON_ERROR 0x10000 +#define EXT2_FLAG_DIRECT_IO 0x20000 /* * Special flag in the ext2 inode i_flag field that means that this is diff --git a/lib/ext2fs/openfs.c b/lib/ext2fs/openfs.c index a6b08c35..52157d85 100644 --- a/lib/ext2fs/openfs.c +++ b/lib/ext2fs/openfs.c @@ -121,6 +121,8 @@ errcode_t ext2fs_open2(const char *name, const char *io_options, io_flags |= IO_FLAG_RW; if (flags & EXT2_FLAG_EXCLUSIVE) io_flags |= IO_FLAG_EXCLUSIVE; + if (flags & EXT2_FLAG_DIRECT_IO) + io_flags |= IO_FLAG_DIRECT_IO; retval = manager->open(fs->device_name, io_flags, &fs->io); if (retval) goto cleanup; diff --git a/lib/ext2fs/unix_io.c b/lib/ext2fs/unix_io.c index 78a1d1cb..1df1fddc 100644 --- a/lib/ext2fs/unix_io.c +++ b/lib/ext2fs/unix_io.c @@ -17,6 +17,7 @@ #define _LARGEFILE_SOURCE #define _LARGEFILE64_SOURCE +#define _GNU_SOURCE #include #include @@ -47,10 +48,16 @@ #include #endif -#if defined(__linux__) && defined(_IO) && !defined(BLKGETSIZE) +#if defined(__linux__) && defined(_IO) && !defined(BLKROGET) #define BLKROGET _IO(0x12, 94) /* Get read-only status (0 = read_write). */ #endif +#if defined(__linux__) && defined(_IO) && !defined(BLKSSZGET) +#define BLKSSZGET _IO(0x12,104)/* get block device sector size */ +#endif + +#undef ALIGN_DEBUG + #include "ext2_fs.h" #include "ext2fs.h" @@ -77,12 +84,17 @@ struct unix_private_data { int magic; int dev; int flags; + int align; int access_time; ext2_loff_t offset; struct unix_cache cache[CACHE_SIZE]; + void *bounce; struct struct_io_stats io_stats; }; +#define IS_ALIGNED(n, align) ((((unsigned long) n) & \ + ((unsigned long) ((align)-1))) == 0) + static errcode_t unix_open(const char *name, int flags, io_channel *channel); static errcode_t unix_close(io_channel channel); static errcode_t unix_set_blksize(io_channel channel, int blksize); @@ -104,14 +116,6 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block, static errcode_t unix_write_blk64(io_channel channel, unsigned long long block, int count, const void *data); -/* __FreeBSD_kernel__ is defined by GNU/kFreeBSD - the FreeBSD kernel - * does not know buffered block devices - everything is raw. */ -#if defined(__CYGWIN__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) -#define NEED_BOUNCE_BUFFER -#else -#undef NEED_BOUNCE_BUFFER -#endif - static struct struct_io_manager struct_unix_manager = { EXT2_ET_MAGIC_IO_MANAGER, "Unix I/O Manager", @@ -121,11 +125,7 @@ static struct struct_io_manager struct_unix_manager = { unix_read_blk, unix_write_blk, unix_flush, -#ifdef NEED_BOUNCE_BUFFER - 0, -#else unix_write_byte, -#endif unix_set_option, unix_get_stats, unix_read_blk64, @@ -153,7 +153,6 @@ static errcode_t unix_get_stats(io_channel channel, io_stats *stats) /* * Here are the raw I/O functions */ -#ifndef NEED_BOUNCE_BUFFER static errcode_t raw_read_blk(io_channel channel, struct unix_private_data *data, unsigned long long block, @@ -171,77 +170,48 @@ static errcode_t raw_read_blk(io_channel channel, retval = errno ? errno : EXT2_ET_LLSEEK_FAILED; goto error_out; } - actual = read(data->dev, buf, size); - if (actual != size) { - if (actual < 0) - actual = 0; - retval = EXT2_ET_SHORT_READ; - goto error_out; + if ((data->align == 0) || + ((IS_ALIGNED(buf, data->align)) && IS_ALIGNED(size, data->align))) { + actual = read(data->dev, buf, size); + if (actual != size) { + short_read: + if (actual < 0) + actual = 0; + retval = EXT2_ET_SHORT_READ; + goto error_out; + } + return 0; } - return 0; - -error_out: - memset((char *) buf+actual, 0, size-actual); - if (channel->read_error) - retval = (channel->read_error)(channel, block, count, buf, - size, actual, retval); - return retval; -} -#else /* NEED_BOUNCE_BUFFER */ -/* - * Windows and FreeBSD block devices only allow sector alignment IO in offset and size - */ -static errcode_t raw_read_blk(io_channel channel, - struct unix_private_data *data, - unsigned long block, - int count, void *buf) -{ - errcode_t retval; - size_t size, alignsize, fragment; - ext2_loff_t location; - int total = 0, actual; -#define BLOCKALIGN 512 - char sector[BLOCKALIGN]; - size = (count < 0) ? -count : count * channel->block_size; - data->io_stats.bytes_read += size; - location = ((ext2_loff_t) block * channel->block_size) + data->offset; -#ifdef DEBUG - printf("count=%d, size=%d, block=%lu, blk_size=%d, location=%llx\n", - count, size, block, channel->block_size, (long long)location); +#ifdef ALIGN_DEBUG + printf("raw_read_blk: O_DIRECT fallback: %p %lu\n", buf, + (unsigned long) size); #endif - if (ext2fs_llseek(data->dev, location, SEEK_SET) != location) { - retval = errno ? errno : EXT2_ET_LLSEEK_FAILED; - goto error_out; - } - fragment = size % BLOCKALIGN; - alignsize = size - fragment; - if (alignsize) { - actual = read(data->dev, buf, alignsize); - if (actual != alignsize) - goto short_read; - } - if (fragment) { - actual = read(data->dev, sector, BLOCKALIGN); - if (actual != BLOCKALIGN) + + /* + * The buffer or size which we're trying to read isn't aligned + * to the O_DIRECT rules, so we need to do this the hard way... + */ + while (size > 0) { + actual = read(data->dev, data->bounce, channel->block_size); + if (actual != channel->block_size) goto short_read; - memcpy(buf+alignsize, sector, fragment); + actual = size; + if (size > channel->block_size) + actual = channel->block_size; + memcpy(buf, data->bounce, actual); + size -= actual; + buf += actual; } return 0; -short_read: - if (actual>0) - total += actual; - retval = EXT2_ET_SHORT_READ; - error_out: - memset((char *) buf+total, 0, size-actual); + memset((char *) buf+actual, 0, size-actual); if (channel->read_error) retval = (channel->read_error)(channel, block, count, buf, size, actual, retval); return retval; } -#endif static errcode_t raw_write_blk(io_channel channel, struct unix_private_data *data, @@ -269,10 +239,43 @@ static errcode_t raw_write_blk(io_channel channel, goto error_out; } - actual = write(data->dev, buf, size); - if (actual != size) { - retval = EXT2_ET_SHORT_WRITE; - goto error_out; + if ((data->align == 0) || + ((IS_ALIGNED(buf, data->align)) && IS_ALIGNED(size, data->align))) { + actual = write(data->dev, buf, size); + if (actual != size) { + short_write: + retval = EXT2_ET_SHORT_WRITE; + goto error_out; + } + return 0; + } + +#ifdef ALIGN_DEBUG + printf("raw_write_blk: O_DIRECT fallback: %p %lu\n", buf, + (unsigned long) size); +#endif + /* + * The buffer or size which we're trying to write isn't aligned + * to the O_DIRECT rules, so we need to do this the hard way... + */ + while (size > 0) { + if (size < channel->block_size) { + actual = read(data->dev, data->bounce, + channel->block_size); + if (actual != channel->block_size) { + retval = EXT2_ET_SHORT_READ; + goto error_out; + } + } + actual = size; + if (size > channel->block_size) + actual = channel->block_size; + memcpy(data->bounce, buf, actual); + actual = write(data->dev, data->bounce, channel->block_size); + if (actual != channel->block_size) + goto short_write; + size -= actual; + buf += actual; } return 0; @@ -304,11 +307,18 @@ static errcode_t alloc_cache(io_channel channel, cache->in_use = 0; if (cache->buf) ext2fs_free_mem(&cache->buf); - if ((retval = ext2fs_get_mem(channel->block_size, - &cache->buf))) + retval = ext2fs_get_memalign(channel->block_size, + data->align, &cache->buf); + if (retval) return retval; } - return 0; + if (data->align) { + if (data->bounce) + ext2fs_free_mem(&data->bounce); + retval = ext2fs_get_memalign(channel->block_size, data->align, + &data->bounce); + } + return retval; } /* Free the cache buffers */ @@ -325,8 +335,9 @@ static void free_cache(struct unix_private_data *data) cache->in_use = 0; if (cache->buf) ext2fs_free_mem(&cache->buf); - cache->buf = 0; } + if (data->bounce) + ext2fs_free_mem(&data->bounce); } #ifndef NO_IO_CACHE @@ -449,12 +460,13 @@ static errcode_t unix_open(const char *name, int flags, io_channel *channel) data->magic = EXT2_ET_MAGIC_UNIX_IO_CHANNEL; data->io_stats.num_fields = 2; - if ((retval = alloc_cache(io, data))) - goto cleanup; - open_flags = (flags & IO_FLAG_RW) ? O_RDWR : O_RDONLY; if (flags & IO_FLAG_EXCLUSIVE) open_flags |= O_EXCL; + if (flags & IO_FLAG_DIRECT_IO) + open_flags |= O_DIRECT; + data->flags = flags; + #ifdef HAVE_OPEN64 data->dev = open64(io->name, open_flags); #else @@ -465,6 +477,25 @@ static errcode_t unix_open(const char *name, int flags, io_channel *channel) goto cleanup; } +#ifdef BLKSSZGET + if (flags & IO_FLAG_DIRECT_IO) { + if (ioctl(data->dev, BLKSSZGET, &data->align) != 0) + data->align = io->block_size; + } +#endif + +#if defined(__CYGWIN__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) + /* + * Some operating systems require that the buffers be aligned, + * regardless of O_DIRECT + */ + data->align = 512; +#endif + + + if ((retval = alloc_cache(io, data))) + goto cleanup; + #ifdef BLKROGET if (flags & IO_FLAG_RW) { int error; @@ -617,6 +648,22 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block, cp += channel->block_size; continue; } + if (count == 1) { + /* + * Special case where we read directly into the + * cache buffer; important in the O_DIRECT case + */ + cache = reuse[0]; + reuse_cache(channel, data, cache, block); + if ((retval = raw_read_blk(channel, data, block, 1, + cache->buf))) { + cache->in_use = 0; + return retval; + } + memcpy(cp, cache->buf, channel->block_size); + return 0; + } + /* * Find the number of uncached blocks so we can do a * single read request @@ -718,6 +765,13 @@ static errcode_t unix_write_byte(io_channel channel, unsigned long offset, data = (struct unix_private_data *) channel->private_data; EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_UNIX_IO_CHANNEL); + if (data->align != 0) { +#ifdef ALIGN_DEBUG + printf("unix_write_byte: O_DIRECT fallback\n"); +#endif + return EXT2_ET_UNIMPLEMENTED; + } + #ifndef NO_IO_CACHE /* * Flush out the cache completely -- cgit v1.2.3