diff -urN /md0/kernels/2.4/v2.4.18-pre3/AIO-NOTES linux.aio/AIO-NOTES --- /md0/kernels/2.4/v2.4.18-pre3/AIO-NOTES Wed Dec 31 19:00:00 1969 +++ linux.aio/AIO-NOTES Wed Jan 23 01:41:06 2002 @@ -0,0 +1,11 @@ +20020102 + - ABI adjustment to io_getevents: the prototype is now + long io_getevents(aio_context_t ctx_id, + long nr, + struct io_event *events, + struct timespec *timeout) + which does not affect binary compatibility on x86, only + for 64 bit machines. + + + diff -urN /md0/kernels/2.4/v2.4.18-pre3/MAINTAINERS linux.aio/MAINTAINERS --- /md0/kernels/2.4/v2.4.18-pre3/MAINTAINERS Mon Jan 14 21:05:57 2002 +++ linux.aio/MAINTAINERS Wed Jan 23 01:41:06 2002 @@ -220,6 +220,12 @@ L: linux-net@vger.kernel.org S: Maintained +ASYNC IO +P: Benjamin LaHaise +M: bcrl@redhat.com +L: linux-aio@kvack.org +S: Maintained + AX.25 NETWORK LAYER P: Matthias Welwarsky M: dg2fef@afthd.tu-darmstadt.de diff -urN /md0/kernels/2.4/v2.4.18-pre3/Makefile linux.aio/Makefile --- /md0/kernels/2.4/v2.4.18-pre3/Makefile Mon Jan 14 21:05:57 2002 +++ linux.aio/Makefile Wed Jan 23 01:41:06 2002 @@ -226,7 +226,7 @@ drivers/sound/pndsperm.c \ drivers/sound/pndspini.c \ drivers/atm/fore200e_*_fw.c drivers/atm/.fore200e_*.fw \ - .version .config* config.in config.old \ + .uniquebytes .version .config* config.in config.old \ scripts/tkparse scripts/kconfig.tk scripts/kconfig.tmp \ scripts/lxdialog/*.o scripts/lxdialog/lxdialog \ .menuconfig.log \ @@ -268,6 +268,7 @@ --end-group \ -o vmlinux $(NM) vmlinux | grep -v '\(compiled\)\|\(\.o$$\)\|\( [aUw] \)\|\(\.\.ng$$\)\|\(LASH[RL]DI\)' | sort > System.map + @$(MAKE) -C ulib symlinks: rm -f include/asm @@ -305,6 +306,8 @@ newversion: . scripts/mkversion > .tmpversion @mv -f .tmpversion .version + rm -f .uniquebytes + dd if=/dev/urandom of=.uniquebytes bs=1 count=16 include/linux/compile.h: $(CONFIGURATION) include/linux/version.h newversion @echo -n \#define UTS_VERSION \"\#`cat .version` > .ver @@ -328,6 +331,9 @@ @echo \#define UTS_RELEASE \"$(KERNELRELEASE)\" > .ver @echo \#define LINUX_VERSION_CODE `expr $(VERSION) \\* 65536 + $(PATCHLEVEL) \\* 256 + $(SUBLEVEL)` >> .ver @echo '#define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c))' >>.ver + @echo -n \#"define LINUX_UNIQUE_BYTES " >>.ver + @hexdump -v -e '1/1 "0x%02x, "' .uniquebytes | sed -e 's/, $$//g' >>.ver + @echo "" >>.ver @mv -f .ver $@ comma := , @@ -420,6 +426,7 @@ rm -f $(CLEAN_FILES) rm -rf $(CLEAN_DIRS) $(MAKE) -C Documentation/DocBook clean + $(MAKE) -C ulib clean mrproper: clean archmrproper find . \( -size 0 -o -name .depend \) -type f -print | xargs rm -f diff -urN /md0/kernels/2.4/v2.4.18-pre3/arch/i386/Makefile linux.aio/arch/i386/Makefile --- /md0/kernels/2.4/v2.4.18-pre3/arch/i386/Makefile Thu May 3 11:22:07 2001 +++ linux.aio/arch/i386/Makefile Wed Jan 23 01:41:06 2002 @@ -98,7 +98,7 @@ DRIVERS += arch/i386/math-emu/math.o endif -arch/i386/kernel: dummy +arch/i386/kernel: dummy include/linux/compile.h $(MAKE) linuxsubdirs SUBDIRS=arch/i386/kernel arch/i386/mm: dummy diff -urN /md0/kernels/2.4/v2.4.18-pre3/arch/i386/kernel/Makefile linux.aio/arch/i386/kernel/Makefile --- /md0/kernels/2.4/v2.4.18-pre3/arch/i386/kernel/Makefile Mon Nov 26 23:43:07 2001 +++ linux.aio/arch/i386/kernel/Makefile Wed Jan 23 01:41:06 2002 @@ -20,6 +20,9 @@ ptrace.o i8259.o ioport.o ldt.o setup.o time.o sys_i386.o \ pci-dma.o i386_ksyms.o i387.o bluesmoke.o dmi_scan.o +obj-y += vsysdata.o vunique.o dynamic_syscall.o + +vunique.o: $(TOPDIR)/include/linux/compile.h ifdef CONFIG_PCI obj-y += pci-i386.o diff -urN /md0/kernels/2.4/v2.4.18-pre3/arch/i386/kernel/dynamic_syscall.c linux.aio/arch/i386/kernel/dynamic_syscall.c --- /md0/kernels/2.4/v2.4.18-pre3/arch/i386/kernel/dynamic_syscall.c Wed Dec 31 19:00:00 1969 +++ linux.aio/arch/i386/kernel/dynamic_syscall.c Wed Jan 23 01:41:06 2002 @@ -0,0 +1,69 @@ +/* arch/i386/kernel/dynamic_syscall.c + * Entry code for dynamic syscalls on i386. + */ +#include +#include +#include +#include +#include +#include + +struct dummy_args { + long data[8]; +}; + +extern struct vsyscall_entry { + long eip; + long (*call)(struct dummy_args args); +} vsyscall_list_begin, vsyscall_list_end; + +long sys_dynamic_syscall(struct pt_regs regs) __attribute__((regparm(0))); + +long sys_dynamic_syscall(struct pt_regs regs) +{ + struct dummy_args dummy_args; + struct vsyscall_entry *ent = (void *)regs.edx; + void *args = (void *)regs.ecx; + long ret; + + pr_debug("ent = %p args = %p\n", ent, args); + pr_debug("eip = 0x%08lx\n", regs.eip); + + if (unlikely(!current->mm->vsys_mapped)) + goto err; + + /* The pointer must be aligned in the table. */ + if (unlikely((long)ent & (sizeof(*ent) - 1))) { + pr_debug("unaligned\n"); + goto err; + } + + /* Bounds checking... */ + if (unlikely(ent < &vsyscall_list_begin) || + unlikely(ent >= &vsyscall_list_end)) { + pr_debug("out of range %p <= %p < %p\n", + &vsyscall_list_begin, ent, + &vsyscall_list_end); + goto err; + } + /* The entry should be valid now. Verify that the caller's eip + * is correct. + */ + if (unlikely(ent->eip != regs.eip)) { + pr_debug("eip mismatch (0x%lx vs 0x%lx)\n", ent->eip, regs.eip); + goto err; + } + + pr_debug("ent->call = %p\n", ent->call); + + if (unlikely(copy_from_user(&dummy_args, args, sizeof(dummy_args)))) + return -EFAULT; + + ret = ent->call(dummy_args); + + pr_debug("ret = 0x%08lx\n", ret); + + return ret; +err: + return -ENOSYS; +} diff -urN /md0/kernels/2.4/v2.4.18-pre3/arch/i386/kernel/entry.S linux.aio/arch/i386/kernel/entry.S --- /md0/kernels/2.4/v2.4.18-pre3/arch/i386/kernel/entry.S Mon Nov 12 17:49:47 2001 +++ linux.aio/arch/i386/kernel/entry.S Wed Jan 23 01:41:06 2002 @@ -45,6 +45,7 @@ #include #include #include +#include EBX = 0x00 ECX = 0x04 @@ -623,6 +624,11 @@ .long SYMBOL_NAME(sys_gettid) .long SYMBOL_NAME(sys_readahead) /* 225 */ + .rept __NR_sys_dynamic_syscall-(.-sys_call_table)/4 + .long SYMBOL_NAME(sys_ni_syscall) + .endr + .long SYMBOL_NAME(sys_dynamic_syscall) + .rept NR_syscalls-(.-sys_call_table)/4 .long SYMBOL_NAME(sys_ni_syscall) .endr diff -urN /md0/kernels/2.4/v2.4.18-pre3/arch/i386/kernel/vsysdata.c linux.aio/arch/i386/kernel/vsysdata.c --- /md0/kernels/2.4/v2.4.18-pre3/arch/i386/kernel/vsysdata.c Wed Dec 31 19:00:00 1969 +++ linux.aio/arch/i386/kernel/vsysdata.c Wed Jan 23 01:41:06 2002 @@ -0,0 +1,11 @@ +/* vsysdata.c - declarations for variables shared with the kernel + * + * Items placed in .data.vsyscall have a kernel virtual address + * and are read/write from kernel space only. The copy placed + * in .vsyscall_data are linked at a userspace address and are + * read only accessible from userland. + */ +#include + +union vsys_union user_vsys_cpudata[256] __attribute__((section(".vsyscall_data"))); +//asm(".globl vsys_cpudata ; bobbob = user_vsys_cpudata - vsyscall_text_begin ; vsys_cpudata = bobbob + VSYSCALL_text"); diff -urN /md0/kernels/2.4/v2.4.18-pre3/arch/i386/kernel/vunique.S linux.aio/arch/i386/kernel/vunique.S --- /md0/kernels/2.4/v2.4.18-pre3/arch/i386/kernel/vunique.S Wed Dec 31 19:00:00 1969 +++ linux.aio/arch/i386/kernel/vunique.S Wed Jan 23 01:41:06 2002 @@ -0,0 +1,7 @@ +#include + + .section .first_vsyscall_text,"xa" + .globl signature +signature: + .byte LINUX_UNIQUE_BYTES + .size signature,.-signature diff -urN /md0/kernels/2.4/v2.4.18-pre3/arch/i386/vmlinux.lds linux.aio/arch/i386/vmlinux.lds --- /md0/kernels/2.4/v2.4.18-pre3/arch/i386/vmlinux.lds Mon Jan 14 21:05:57 2002 +++ linux.aio/arch/i386/vmlinux.lds Wed Jan 23 01:41:06 2002 @@ -14,6 +14,27 @@ *(.gnu.warning) } = 0x9090 + /* Note: most of these declarations are in kernel/vsysdata.c,vsyscall.S. + * We use two segments for the data liked at a kernel virtual address + * (.data.vsyscall) and user virtual address (.vsyscall_data). + * .vsyscall_text is linked at a kernel virtual address + */ + . = ALIGN(4096); + VSYSCALL_text = .; + VSYSCALL 0xbfff0000 : AT ( VSYSCALL_text ) { + vsyscall_text_begin = .; + *(.first_vsyscall_text) + *(.vsyscall_text) + . = ALIGN(4096); + vsyscall_text_end = .; + *(.vsyscall_data) + . = ALIGN(4096); + vsyscall_data_end = .; + } + vsys_cpudata = user_vsys_cpudata - vsyscall_text_begin + VSYSCALL_text; + . = VSYSCALL_text + SIZEOF(VSYSCALL); + VSYSCALL_text_end = .; + _etext = .; /* End of text section */ .rodata : { *(.rodata) *(.rodata.*) } @@ -30,6 +51,17 @@ .data : { /* Data */ *(.data) + + . = ALIGN(8); + vsyscall_list_begin = .; + *(.data.vsyscall_list) + vsyscall_list_end = .; + . = ALIGN(4096); + kernel_vsyscall_data_begin = .; + *(.data.vsyscall) + . = ALIGN(4096); + kernel_vsyscall_data_end = .; + CONSTRUCTORS } @@ -79,4 +111,15 @@ .stab.index 0 : { *(.stab.index) } .stab.indexstr 0 : { *(.stab.indexstr) } .comment 0 : { *(.comment) } +/* +# VSYSCALL : { +# / * vsyscall area *i / +# __vsyscall_begin = .; +# *(vsyscall_text) +# . = ALIGN(4096); +# *(.data.vsyscall) +# . = ALIGN(4096); +# __vsyscall_end = .; +# } >vsyscall_area +*/ } diff -urN /md0/kernels/2.4/v2.4.18-pre3/drivers/block/loop.c linux.aio/drivers/block/loop.c --- /md0/kernels/2.4/v2.4.18-pre3/drivers/block/loop.c Tue Jan 1 14:09:34 2002 +++ linux.aio/drivers/block/loop.c Wed Jan 23 01:41:06 2002 @@ -280,7 +280,7 @@ spin_lock_irq(&lo->lo_lock); file = lo->lo_backing_file; spin_unlock_irq(&lo->lo_lock); - do_generic_file_read(file, &pos, &desc, lo_read_actor); + do_generic_file_read(file, &pos, &desc, lo_read_actor, 0); return desc.error; } diff -urN /md0/kernels/2.4/v2.4.18-pre3/drivers/char/mem.c linux.aio/drivers/char/mem.c --- /md0/kernels/2.4/v2.4.18-pre3/drivers/char/mem.c Tue Jan 1 14:09:34 2002 +++ linux.aio/drivers/char/mem.c Wed Jan 23 01:41:06 2002 @@ -25,6 +25,7 @@ #include #include #include +#include #ifdef CONFIG_I2C extern int i2c_init_all(void); @@ -572,6 +573,87 @@ write: write_full, }; +void vsys_mmap_close(struct vm_area_struct *area) +{ + area->vm_mm->vsys_mapped = 0; +} + +static struct vm_operations_struct vsys_mmap_ops = { + close: vsys_mmap_close, +}; + +int vsys_mmap(struct file *file, struct vm_area_struct *vma) +{ + extern unsigned char vsyscall_text_begin, vsyscall_text_end, VSYSCALL_text[]; + unsigned long offset = vma->vm_pgoff << PAGE_SHIFT; + unsigned long len = vma->vm_end - vma->vm_start; + unsigned long actual_len = &vsyscall_text_end - &vsyscall_text_begin; + + if ((offset + len) > actual_len) + len = actual_len - offset; + + pr_debug("len = 0x%lx, actual_len = 0x%lx\n", len, actual_len); + + vma->vm_start = (unsigned long)&vsyscall_text_begin + offset; + vma->vm_end = vma->vm_start + len; + vma->vm_flags |= VM_RESERVED; + + pr_debug("vm_start = 0x%lx, vm_end = 0x%lx\n", + vma->vm_start, vma->vm_end); + pr_debug("va=%p pa=0x%lx\n", + VSYSCALL_text + offset, + __pa(VSYSCALL_text) + offset); + + if (vma->vm_start < (unsigned long)&vsyscall_text_begin) { + pr_debug("vsys_mmap: start < begin\n"); + return -EINVAL; + } + + if (vma->vm_end < (unsigned long)&vsyscall_text_begin) { + pr_debug("vsys_mmap: end < begin\n"); + return -EINVAL; + } + + if (vma->vm_end > (unsigned long)&vsyscall_text_end) { + pr_debug("vsys_mmap: end(%lx) > text_end(%p)\n", + vma->vm_end, &vsyscall_text_end); + return -EINVAL; + } + + if (vma->vm_start >= vma->vm_end) { + pr_debug("vsys_mmap: end\n"); + return -EINVAL; + } + + if (find_vma_intersection(current->mm, vma->vm_start, vma->vm_end)) { + pr_debug("vsyscall: mapping collision\n"); + return -EINVAL; + } + + if ((vma->vm_flags & (VM_SHARED | VM_WRITE)) == (VM_SHARED | VM_WRITE)) { + pr_debug("vsyscall: attempt to write to mapping\n"); + return -EPERM; + } + + if (remap_page_range(vma->vm_start, + __pa(VSYSCALL_text) + offset, + vma->vm_end-vma->vm_start, + vma->vm_page_prot)) + return -EAGAIN; + + pr_debug("VSYSCALL_text(%p): %02x %02x %02x %02x\n", + VSYSCALL_text, + VSYSCALL_text[0], VSYSCALL_text[1], + VSYSCALL_text[2], VSYSCALL_text[3]); + + current->mm->vsys_mapped = 1; + return 0; +} + +static struct file_operations vsys_fops = { + mmap: vsys_mmap, +}; + static int memory_open(struct inode * inode, struct file * filp) { switch (MINOR(inode->i_rdev)) { @@ -601,6 +683,9 @@ case 9: filp->f_op = &urandom_fops; break; + case 10: + filp->f_op = &vsys_fops; + break; default: return -ENXIO; } @@ -625,7 +710,8 @@ {5, "zero", S_IRUGO | S_IWUGO, &zero_fops}, {7, "full", S_IRUGO | S_IWUGO, &full_fops}, {8, "random", S_IRUGO | S_IWUSR, &random_fops}, - {9, "urandom", S_IRUGO | S_IWUSR, &urandom_fops} + {9, "urandom", S_IRUGO | S_IWUSR, &urandom_fops}, + {10,"vsys", S_IRUGO, &vsys_fops}, }; int i; diff -urN /md0/kernels/2.4/v2.4.18-pre3/drivers/char/raw.c linux.aio/drivers/char/raw.c --- /md0/kernels/2.4/v2.4.18-pre3/drivers/char/raw.c Mon Sep 24 02:16:03 2001 +++ linux.aio/drivers/char/raw.c Wed Jan 23 01:41:06 2002 @@ -16,6 +16,8 @@ #include #include #include +#include +#include #define dprintk(x...) @@ -34,13 +36,18 @@ int raw_open(struct inode *, struct file *); int raw_release(struct inode *, struct file *); int raw_ctl_ioctl(struct inode *, struct file *, unsigned int, unsigned long); - +int raw_kvec_read(struct file *filp, kvec_cb_t cb, size_t size, loff_t pos); +int raw_kvec_write(struct file *filp, kvec_cb_t cb, size_t size, loff_t pos); static struct file_operations raw_fops = { read: raw_read, write: raw_write, open: raw_open, release: raw_release, + aio_read: generic_file_aio_read, + aio_write: generic_file_aio_write, + kvec_read: raw_kvec_read, + kvec_write: raw_kvec_write, }; static struct file_operations raw_ctl_fops = { @@ -250,7 +257,6 @@ } - ssize_t raw_read(struct file *filp, char * buf, size_t size, loff_t *offp) { @@ -381,3 +387,101 @@ out: return err; } + +static int raw_kvec_rw(struct file *filp, int rw, kvec_cb_t cb, size_t size, loff_t pos); +int raw_kvec_read(struct file *file, kvec_cb_t cb, size_t size, loff_t pos) +{ + return raw_kvec_rw(file, READ, cb, size, pos); +} + +int raw_kvec_write(struct file *file, kvec_cb_t cb, size_t size, loff_t pos) +{ + return raw_kvec_rw(file, WRITE, cb, size, pos); +} + +int raw_kvec_rw(struct file *filp, int rw, kvec_cb_t cb, size_t size, loff_t pos) +{ + int err; + unsigned minor; + kdev_t dev; + unsigned long limit, blocknr, blocks; + + unsigned sector_size, sector_bits, sector_mask; + unsigned max_sectors; + unsigned i; + + pr_debug("raw_kvec_rw: %p %d %d %p %d %d %Lu\n", filp, rw, nr, kiovec, flags, size, pos); + /* + * First, a few checks on device size limits + */ + + minor = MINOR(filp->f_dentry->d_inode->i_rdev); + dev = to_kdev_t(raw_devices[minor].binding->bd_dev); + sector_size = raw_devices[minor].sector_size; + sector_bits = raw_devices[minor].sector_bits; + sector_mask = sector_size- 1; + max_sectors = 25000; //KIO_MAX_SECTORS >> (sector_bits - 9); + + if (blk_size[MAJOR(dev)]) + limit = (((loff_t) blk_size[MAJOR(dev)][MINOR(dev)]) << BLOCK_SIZE_BITS) >> sector_bits; + else + limit = INT_MAX; + pr_debug ("raw_kvec_rw: dev %d:%d (+%d)\n", + MAJOR(dev), MINOR(dev), limit); + + /* EOF at the end */ + err = 0; + if (!size || (pos >> sector_bits) == limit) { + pr_debug("raw_kvec_rw: %Lu > %lu, %d\n", pos >> sector_bits, limit, sector_bits); + cb.fn(cb.data, cb.vec, err); + return 0; + } + + /* ENXIO for io beyond the end */ + err = -ENXIO; + if ((pos >> sector_bits) >= limit) { + pr_debug("raw_kvec_rw: %Lu > %lu, %d\n", pos >> sector_bits, limit, sector_bits); + goto out; + } + + err = -EINVAL; + if ((pos < 0) || (pos & sector_mask) || (size & sector_mask)) { + pr_debug("pos(%Ld)/size(%lu) wrong(%d)\n", pos, size, sector_mask); + goto out; + } + + /* Verify that the scatter-gather list is sector aligned. */ + for (i=0; inr; i++) + if ((cb.vec->veclet[i].offset & sector_mask) || + (cb.vec->veclet[i].length & sector_mask)) { + pr_debug("veclet offset/length wrong"); + goto out; + } + + /* + * Split the IO into KIO_MAX_SECTORS chunks, mapping and + * unmapping the single kiobuf as we go to perform each chunk of + * IO. + */ + + blocknr = pos >> sector_bits; + blocks = size >> sector_bits; + if (blocks > max_sectors) + blocks = max_sectors; + if (blocks > limit - blocknr) + blocks = limit - blocknr; + err = -ENXIO; + if (!blocks) { + pr_debug("raw: !blocks %d %ld %ld\n", max_sectors, limit, blocknr); + goto out; + } + + err = brw_kvec_async(rw, cb, dev, blocks, blocknr, sector_bits); + if (err) + printk("raw_kvec_rw: %d\n", err); + +out: + pr_debug("raw_kvec_rw: ret is %d\n", err); + return err; +} + diff -urN /md0/kernels/2.4/v2.4.18-pre3/fs/Makefile linux.aio/fs/Makefile --- /md0/kernels/2.4/v2.4.18-pre3/fs/Makefile Mon Nov 26 23:43:08 2001 +++ linux.aio/fs/Makefile Wed Jan 23 01:41:06 2002 @@ -22,6 +22,8 @@ obj-y += noquot.o endif +obj-y += aio.o + subdir-$(CONFIG_PROC_FS) += proc subdir-y += partitions diff -urN /md0/kernels/2.4/v2.4.18-pre3/fs/aio.c linux.aio/fs/aio.c --- /md0/kernels/2.4/v2.4.18-pre3/fs/aio.c Wed Dec 31 19:00:00 1969 +++ linux.aio/fs/aio.c Wed Jan 23 01:41:06 2002 @@ -0,0 +1,937 @@ +/* fs/aio.c + * An async IO implementation for Linux + * Written by Benjamin LaHaise + * + * Implements an efficient asynchronous io interface. + * + * Copyright 2000, 2001 Red Hat, Inc. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ +//#define DEBUG 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +//#define static /**/ +//#define inline /**/ +#if DEBUG > 1 +#define dprintk printk +#else +#define dprintk(x...) do { ; } while (0) +#endif + +static spinlock_t aio_read_lock = SPIN_LOCK_UNLOCKED; +static rwlock_t aio_req_lock = RW_LOCK_UNLOCKED; + +static kmem_cache_t *kiocb_cachep; +static kmem_cache_t *kioctx_cachep; + +/* tunable. Needs to be added to sysctl. */ +int max_aio_reqs = 0x10000; + +/* Used for rare fput completion. */ +static void aio_fput_routine(void *); +static struct tq_struct fput_tqueue = { + routine: aio_fput_routine, +}; + +static spinlock_t fput_lock = SPIN_LOCK_UNLOCKED; +static struct list_head fput_head; + +/* aio_setup + * Creates the slab caches used by the aio routines, panic on + * failure as this is done early during the boot sequence. + */ +static int __init aio_setup(void) +{ + kiocb_cachep = kmem_cache_create("kiocb", sizeof(struct kiocb), + 0, SLAB_HWCACHE_ALIGN, NULL, NULL); + if (!kiocb_cachep) + panic("unable to create kiocb cache\n"); + + kioctx_cachep = kmem_cache_create("kioctx", sizeof(struct kioctx), + 0, SLAB_HWCACHE_ALIGN, NULL, NULL); + if (!kioctx_cachep) + panic("unable to create kioctx cache"); + + printk(KERN_NOTICE "aio_setup: okay!\n"); + printk(KERN_NOTICE "aio_setup: sizeof(struct page) = %d\n", (int)sizeof(struct page)); + + return 0; +} + +static void ioctx_free_reqs(struct kioctx *ctx) +{ + struct list_head *pos, *next; + list_for_each_safe(pos, next, &ctx->free_reqs) { + struct kiocb *iocb = list_kiocb(pos); + list_del(&iocb->list); + kmem_cache_free(kiocb_cachep, iocb); + } +} + +/* ioctx_alloc + * Allocates and initializes an ioctx. Returns an ERR_PTR if it failed. + */ +static struct kioctx *ioctx_alloc(unsigned nr_reqs) +{ + struct kioctx *ctx; + unsigned i; + long size; + + /* Round off to a power of 2. Needed for cheap mask operations */ + for (i=1; i (0x10000000U / sizeof(struct io_event))) || + (nr_reqs > (0x10000000U / sizeof(struct kiocb)))) { + pr_debug("ENOMEM: nr_reqs too high\n"); + return ERR_PTR(-ENOMEM); + } + + ctx = kmem_cache_alloc(kioctx_cachep, GFP_KERNEL); + if (!ctx) + return ERR_PTR(-ENOMEM); + + memset(ctx, 0, sizeof(*ctx)); + ctx->max_reqs = nr_reqs; + + atomic_set(&ctx->users, 1); + spin_lock_init(&ctx->lock); + init_waitqueue_head(&ctx->wait); + + INIT_LIST_HEAD(&ctx->free_reqs); + INIT_LIST_HEAD(&ctx->active_reqs); + + /* Allocate nr_reqs iocbs for io. Free iocbs are on the + * ctx->free_reqs list. When active they migrate to the + * active_reqs list. During completion and cancellation + * the request may temporarily not be on any list. + */ + for (i=0; ictx = ctx; + iocb->filp = NULL; + iocb->cancel = NULL; + iocb->user_data = 0; + iocb->nr_atomic = 0; + iocb->key = i; + list_add(&iocb->list, &ctx->free_reqs); + } + + /* Compensate for the ring buffer's head/tail overlap entry */ + nr_reqs *= 2; + + size = sizeof(struct aio_ring); + size += sizeof(struct io_event) * nr_reqs; + + /* Try to use alloc_pages first... */ + for (i=0; (PAGE_SIZE << i) < size; i++) + ; + ctx->ring_order = i; + ctx->ring = (void *)__get_free_pages(GFP_KERNEL, i); + if (!ctx->ring) { + /* vmalloc it is... */ + ctx->ring = vmalloc(size); + if (!ctx->ring) + goto out_freereqs; + ctx->ring_was_vmallocd = 1; + } else + ctx->ring_was_vmallocd = 0; + + memset(ctx->ring, 0, size); + ctx->ring_mask = nr_reqs - 1; /* trusted copy */ + ctx->ring->mask = ctx->ring_mask; /* user copy */ + + /* now link into global list. kludge. FIXME */ + write_lock(&aio_req_lock); /* FIXME */ + ctx->ring->id = ctx->user_id = current->mm->new_ioctx_id++; + ctx->next = current->mm->ioctx_list; + current->mm->ioctx_list = ctx; + write_unlock(&aio_req_lock); /* FIXME */ + + dprintk("aio: allocated ioctx %p[%ld]: mm=%p mask=0x%x\n", + ctx, ctx->user_id, current->mm, ctx->ring->mask); + return ctx; + +out_freereqs: + ioctx_free_reqs(ctx); +out_freectx: + kmem_cache_free(kioctx_cachep, ctx); + ctx = ERR_PTR(-ENOMEM); + + dprintk("aio: error allocating ioctx %p\n", ctx); + return ctx; +} + +void exit_aio(struct mm_struct *mm) +{ + struct kioctx *ctx = mm->ioctx_list; + + mm->ioctx_list = NULL; + while (ctx) { + struct kioctx *next = ctx->next; + put_ioctx(ctx); + ctx = next; + } +} + +/* __put_ioctx + * Called when the last user of an aio context has gone away, + * and the struct needs to be freed. + */ +void __put_ioctx(struct kioctx *ctx) +{ + if (ctx->ring_was_vmallocd) + vfree(ctx->ring); + else + free_pages((unsigned long)ctx->ring, ctx->ring_order); + + ioctx_free_reqs(ctx); + kmem_cache_free(kioctx_cachep, ctx); +} + +/* aio_get_req + * Allocate a slot for an aio request. Increments the users count + * of the kioctx so that the kioctx stays around until all requests are + * complete. Returns -EAGAIN if no requests are free. + */ +static inline struct kiocb *aio_get_req(struct kioctx *ctx) +{ + struct kiocb *req; + + /* Use cmpxchg instead of spin_lock? */ + spin_lock_irq(&ctx->lock); + if (!list_empty(&ctx->free_reqs) && + (ctx->reqs_active < aio_ring_avail(ctx->ring))) { + req = list_kiocb(ctx->free_reqs.next); + list_del(&req->list); + list_add(&req->list, &ctx->active_reqs); + ctx->reqs_active++; + spin_unlock_irq(&ctx->lock); + req->user_obj = NULL; + + get_ioctx(ctx); + return req; + } + spin_unlock_irq(&ctx->lock); + + return NULL; +} + +static inline void really_put_req(struct kioctx *ctx, struct kiocb *req) +{ + req->filp = NULL; + req->user_obj = NULL; + ctx->reqs_active--; + list_add(&req->list, &ctx->free_reqs); +} + +static void aio_fput_routine(void *data) +{ + struct list_head head; + + spin_lock_irq(&fput_lock); + head = fput_head; + INIT_LIST_HEAD(&fput_head); + spin_unlock_irq(&fput_lock); + + while (!list_empty(&head)) { + struct kiocb *req = list_kiocb(head.next); + struct kioctx *ctx = req->ctx; + struct kiocb *next = req->user_obj; + + list_del(&req->list); + + /* Complete the fput */ + __fput(req->filp); + + /* Link the iocb into the context's free list */ + spin_lock_irq(&ctx->lock); + really_put_req(ctx, req); + spin_unlock_irq(&ctx->lock); + + put_ioctx(ctx); + } +} + +static void __aio_put_req(struct kioctx *ctx, struct kiocb *req) +{ + dprintk(KERN_DEBUG "aio_put(%p): f_count=%d\n", + req, atomic_read(&req->filp->f_count)); + + list_del(&req->list); /* remove from active_reqs */ + req->cancel = NULL; + + /* Must be done under the lock to serialise against cancellations */ + if (unlikely(atomic_dec_and_test(&req->filp->f_count))) { + get_ioctx(ctx); + spin_lock(&fput_lock); + list_add(&req->list, &fput_head); + spin_unlock(&fput_lock); + schedule_task(&fput_tqueue); + } else + really_put_req(ctx, req); +} + +static void aio_put_req(struct kioctx *ctx, struct kiocb *req) +{ + /* FIXME: use cmpxchg instead of spin_lock? */ + spin_lock_irq(&ctx->lock); + __aio_put_req(ctx, req); + spin_unlock_irq(&ctx->lock); + put_ioctx(ctx); +} + +/* Lookup an ioctx id. ioctx_list is lockless for reads. + * FIXME: this is O(n) and is only suitable for development. + */ +static inline struct kioctx *lookup_ioctx(unsigned long ctx_id) +{ + struct kioctx *ioctx; + struct mm_struct *mm = current->mm; + + read_lock(&aio_req_lock); + for (ioctx = current->mm->ioctx_list; ioctx; ioctx = ioctx->next) + if (likely(ioctx->user_id == ctx_id && !ioctx->dead)) { + get_ioctx(ioctx); + break; + } + read_unlock(&aio_req_lock); + + return ioctx; +} + +/* aio_complete + * Called when the io request on the given iocb is complete. + */ +void aio_complete(struct kiocb *iocb, long res, long res2) +{ + struct kioctx *ctx = iocb->ctx; + struct aio_ring *ring = ctx->ring; + struct io_event *event; + unsigned long flags; + unsigned long tail; + + /* add a completion event to the ring buffer. + * must be done holding ctx->lock to prevent + * other code from messing with the tail + * pointer since we might be called from irq + * context. + */ + spin_lock_irqsave(&ctx->lock, flags); + + tail = ring->tail; + event = &ring->io_events[tail]; + tail = (tail + 1) & ring->mask; + + event->obj = (u64)(unsigned long)iocb->user_obj; + event->data = iocb->user_data; + event->res = res; + event->res2 = res2; + + dprintk("aio_complete: %p[%lu]: %p: %p %Lx %lx %lx\n", + ctx, tail, iocb, iocb->user_obj, iocb->user_data, res, res2); + + /* after flagging the request as done, we + * must never even look at it again + */ + barrier(); + + ring->tail = tail; + + wmb(); + if (!ring->woke) + ring->woke = 1; + + pr_debug("added to ring %p at [%lu]\n", iocb, tail); +#if 0 + if (!wake) { + printk("kio_complete: should send user of %p a signal...\n", ctx); + } +#endif + + wake_up(&ctx->wait); + + /* everything turned out well, dispose of the aiocb. */ + __aio_put_req(ctx, iocb); + + spin_unlock_irqrestore(&ctx->lock, flags); + put_ioctx(ctx); +} + +/* aio_read_evt + * Pull an event off of the ioctx's event ring. Returns the number of + * events fetched (0 or 1 ;-) + * FIXME: make this use cmpxchg. + * TODO: make the ringbuffer user mmap()able (requires FIXME). + */ +static int aio_read_evt(struct kioctx *ioctx, struct io_event *ent) +{ + struct aio_ring *ring = ioctx->ring; + unsigned long head; + int ret = 0; + + dprintk("in aio_read_evt h%lu t%lu m%lu\n", + (unsigned long)ring->head, (unsigned long)ring->tail, + (unsigned long)ring->mask); + barrier(); + if (ring->head == ring->tail) + goto out; + + spin_lock(&aio_read_lock); /* investigate the value of making this per-ctx */ + + head = ring->head; + if (head != ring->tail) { + *ent = ring->io_events[head]; + head = (head + 1) & ioctx->ring_mask; + barrier(); + ring->head = head; + ret = 1; + } + spin_unlock(&aio_read_lock); + +out: + dprintk("leaving aio_read_evt: %d h%lu t%lu\n", ret, + (unsigned long)ring->head, (unsigned long)ring->tail); + return ret; +} + +struct timeout { + struct timer_list timer; + int timed_out; + wait_queue_head_t wait; +}; + +static void timeout_func(unsigned long data) +{ + struct timeout *to = (struct timeout *)data; + + to->timed_out = 1; + wake_up(&to->wait); +} + +static inline void init_timeout(struct timeout *to) +{ + init_timer(&to->timer); + to->timer.data = (unsigned long)to; + to->timer.function = timeout_func; + to->timed_out = 0; + init_waitqueue_head(&to->wait); +} + +static inline void set_timeout(struct timeout *to, struct timespec *ts) +{ + unsigned long how_long; + + if (!ts->tv_sec && !ts->tv_nsec) { + to->timed_out = 1; + return; + } + + how_long = ts->tv_sec * HZ; +#define HZ_NS (1000000000 / HZ) + how_long += (ts->tv_nsec + HZ_NS - 1) / HZ_NS; + + to->timer.expires = jiffies + how_long; + add_timer(&to->timer); +} + +static inline void clear_timeout(struct timeout *to) +{ + del_timer_sync(&to->timer); +} + +static int read_events(struct kioctx *ctx, int nr, struct io_event *event, + struct timespec *timeout) +{ + struct task_struct *tsk = current; + DECLARE_WAITQUEUE(wait, tsk); + DECLARE_WAITQUEUE(to_wait, tsk); + int ret; + int i = 0; + struct io_event ent; + struct timespec ts; + struct timeout to; + + init_timeout(&to); + + if (timeout) { + ret = -EFAULT; + if (copy_from_user(&ts, timeout, sizeof(ts))) + goto out; + + set_timeout(&to, &ts); + if (to.timed_out) + timeout = 0; + } + + memset(&ent, 0, sizeof(ent)); + ret = 0; + + while (i < nr) { + ret = aio_read_evt(ctx, &ent); + if (ret <= 0) { + if (i) + break; + + ret = 0; + if (to.timed_out) + break; + + add_wait_queue(&ctx->wait, &wait); + add_wait_queue(&to.wait, &to_wait); + do { + set_task_state(tsk, TASK_INTERRUPTIBLE); + + ret = aio_read_evt(ctx, &ent); + if (ret <= 0) + break; + ret = 0; + if (to.timed_out) + break; + schedule(); + if (to.timed_out) + break; + if (signal_pending(tsk)) { + ret = -EINTR; + break; + } + ret = aio_read_evt(ctx, &ent); + } while (ret <= 0) ; + + set_task_state(tsk, TASK_RUNNING); + remove_wait_queue(&ctx->wait, &wait); + remove_wait_queue(&to.wait, &to_wait); + } + + if (unlikely(ret <= 0)) + break; + + dprintk("read event: %Lx %Lx %Lx %Lx\n", + ent.data, ent.obj, ent.res, ent.res2); + + /* FIXME: split checks in two */ + ret = -EFAULT; + if (unlikely(copy_to_user(event, &ent, sizeof(ent)))) { + printk(KERN_DEBUG "aio: lost an event due to EFAULT.\n"); + break; + } + + /* Now complete the aio request and copy the result codes to userland. */ + event ++; + i ++; + } + + if (timeout) + clear_timeout(&to); +out: + return i ? i : ret; +} + +asmlinkage long sys_io_setup(unsigned nr_reqs, aio_context_t *ctxp) +{ + struct kioctx *ioctx = NULL; + unsigned long ctx; + long ret; + + ret = get_user(ctx, ctxp); + if (unlikely(ret)) + goto out; + + ret = -EINVAL; + if (ctx || nr_reqs > max_aio_reqs) { + pr_debug("EINVAL: io_setup: !ctx or nr_reqs > max\n"); + goto out; + } + + ioctx = ioctx_alloc(nr_reqs); + ret = PTR_ERR(ioctx); + if (!IS_ERR(ioctx)) { + ret = put_user(ioctx->user_id, ctxp); + if (!ret) + return 0; + ioctx->dead = 1; + put_ioctx(ioctx); + } + +out: + return ret; +} + +/* aio_release + * Release the kioctx associated with the userspace handle. + */ +asmlinkage long sys_io_destroy(aio_context_t ctx) +{ + struct kioctx *ioctx = lookup_ioctx(ctx); + if (likely(NULL != ioctx)) { + struct kioctx **tmp; + int was_dead; + + /* delete the entry from the list is someone else hasn't already */ + write_lock(&aio_req_lock); + was_dead = ioctx->dead; + ioctx->dead = 1; + for (tmp = ¤t->mm->ioctx_list; *tmp && *tmp != ioctx; + tmp = &(*tmp)->next) + ; + if (*tmp) + *tmp = ioctx->next; + write_unlock(&aio_req_lock); + + dprintk("aio_release(%p)\n", ioctx); + put_ioctx(ioctx); /* once for the lookup */ + if (likely(!was_dead)) + put_ioctx(ioctx); /* twice for the list */ + return 0; + } + pr_debug("EINVAL: io_destroy: invalid context id\n"); + return -EINVAL; +} + +int generic_aio_poll(struct file *file, struct kiocb *req, struct iocb iocb) +{ + unsigned events = iocb.aio_buf; + + /* Did the user set any bits they weren't supposed to? (The + * above is actually a cast. + */ + if (unlikely(events != iocb.aio_buf)) + return -EINVAL; + + return async_poll(req, events); +} + +/* sys_io_submit + * Copy an aiocb from userspace into kernel space, then convert it to + * a kiocb, submit and repeat until done. Error codes on copy/submit + * only get returned for the first aiocb copied as otherwise the size + * of aiocbs copied is returned (standard write sematics). + */ +asmlinkage long sys_io_submit(aio_context_t ctx_id, long nr, struct iocb **iocbpp) +{ + struct kioctx *ctx; + long ret = 0; + int i; + + ctx = lookup_ioctx(ctx_id); + if (unlikely(!ctx)) { + pr_debug("EINVAL: io_submit: invalid context id\n"); + return -EINVAL; + } + + for (i=0; ifilp = file; + tmp.aio_key = req->key; + ret = put_user(tmp.aio_key, &iocbp->aio_key); + if (unlikely(ret)) + goto out_fput; + + req->user_obj = iocbp; + req->user_data = tmp.aio_data; + + switch (tmp.aio_lio_opcode) { + case IOCB_CMD_PREAD: op = file->f_op->aio_read; break; + case IOCB_CMD_PREADX: op = file->f_op->aio_readx; break; + case IOCB_CMD_PWRITE: op = file->f_op->aio_write; break; + case IOCB_CMD_FSYNC: op = file->f_op->aio_fsync; break; + case IOCB_CMD_POLL: op = generic_aio_poll; break; + default: op = NULL; break; + } + + ret = -EINVAL; + if (unlikely(!op)) + pr_debug("EINVAL: io_submit: no operation provided\n"); + else { + ret = op(file, req, tmp); + if (likely(!ret)) + continue; + } + + pr_debug("io_submit: op returned %ld\n", ret); + aio_complete(req, ret, 0); + ret = 0; /* A completion event was sent, so + * submit is a success. */ + continue; + + out_fput: + fput(file); + break; + } + + put_ioctx(ctx); + run_task_queue(&tq_disk); + return i ? i : ret; +} + +static inline void generic_aio_complete_rw(int rw, void *_iocb, struct kvec *vec, ssize_t res) +{ + struct kiocb *iocb = _iocb; + ssize_t total = iocb->nr_atomic; + + if (res > 0) + total += res; + + aio_complete(iocb, total ? total : res, 0); + //unmap_kvec(vec, rw); + //free_kvec(vec); +} + +void generic_aio_complete_read(void *_iocb, struct kvec *vec, ssize_t res) +{ + generic_aio_complete_rw(1, _iocb, vec, res); +} + +void generic_aio_complete_write(void *_iocb, struct kvec *vec, ssize_t res) +{ + generic_aio_complete_rw(0, _iocb, vec, res); +} + +ssize_t generic_aio_read(struct file *file, struct kiocb *req, struct iocb iocb, size_t min_size) +{ + unsigned long buf = iocb.aio_buf; + size_t size = iocb.aio_nbytes; + ssize_t nr_read = 0; + loff_t pos = iocb.aio_offset; + kvec_cb_t cb; + + if (likely(NULL != file->f_op->new_read)) { + nr_read = file->f_op->new_read(file, (void *)buf, size, + &pos, F_ATOMIC); + dprintk("from new_read: nr_read: %ld\n", (long)nr_read); + if ((-EAGAIN == nr_read) || (-EWOULDBLOCKIO == nr_read)) + nr_read = 0; + else if ((nr_read >= min_size) || (nr_read < 0)) { + dprintk("returning nr_read: %ld\n", (long)nr_read); + return nr_read; + } + } + dprintk("nr_read: %ld\n", (long)nr_read); + + req->nr_atomic = nr_read; + size -= nr_read; + buf += nr_read; + cb.vec = map_user_kvec(READ, buf, size); + cb.fn = generic_aio_complete_read; + cb.data = req; + + dprintk("generic_aio_read: cb.vec=%p\n", cb.vec); + if (IS_ERR(cb.vec)) + return nr_read ? nr_read : PTR_ERR(cb.vec); + + return file->f_op->kvec_read(file, cb, size, pos); +} + +ssize_t generic_file_aio_read(struct file *file, struct kiocb *req, struct iocb iocb) +{ + return generic_aio_read(file, req, iocb, iocb.aio_nbytes); +} + +ssize_t generic_sock_aio_read(struct file *file, struct kiocb *req, struct iocb iocb) +{ + return generic_aio_read(file, req, iocb, 1); +} + +ssize_t generic_aio_write(struct file *file, struct kiocb *req, struct iocb iocb, size_t min_size) +{ + unsigned long buf = iocb.aio_buf; + size_t size = iocb.aio_nbytes; + loff_t pos = iocb.aio_offset; + ssize_t nr_written = 0; + kvec_cb_t cb; + long res; + + if (likely(NULL != file->f_op->new_write)) { + nr_written = file->f_op->new_write(file, (void *)buf, size, + &pos, F_ATOMIC); + pr_debug("generic_aio_write: new_write: %ld\n", (long)nr_written); + if (-EAGAIN == nr_written) + nr_written = 0; + if ((nr_written >= min_size) || (nr_written < 0)) + return nr_written; + } + + req->nr_atomic = nr_written; + size -= nr_written; + buf += nr_written; + cb.vec = map_user_kvec(WRITE, buf, size); + cb.fn = generic_aio_complete_write; + cb.data = req; + + if (IS_ERR(cb.vec)) { + pr_debug("generic_aio_write: map_user_kvec: %ld\n", PTR_ERR(cb.vec)); + return nr_written ? nr_written : PTR_ERR(cb.vec); + } + + res = file->f_op->kvec_write(file, cb, size, iocb.aio_offset); + pr_debug("generic_aio_write: kvec_write: %ld\n", res); + if (unlikely(res < 0)) { + if (nr_written) + res = nr_written; + } + return res; +} + +ssize_t generic_file_aio_write(struct file *file, struct kiocb *req, struct iocb iocb) +{ + return generic_aio_write(file, req, iocb, iocb.aio_nbytes); +} + +struct kiocb *lookup_kiocb(struct kioctx *ctx, struct iocb *iocb, u32 key) +{ + /* todo. */ + return NULL; +} + +asmlinkage long sys_io_cancel(aio_context_t ctx_id, struct iocb *iocb) +{ + struct kioctx *ctx; + struct kiocb *kiocb; + u64 user_data; + u32 key; + int ret; + + ret = get_user(key, &iocb->aio_key); + if (unlikely(ret)) + return ret; + + ctx = lookup_ioctx(ctx_id); + if (unlikely(!ctx)) + return -EINVAL; + + spin_lock_irq(&ctx->lock); + ret = -EAGAIN; + kiocb = lookup_kiocb(ctx, iocb, key); + if (kiocb) + ret = kiocb->cancel(ctx, kiocb); + spin_unlock_irq(&ctx->lock); + + put_ioctx(ctx); + + return ret; +} + +asmlinkage long sys_io_wait(aio_context_t ctx_id, struct iocb *iocb, struct timespec *timeout) +{ +#if 0 /* FIXME. later. */ + struct kioctx *ioctx; + long ret = -EINVAL; + unsigned key; + long obj = (long)iocb; + + ioctx = lookup_ioctx(ctx_id); + if (!ioctx) + goto out; + + ret = get_user(key, &iocb->aio_key); + if (ret) + goto out; + + ret = __aio_complete(ioctx, key, obj, !!timeout); + put_ioctx(ioctx); + +out: + return ret; +#endif + return -ENOSYS; +} + +asmlinkage long sys_io_getevents(aio_context_t ctx_id, + long nr, + struct io_event *events, + struct timespec *timeout) +{ + struct kioctx *ioctx = lookup_ioctx(ctx_id); + long ret = -EINVAL; + + if (likely(NULL != ioctx)) { + ret = read_events(ioctx, nr, events, timeout); + put_ioctx(ioctx); + } + + return ret; +} + +__initcall(aio_setup); + +add_dynamic_syscall(sys_io_setup); +add_dynamic_syscall(sys_io_destroy); +add_dynamic_syscall(sys_io_submit); +add_dynamic_syscall(sys_io_cancel); +add_dynamic_syscall(sys_io_wait); +add_dynamic_syscall(sys_io_getevents); diff -urN /md0/kernels/2.4/v2.4.18-pre3/fs/buffer.c linux.aio/fs/buffer.c --- /md0/kernels/2.4/v2.4.18-pre3/fs/buffer.c Tue Jan 1 14:09:35 2002 +++ linux.aio/fs/buffer.c Wed Jan 23 01:41:06 2002 @@ -125,8 +125,7 @@ clear_bit(BH_launder, &bh->b_state); clear_bit(BH_Lock, &bh->b_state); smp_mb__after_clear_bit(); - if (waitqueue_active(&bh->b_wait)) - wake_up(&bh->b_wait); + wake_up(&bh->b_wait); } /* @@ -2864,3 +2863,222 @@ module_init(bdflush_init) +/* async kio interface */ +struct brw_cb { + kvec_cb_t cb; + atomic_t io_count; + int nr; + struct buffer_head *bh[1]; +}; + +static inline void brw_cb_put(struct brw_cb *brw_cb) +{ + if (atomic_dec_and_test(&brw_cb->io_count)) { + ssize_t res = 0, err = 0; + int nr; + + /* Walk the buffer heads associated with this kiobuf + * checking for errors and freeing them as we go. + */ + for (nr=0; nr < brw_cb->nr; nr++) { + struct buffer_head *bh = brw_cb->bh[nr]; + if (!err && buffer_uptodate(bh)) + res += bh->b_size; + else + err = -EIO; + kmem_cache_free(bh_cachep, bh); + } + + if (!res) + res = err; + + brw_cb->cb.fn(brw_cb->cb.data, brw_cb->cb.vec, res); + + kfree(brw_cb); + } +} + +/* + * IO completion routine for a buffer_head being used for kiobuf IO: we + * can't dispatch the kiobuf callback until io_count reaches 0. + */ + +static void end_buffer_io_kiobuf_async(struct buffer_head *bh, int uptodate) +{ + struct brw_cb *brw_cb; + + mark_buffer_uptodate(bh, uptodate); + + brw_cb = bh->b_private; + unlock_buffer(bh); + + brw_cb_put(brw_cb); +} + + +/* + * Start I/O on a physical range of kernel memory, defined by a vector + * of kiobuf structs (much like a user-space iovec list). + * + * The kiobuf must already be locked for IO. IO is submitted + * asynchronously: you need to check page->locked, page->uptodate, and + * maybe wait on page->wait. + * + * It is up to the caller to make sure that there are enough blocks + * passed in to completely map the iobufs to disk. + */ + +int brw_kvec_async(int rw, kvec_cb_t cb, kdev_t dev, unsigned blocks, unsigned long blknr, int sector_shift) +{ + struct kvec *vec = cb.vec; + struct kveclet *veclet; + int err; + int length; + unsigned sector_size = 1 << sector_shift; + int i; + + struct brw_cb *brw_cb; + + if (!vec->nr) + BUG(); + + /* + * First, do some alignment and validity checks + */ + length = 0; + for (veclet=vec->veclet, i=0; i < vec->nr; i++,veclet++) { + length += veclet->length; + if ((veclet->offset & (sector_size-1)) || + (veclet->length & (sector_size-1))) { + printk("brw_kiovec_async: tuple[%d]->offset=0x%x length=0x%x sector_size: 0x%x\n", i, veclet->offset, veclet->length, sector_size); + return -EINVAL; + } + } + + if (length < (blocks << sector_shift)) + BUG(); + + /* + * OK to walk down the iovec doing page IO on each page we find. + */ + err = 0; + + if (!blocks) { + printk("brw_kiovec_async: !i\n"); + return -EINVAL; + } + + /* FIXME: tie into userbeans here */ + brw_cb = kmalloc(sizeof(*brw_cb) + (blocks * sizeof(struct buffer_head *)), GFP_KERNEL); + if (!brw_cb) + return -ENOMEM; + + brw_cb->cb = cb; + brw_cb->nr = 0; + + /* This is ugly. FIXME. */ + for (i=0, veclet=vec->veclet; inr; i++,veclet++) { + struct page *page = veclet->page; + unsigned offset = veclet->offset; + unsigned length = veclet->length; + + if (!page) + BUG(); + + while (length > 0) { + struct buffer_head *tmp; + tmp = kmem_cache_alloc(bh_cachep, GFP_NOIO); + err = -ENOMEM; + if (!tmp) + goto error; + + memset(tmp, 0, sizeof(*tmp)); + init_waitqueue_head(&tmp->b_wait); + tmp->b_dev = B_FREE; + tmp->b_size = sector_size; + set_bh_page(tmp, page, offset); + tmp->b_this_page = tmp; + + init_buffer(tmp, end_buffer_io_kiobuf_async, NULL); + tmp->b_dev = dev; + tmp->b_blocknr = blknr++; + tmp->b_state = (1 << BH_Mapped) | (1 << BH_Lock) + | (1 << BH_Req); + tmp->b_private = brw_cb; + + if (rw == WRITE) { + set_bit(BH_Uptodate, &tmp->b_state); + clear_bit(BH_Dirty, &tmp->b_state); + } + + brw_cb->bh[brw_cb->nr++] = tmp; + length -= sector_size; + offset += sector_size; + + if (offset >= PAGE_SIZE) { + offset = 0; + break; + } + + if (brw_cb->nr >= blocks) + goto submit; + } /* End of block loop */ + } /* End of page loop */ + +submit: + atomic_set(&brw_cb->io_count, brw_cb->nr+1); + /* okay, we've setup all our io requests, now fire them off! */ + for (i=0; inr; i++) + submit_bh(rw, brw_cb->bh[i]); + brw_cb_put(brw_cb); + + return 0; + +error: + /* Walk brw_cb_table freeing all the goop associated with each kiobuf */ + if (brw_cb) { + /* We got an error allocating the bh'es. Just free the current + buffer_heads and exit. */ + for (i = brw_cb->nr-1; i--; ) + kmem_cache_free(bh_cachep, brw_cb->bh[i]); + kfree(brw_cb); + } + + return err; +} +#if 0 +int brw_kiovec(int rw, int nr, struct kiobuf *iovec[], + kdev_t dev, int nr_blocks, unsigned long b[], int sector_size) +{ + int i; + int transferred = 0; + int err = 0; + + if (!nr) + return 0; + + /* queue up and trigger the io */ + err = brw_kiovec_async(rw, nr, iovec, dev, nr_blocks, b, sector_size); + if (err) + goto out; + + /* wait on the last iovec first -- it's more likely to finish last */ + for (i=nr; --i >= 0; ) + kiobuf_wait_for_io(iovec[i]); + + run_task_queue(&tq_disk); + + /* okay, how much data actually got through? */ + for (i=0; ierrno) { + if (!err) + err = iovec[i]->errno; + break; + } + transferred += iovec[i]->length; + } + +out: + return transferred ? transferred : err; +} +#endif diff -urN /md0/kernels/2.4/v2.4.18-pre3/fs/ext2/file.c linux.aio/fs/ext2/file.c --- /md0/kernels/2.4/v2.4.18-pre3/fs/ext2/file.c Thu Nov 1 16:40:02 2001 +++ linux.aio/fs/ext2/file.c Wed Jan 23 01:41:06 2002 @@ -40,6 +40,9 @@ */ struct file_operations ext2_file_operations = { llseek: generic_file_llseek, + kvec_read: generic_file_kvec_read, + kvec_write: generic_file_kvec_write, + new_read: generic_file_new_read, read: generic_file_read, write: generic_file_write, ioctl: ext2_ioctl, @@ -47,6 +50,8 @@ open: generic_file_open, release: ext2_release_file, fsync: ext2_sync_file, + aio_read: generic_file_aio_read, + aio_write: generic_file_aio_write, }; struct inode_operations ext2_file_inode_operations = { diff -urN /md0/kernels/2.4/v2.4.18-pre3/fs/ext3/file.c linux.aio/fs/ext3/file.c --- /md0/kernels/2.4/v2.4.18-pre3/fs/ext3/file.c Mon Nov 26 23:43:08 2001 +++ linux.aio/fs/ext3/file.c Wed Jan 23 01:41:06 2002 @@ -78,6 +78,9 @@ struct file_operations ext3_file_operations = { llseek: generic_file_llseek, /* BKL held */ + kvec_read: generic_file_kvec_read, + kvec_write: generic_file_kvec_write, /* FIXME: attributes */ + new_read: generic_file_new_read, read: generic_file_read, /* BKL not held. Don't need */ write: ext3_file_write, /* BKL not held. Don't need */ ioctl: ext3_ioctl, /* BKL held */ @@ -85,6 +88,8 @@ open: ext3_open_file, /* BKL not held. Don't need */ release: ext3_release_file, /* BKL not held. Don't need */ fsync: ext3_sync_file, /* BKL held */ + aio_read: generic_file_aio_read, + aio_write: generic_file_aio_write, }; struct inode_operations ext3_file_inode_operations = { diff -urN /md0/kernels/2.4/v2.4.18-pre3/fs/file_table.c linux.aio/fs/file_table.c --- /md0/kernels/2.4/v2.4.18-pre3/fs/file_table.c Mon Sep 24 02:16:04 2001 +++ linux.aio/fs/file_table.c Wed Jan 23 01:41:06 2002 @@ -99,31 +99,35 @@ void fput(struct file * file) { + if (atomic_dec_and_test(&file->f_count)) + __fput(file); +} + +void __fput(struct file * file) +{ struct dentry * dentry = file->f_dentry; struct vfsmount * mnt = file->f_vfsmnt; struct inode * inode = dentry->d_inode; - if (atomic_dec_and_test(&file->f_count)) { - locks_remove_flock(file); + locks_remove_flock(file); - if (file->f_iobuf) - free_kiovec(1, &file->f_iobuf); + if (file->f_iobuf) + free_kiovec(1, &file->f_iobuf); - if (file->f_op && file->f_op->release) - file->f_op->release(inode, file); - fops_put(file->f_op); - if (file->f_mode & FMODE_WRITE) - put_write_access(inode); - file_list_lock(); - file->f_dentry = NULL; - file->f_vfsmnt = NULL; - list_del(&file->f_list); - list_add(&file->f_list, &free_list); - files_stat.nr_free_files++; - file_list_unlock(); - dput(dentry); - mntput(mnt); - } + if (file->f_op && file->f_op->release) + file->f_op->release(inode, file); + fops_put(file->f_op); + if (file->f_mode & FMODE_WRITE) + put_write_access(inode); + file_list_lock(); + file->f_dentry = NULL; + file->f_vfsmnt = NULL; + list_del(&file->f_list); + list_add(&file->f_list, &free_list); + files_stat.nr_free_files++; + file_list_unlock(); + dput(dentry); + mntput(mnt); } struct file * fget(unsigned int fd) diff -urN /md0/kernels/2.4/v2.4.18-pre3/fs/nfs/file.c linux.aio/fs/nfs/file.c --- /md0/kernels/2.4/v2.4.18-pre3/fs/nfs/file.c Mon Jan 14 21:06:00 2002 +++ linux.aio/fs/nfs/file.c Wed Jan 23 01:41:06 2002 @@ -39,9 +39,13 @@ static ssize_t nfs_file_write(struct file *, const char *, size_t, loff_t *); static int nfs_file_flush(struct file *); static int nfs_fsync(struct file *, struct dentry *dentry, int datasync); +static int nfs_kvec_write(struct file *file, kvec_cb_t cb, size_t count, loff_t pos); +static int nfs_kvec_read(struct file *file, kvec_cb_t cb, size_t count, loff_t pos); struct file_operations nfs_file_operations = { llseek: generic_file_llseek, + kvec_read: nfs_kvec_read, + kvec_write: nfs_kvec_write, read: nfs_file_read, write: nfs_file_write, mmap: nfs_file_mmap, @@ -50,6 +54,8 @@ release: nfs_release, fsync: nfs_fsync, lock: nfs_lock, + aio_read: generic_file_aio_read, + aio_write: generic_file_aio_write, }; struct inode_operations nfs_file_inode_operations = { @@ -88,6 +94,28 @@ return status; } +static int nfs_kvec_write(struct file *file, kvec_cb_t cb, size_t count, loff_t pos) +{ + struct dentry * dentry = file->f_dentry; + struct inode * inode = dentry->d_inode; + int ret; + ret = nfs_revalidate_inode(NFS_SERVER(inode), inode); + if (!ret) + return generic_file_kvec_write(file, cb, count, pos); + return ret; +} + +static int nfs_kvec_read(struct file *file, kvec_cb_t cb, size_t count, loff_t pos) +{ + struct dentry * dentry = file->f_dentry; + struct inode * inode = dentry->d_inode; + int ret; + ret = nfs_revalidate_inode(NFS_SERVER(inode), inode); + if (!ret) + return generic_file_kvec_read(file, cb, count, pos); + return ret; +} + static ssize_t nfs_file_read(struct file * file, char * buf, size_t count, loff_t *ppos) { diff -urN /md0/kernels/2.4/v2.4.18-pre3/fs/select.c linux.aio/fs/select.c --- /md0/kernels/2.4/v2.4.18-pre3/fs/select.c Mon Sep 24 02:16:05 2001 +++ linux.aio/fs/select.c Wed Jan 23 01:41:06 2002 @@ -12,6 +12,10 @@ * 24 January 2000 * Changed sys_poll()/do_poll() to use PAGE_SIZE chunk-based allocation * of fds to overcome nfds < 16390 descriptors limit (Tigran Aivazian). + * June 2001 + * Added async_poll implementation. -bcrl + * Nov 2001 + * Async poll improvments from Suparna Bhattacharya */ #include @@ -19,6 +23,8 @@ #include #include /* for STICKY_TIMEOUTS */ #include +#include +#include #include @@ -26,19 +32,36 @@ #define DEFAULT_POLLMASK (POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM) struct poll_table_entry { - struct file * filp; wait_queue_t wait; wait_queue_head_t * wait_address; + struct file * filp; + poll_table * p; }; struct poll_table_page { + unsigned long size; struct poll_table_page * next; struct poll_table_entry * entry; struct poll_table_entry entries[0]; }; #define POLL_TABLE_FULL(table) \ - ((unsigned long)((table)->entry+1) > PAGE_SIZE + (unsigned long)(table)) + ((unsigned long)((table)->entry+1) > \ + (table)->size + (unsigned long)(table)) + +/* async poll uses only one entry per poll table as it is linked to an iocb */ +typedef struct async_poll_table_struct { + poll_table pt; + struct worktodo wtd; + int events; /* event mask for async poll */ + int wake; + long sync; + struct poll_table_page pt_page; /* one poll table page hdr */ + struct poll_table_entry entries[1]; /* space for a single entry */ +} async_poll_table; + + +static kmem_cache_t *async_poll_table_cache; /* * Ok, Peter made a complicated, but straightforward multiple_wait() function. @@ -61,6 +84,8 @@ struct poll_table_page *old; entry = p->entry; + if (entry == p->entries) /* may happen with async poll */ + break; do { entry--; remove_wait_queue(entry->wait_address,&entry->wait); @@ -68,10 +93,83 @@ } while (entry > p->entries); old = p; p = p->next; - free_page((unsigned long) old); + if (old->size == PAGE_SIZE) + free_page((unsigned long) old); + } + if (pt->iocb) /* async poll */ + kmem_cache_free(async_poll_table_cache, pt); +} + +void async_poll_complete(void *data) +{ + async_poll_table *pasync = data; + poll_table *p = data; + struct kiocb *iocb = p->iocb; + unsigned int mask; + + pasync->wake = 0; + wmb(); + do { + mask = iocb->filp->f_op->poll(iocb->filp, p); + mask &= pasync->events | POLLERR | POLLHUP; + if (mask) { + poll_freewait(p); + aio_complete(iocb, mask, 0); + return; + } + pasync->sync = 0; + wmb(); + } while (pasync->wake); + +} + +static void async_poll_waiter(wait_queue_t *wait) +{ + struct poll_table_entry *entry = (struct poll_table_entry *)wait; + async_poll_table *pasync = (async_poll_table *)(entry->p); + + /* avoid writes to the cacheline if possible for SMP */ + if (!pasync->wake) { + pasync->wake = 1; + /* ensure only one wake up queues the wtd */ + if (!pasync->sync && !test_and_set_bit(0, &pasync->sync)) + wtd_queue(&pasync->wtd); } } +int async_poll(struct kiocb *iocb, int events) +{ + unsigned int mask; + async_poll_table *pasync; + poll_table *p; + + pasync = kmem_cache_alloc(async_poll_table_cache, SLAB_KERNEL); + if (!pasync) + return -ENOMEM; + + p = (poll_table *)pasync; + poll_initwait(p); + wtd_set_action(&pasync->wtd, async_poll_complete, pasync); + p->iocb = iocb; + pasync->wake = 0; + pasync->sync = 0; + pasync->events = events; + pasync->pt_page.entry = pasync->pt_page.entries; + pasync->pt_page.size = sizeof(pasync->pt_page); + p->table = &pasync->pt_page; + + mask = DEFAULT_POLLMASK; + if (iocb->filp->f_op && iocb->filp->f_op->poll) + mask = iocb->filp->f_op->poll(iocb->filp, p); + mask &= events | POLLERR | POLLHUP; + if (mask) { + poll_freewait(p); + aio_complete(iocb, mask, 0); + } + + return 0; +} + void __pollwait(struct file * filp, wait_queue_head_t * wait_address, poll_table *p) { struct poll_table_page *table = p->table; @@ -85,6 +183,7 @@ __set_current_state(TASK_RUNNING); return; } + new_table->size = PAGE_SIZE; new_table->entry = new_table->entries; new_table->next = table; p->table = new_table; @@ -98,7 +197,11 @@ get_file(filp); entry->filp = filp; entry->wait_address = wait_address; - init_waitqueue_entry(&entry->wait, current); + entry->p = p; + if (p->iocb) /* async poll */ + init_waitqueue_func_entry(&entry->wait, async_poll_waiter); + else + init_waitqueue_entry(&entry->wait, current); add_wait_queue(wait_address,&entry->wait); } } @@ -494,3 +597,14 @@ poll_freewait(&table); return err; } + +static int __init async_poll_init(void) +{ + async_poll_table_cache = kmem_cache_create("async poll table", + sizeof(async_poll_table), 0, 0, NULL, NULL); + if (!async_poll_table_cache) + panic("unable to alloc poll_table_cache"); + return 0; +} + +module_init(async_poll_init); diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/asm-i386/a.out.h linux.aio/include/asm-i386/a.out.h --- /md0/kernels/2.4/v2.4.18-pre3/include/asm-i386/a.out.h Fri Jun 16 14:33:06 1995 +++ linux.aio/include/asm-i386/a.out.h Wed Jan 23 01:41:06 2002 @@ -19,7 +19,9 @@ #ifdef __KERNEL__ -#define STACK_TOP TASK_SIZE +#define VSYSCALL_SIZE 0x10000 /* 64KB for vsyscalls */ +#define STACK_GUARD_SIZE 0x02000 /* 8KB guard area */ +#define STACK_TOP (TASK_SIZE - VSYSCALL_SIZE - STACK_GUARD_SIZE) #endif diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/asm-i386/unistd.h linux.aio/include/asm-i386/unistd.h --- /md0/kernels/2.4/v2.4.18-pre3/include/asm-i386/unistd.h Thu Nov 1 16:40:03 2001 +++ linux.aio/include/asm-i386/unistd.h Wed Jan 23 01:41:06 2002 @@ -231,6 +231,8 @@ #define __NR_gettid 224 #define __NR_readahead 225 +#define __NR_sys_dynamic_syscall 230 + /* user-visible error numbers are in the range -1 - -124: see */ #define __syscall_return(type, res) \ diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/asm-i386/vsyscall.h linux.aio/include/asm-i386/vsyscall.h --- /md0/kernels/2.4/v2.4.18-pre3/include/asm-i386/vsyscall.h Wed Dec 31 19:00:00 1969 +++ linux.aio/include/asm-i386/vsyscall.h Wed Jan 23 01:41:06 2002 @@ -0,0 +1,41 @@ +#ifndef __ASM__VSYSCALL_H +#define __ASM__VSYSCALL_H +/* include/asm-i386/vsyscall.h + * Copyright 2002 Red Hat, Inc. + */ +#include +#include + +/* We call sys_dynamic_syscall(long nr, void *args) using regparm(2) + * convention. The .text.vsyscall section is mapped into userspace, + * whereas .data.vsyscall_list is a kernel-only array of the vsyscalls + * and the valid userspace address to call them from. All vsyscalls + * are called with C calling convention (ie args on the stack for x86). + * + * Note: the layout of .data.vsyscall_list must match the entries in + * dynamic_syscall.c. + */ +#define STRINGIFYa(x) #x +#define STRINGIFY(x) STRINGIFYa(x) +#define NR_dyn_sys STRINGIFY(__NR_sys_dynamic_syscall) +#define add_dynamic_syscall(name) \ + __asm__(" \n\ + .section .vsyscall_text, \"xa\" \n\ + .globl v" #name " \n\ + v" #name ": \n\ + push %ecx \n\ + push %edx \n\ + movl $" NR_dyn_sys ",%eax \n\ + movl $2f,%edx \n\ + leal 12(%esp),%ecx \n\ + int $0x80 \n\ + 1: \n\ + popl %edx \n\ + popl %ecx \n\ + ret \n\ + .size v" #name ",.-v" #name " \n\ + .section .data.vsyscall_list,\"a\" \n\ + 2: .long 1b," #name " \n\ + .previous") + +#endif diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/aio.h linux.aio/include/linux/aio.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/aio.h Wed Dec 31 19:00:00 1969 +++ linux.aio/include/linux/aio.h Wed Jan 23 01:41:06 2002 @@ -0,0 +1,74 @@ +#ifndef __LINUX__AIO_H +#define __LINUX__AIO_H + +#include +#include +#include + +#define AIO_MAXSEGS 4 +#define AIO_KIOGRP_NR_ATOMIC 8 + +struct kioctx; + +struct kiocb { + struct list_head list; + int (*cancel)(struct kioctx *ctx, struct kiocb *kiocb); + struct file *filp; + struct kioctx *ctx; + void *user_obj; + __u64 user_data; + ssize_t nr_atomic; + unsigned key; /* id of this request */ +}; + +#define AIO_RING_PAGES 8 +#define AIO_RING_PAGES 8 +struct kioctx { + atomic_t users; + int dead; + + /* This needs improving */ + unsigned long user_id; + struct kioctx *next; + + wait_queue_head_t wait; + + spinlock_t lock; + + int reqs_active; + struct list_head free_reqs; + struct list_head active_reqs; /* used for cancellation */ + + unsigned max_reqs; + unsigned ring_mask; + struct aio_ring *ring; + int ring_was_vmallocd; + int ring_order; +}; + +extern struct file_operations aio_fops; + +extern void FASTCALL(aio_complete(struct kiocb *iocb, long res, long res2)); +extern void __put_ioctx(struct kioctx *ctx); +struct mm_struct; +extern void exit_aio(struct mm_struct *mm); + +#define get_ioctx(kioctx) atomic_inc(&(kioctx)->users) +#define put_ioctx(kioctx) do { if (atomic_dec_and_test(&(kioctx)->users)) __put_ioctx(kioctx); } while (0) + +#include + +static inline struct kiocb *list_kiocb(struct list_head *h) +{ + return list_entry(h, struct kiocb, list); +} + +struct file; +extern int generic_aio_poll(struct file *file, struct kiocb *req, struct iocb iocb); +extern ssize_t generic_aio_read(struct file *file, struct kiocb *req, struct iocb iocb, size_t min_size); +extern ssize_t generic_aio_write(struct file *file, struct kiocb *req, struct iocb iocb, size_t min_size); +extern ssize_t generic_file_aio_read(struct file *file, struct kiocb *req, struct iocb iocb); +extern ssize_t generic_file_aio_write(struct file *file, struct kiocb *req, struct iocb iocb); +extern ssize_t generic_sock_aio_read(struct file *file, struct kiocb *req, struct iocb iocb); + +#endif /* __LINUX__AIO_H */ diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/aio_abi.h linux.aio/include/linux/aio_abi.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/aio_abi.h Wed Dec 31 19:00:00 1969 +++ linux.aio/include/linux/aio_abi.h Wed Jan 23 01:41:06 2002 @@ -0,0 +1,101 @@ +/* linux/aio_abi.h + * + * Copyright 2000,2001,2002 Red Hat. + * + * Written by Benjamin LaHaise + * + * Permission to use, copy, modify, and distribute this software and its + * documentation is hereby granted, provided that the above copyright + * notice appears in all copies. This software is provided without any + * warranty, express or implied. Red Hat makes no representations about + * the suitability of this software for any purpose. + * + * IN NO EVENT SHALL RED HAT BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, + * SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF + * THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF RED HAT HAS BEEN ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * RED HAT DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND + * RED HAT HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, + * ENHANCEMENTS, OR MODIFICATIONS. + */ +#ifndef __LINUX__AIO_ABI_H +#define __LINUX__AIO_ABI_H + +#include + +typedef unsigned long aio_context_t; + +enum { + IOCB_CMD_PREAD = 0, + IOCB_CMD_PWRITE = 1, + IOCB_CMD_FSYNC = 2, + IOCB_CMD_FDSYNC = 3, + IOCB_CMD_PREADX = 4, + IOCB_CMD_POLL = 5, +}; + +/* read() from /dev/aio returns these structures. */ +struct io_event { + __u64 data; /* the data field from the iocb */ + __u64 obj; /* what iocb this event came from */ + __s64 res; /* result code for this event */ + __s64 res2; /* secondary result */ +}; + +#if defined(__LITTLE_ENDIAN) +#define PADDED(x,y) x, y +#elif defined(__BIG_ENDIAN) +#define PADDED(x,y) y, x +#else +#error edit for your odd byteorder. +#endif + +struct aio_ring { + __u32 PADDED(id, pad1); /* kernel internal index number */ + __u32 PADDED(mask, pad2); /* number of io_events - 1 */ + __u32 PADDED(head, pad3); + __u32 PADDED(tail, pad4); + + __u32 PADDED(woke, pad5); /* set when a wakeup was sent */ + + __u32 pad6[22]; /* pad out to 128 bytes */ + + struct io_event io_events[0]; +}; /* 128 bytes + ring size */ + +#define aio_ring_avail(ring) (((ring)->head + (ring)->mask - (ring)->tail) & (ring)->mask) + +/* + * we always use a 64bit off_t when communicating + * with userland. its up to libraries to do the + * proper padding and aio_error abstraction + */ + +struct iocb { + /* these are internal to the kernel/libc. */ + __u64 aio_data; /* data to be returned in event's data */ + __u32 PADDED(aio_key, aio_reserved1); + /* the kernel sets aio_key to the req # */ + + /* common fields */ + __u16 aio_lio_opcode; /* see IOCB_CMD_ above */ + __s16 aio_reqprio; + __u32 aio_fildes; + + __u64 aio_buf; + __u64 aio_nbytes; + __s64 aio_offset; + + /* extra parameters */ + __u64 aio_reserved2; + __u64 aio_reserved3; +}; /* 64 bytes */ + +#undef IFBIG +#undef IFLITTLE + +#endif /* __LINUX__AIO_ABI_H */ + diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/errno.h linux.aio/include/linux/errno.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/errno.h Tue Nov 6 20:40:27 2001 +++ linux.aio/include/linux/errno.h Wed Jan 23 01:41:06 2002 @@ -21,6 +21,9 @@ #define EBADTYPE 527 /* Type not supported by server */ #define EJUKEBOX 528 /* Request initiated, but will not complete before timeout */ +/* Defined for TUX async IO */ +#define EWOULDBLOCKIO 530 /* Would block due to block-IO */ + #endif #endif diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/file.h linux.aio/include/linux/file.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/file.h Wed Aug 23 14:22:26 2000 +++ linux.aio/include/linux/file.h Wed Jan 23 01:41:06 2002 @@ -5,6 +5,7 @@ #ifndef __LINUX_FILE_H #define __LINUX_FILE_H +extern void FASTCALL(__fput(struct file *)); extern void FASTCALL(fput(struct file *)); extern struct file * FASTCALL(fget(unsigned int fd)); diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/fs.h linux.aio/include/linux/fs.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/fs.h Mon Jan 14 17:30:21 2002 +++ linux.aio/include/linux/fs.h Wed Jan 23 01:41:06 2002 @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include @@ -813,11 +815,17 @@ * read, write, poll, fsync, readv, writev can be called * without the big kernel lock held in all filesystems. */ + +#define F_ATOMIC 0x0001 +#define F_OFFSETOK 0x0002 + struct file_operations { struct module *owner; loff_t (*llseek) (struct file *, loff_t, int); ssize_t (*read) (struct file *, char *, size_t, loff_t *); + ssize_t (*new_read) (struct file *, char *, size_t, loff_t *, int); ssize_t (*write) (struct file *, const char *, size_t, loff_t *); + ssize_t (*new_write) (struct file *, const char *, size_t, loff_t *, int); int (*readdir) (struct file *, void *, filldir_t); unsigned int (*poll) (struct file *, struct poll_table_struct *); int (*ioctl) (struct inode *, struct file *, unsigned int, unsigned long); @@ -832,6 +840,16 @@ ssize_t (*writev) (struct file *, const struct iovec *, unsigned long, loff_t *); ssize_t (*sendpage) (struct file *, struct page *, int, size_t, loff_t *, int); unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); + + /* in-kernel fully async api */ + int (*kvec_read)(struct file *, kvec_cb_t, size_t, loff_t); + int (*kvec_write)(struct file *, kvec_cb_t, size_t, loff_t); + + /* userland aio ops */ + ssize_t (*aio_read)(struct file *, struct kiocb *, struct iocb); + ssize_t (*aio_readx)(struct file *, struct kiocb *, struct iocb); + ssize_t (*aio_write)(struct file *, struct kiocb *, struct iocb); + ssize_t (*aio_fsync)(struct file *, struct kiocb *, struct iocb); }; struct inode_operations { @@ -1396,12 +1414,16 @@ extern int generic_file_mmap(struct file *, struct vm_area_struct *); extern int file_read_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size); extern ssize_t generic_file_read(struct file *, char *, size_t, loff_t *); +extern ssize_t generic_file_new_read(struct file *, char *, size_t, loff_t *, int); extern ssize_t generic_file_write(struct file *, const char *, size_t, loff_t *); -extern void do_generic_file_read(struct file *, loff_t *, read_descriptor_t *, read_actor_t); -extern loff_t no_llseek(struct file *file, loff_t offset, int origin); -extern loff_t generic_file_llseek(struct file *file, loff_t offset, int origin); +extern void do_generic_file_read(struct file *, loff_t *, read_descriptor_t *, read_actor_t, int); +extern int generic_file_kvec_read(struct file *file, kvec_cb_t cb, size_t size, loff_t pos); +extern int generic_file_kvec_write(struct file *file, kvec_cb_t cb, size_t size, loff_t pos); + extern ssize_t generic_read_dir(struct file *, char *, size_t, loff_t *); extern int generic_file_open(struct inode * inode, struct file * filp); +extern loff_t no_llseek(struct file *file, loff_t offset, int origin); +extern loff_t generic_file_llseek(struct file *file, loff_t offset, int origin); extern struct file_operations generic_ro_fops; diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/iobuf.h linux.aio/include/linux/iobuf.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/iobuf.h Mon Jan 14 21:10:07 2002 +++ linux.aio/include/linux/iobuf.h Wed Jan 23 01:41:06 2002 @@ -53,8 +53,10 @@ /* Dynamic state for IO completion: */ atomic_t io_count; /* IOs still in progress */ + int transferred; /* Number of bytes of completed IO at the beginning of the buffer */ int errno; /* Status of completed IO */ void (*end_io) (struct kiobuf *); /* Completion callback */ + void *end_io_data; wait_queue_head_t wait_queue; }; @@ -80,6 +82,8 @@ /* fs/buffer.c */ +int brw_kiovec_async(int rw, int nr, struct kiobuf *iovec[], + kdev_t dev, int nr_blocks, unsigned long b[], int size); int brw_kiovec(int rw, int nr, struct kiobuf *iovec[], kdev_t dev, unsigned long b[], int size); diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/kiovec.h linux.aio/include/linux/kiovec.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/kiovec.h Wed Dec 31 19:00:00 1969 +++ linux.aio/include/linux/kiovec.h Wed Jan 23 01:41:06 2002 @@ -0,0 +1,121 @@ +#ifndef __LINUX__KIOVEC_H +#define __LINUX__KIOVEC_H + +struct page; +struct list; + +struct kveclet { + struct page *page; + unsigned offset; + unsigned length; +}; + +struct kvec { + unsigned max_nr; + unsigned nr; + struct kveclet veclet[0]; +}; + +struct kvec_cb { + struct kvec *vec; + void (*fn)(void *data, struct kvec *vec, ssize_t res); + void *data; +}; + +struct kvec_cb_list { + struct list_head list; + struct kvec_cb cb; +}; + +#ifndef _LINUX_TYPES_H +#include +#endif +#ifndef _LINUX_KDEV_T_H +#include +#endif +#ifndef _ASM_KMAP_TYPES_H +#include +#endif + +extern struct kvec *map_user_kvec(int rw, unsigned long va, size_t len); +extern void unmap_kvec(struct kvec *, int dirtied); +extern void free_kvec(struct kvec *); + +/* brw_kvec_async: + * Performs direct io to/from disk into cb.vec. Count is the number + * of sectors to read, sector_shift is the blocksize (which must be + * compatible with the kernel's current idea of the device's sector + * size) in log2. blknr is the starting sector offset on dev. + * + */ +extern int brw_kvec_async(int rw, kvec_cb_t cb, kdev_t dev, unsigned count, + unsigned long blknr, int sector_shift); + +/* Memory copy helpers usage: + * void foo(... struct kveclet *veclet...) + * + * struct kvec_dst dst; + * + * kvec_dst_init(&dst, KM_USER0); -- resets type + * kvec_set_set(&dst, veclet); -- set target & clear offset + * kvec_dst_map(&dst); -- activates kmap + * for (...) + * memcpy_to_kvec_dst(&dst, data, size); -- each copy appends + * kvec_dst_unmap(&dst); -- releases kmap + * + * Note that scheduling is not permitted between kvec_dst_map() and + * kvec_dst_unmap(). This is because internally the routines make use + * of an atomic kmap. + */ +struct kvec_dst { + char *addr; + char *dst; + struct kveclet *let; + int space; + int offset; + enum km_type type; +}; + + +#define kvec_dst_set(Xdst, Xlet) \ + do { \ + struct kvec_dst *_dst = (Xdst); \ + struct kveclet *_let = (Xlet); \ + _dst->let = _let; \ + _dst->space = _let->length; \ + _dst->offset = 0; \ + } while(0) + +#define kvec_dst_map(Xdst) \ + do { \ + struct kvec_dst *_dst = (Xdst); \ + struct kveclet *_let = _dst->let; \ + _dst->dst = _dst->addr = kmap_atomic(_let->page, _dst->type);\ + _dst->dst += _let->offset + _dst->offset; \ + _dst->space = _let->length - _dst->offset; \ + _dst->offset = 0; \ + } while(0) + +#define kvec_dst_init(Xdst, Xtype) \ + do { \ + (Xdst)->space = 0; \ + (Xdst)->addr = 0; \ + (Xdst)->offset = 0; \ + (Xdst)->type = Xtype; \ + } while(0) + +#define kvec_dst_unmap(Xdst) \ + do { \ + struct kvec_dst *_dst = (Xdst); \ + kunmap_atomic(_dst->addr, _dst->type); \ + _dst->offset = _dst->dst - _dst->addr; \ + _dst->offset -= _dst->let->offset; \ + _dst->addr = NULL; \ + } while(0) + +extern void FASTCALL(memcpy_to_kvec_dst(struct kvec_dst *dst, + const char *from, long len)); +extern void FASTCALL(memcpy_from_kvec_dst(char *to, + struct kvec_dst *from, long len)); + +#endif /* __LINUX__KIOVEC_H */ diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/poll.h linux.aio/include/linux/poll.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/poll.h Mon Jan 14 21:10:19 2002 +++ linux.aio/include/linux/poll.h Wed Jan 23 01:41:06 2002 @@ -9,12 +9,15 @@ #include #include #include +#include struct poll_table_page; +struct kiocb; typedef struct poll_table_struct { - int error; - struct poll_table_page * table; + int error; + struct poll_table_page *table; + struct kiocb *iocb; /* iocb for async poll */ } poll_table; extern void __pollwait(struct file * filp, wait_queue_head_t * wait_address, poll_table *p); @@ -29,8 +32,11 @@ { pt->error = 0; pt->table = NULL; + pt->iocb = NULL; } + extern void poll_freewait(poll_table* pt); +extern int async_poll(struct kiocb *iocb, int events); /* diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/sched.h linux.aio/include/linux/sched.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/sched.h Mon Jan 14 21:10:07 2002 +++ linux.aio/include/linux/sched.h Wed Jan 23 01:41:06 2002 @@ -201,6 +201,7 @@ /* Maximum number of active map areas.. This is a random (large) number */ #define MAX_MAP_COUNT (65536) +struct kioctx; struct mm_struct { struct vm_area_struct * mmap; /* list of VMAs */ rb_root_t mm_rb; @@ -229,6 +230,10 @@ /* Architecture-specific MM context */ mm_context_t context; + + struct kioctx *ioctx_list; + unsigned long new_ioctx_id; + int vsys_mapped; }; extern int mmlist_nr; @@ -241,6 +246,7 @@ mm_count: ATOMIC_INIT(1), \ mmap_sem: __RWSEM_INITIALIZER(name.mmap_sem), \ page_table_lock: SPIN_LOCK_UNLOCKED, \ + vsys_mapped: 0, \ mmlist: LIST_HEAD_INIT(name.mmlist), \ } @@ -791,6 +797,7 @@ extern void FASTCALL(add_wait_queue(wait_queue_head_t *q, wait_queue_t * wait)); extern void FASTCALL(add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t * wait)); +extern void FASTCALL(add_wait_queue_exclusive_lifo(wait_queue_head_t *q, wait_queue_t * wait)); extern void FASTCALL(remove_wait_queue(wait_queue_head_t *q, wait_queue_t * wait)); #define __wait_event(wq, condition) \ diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/tqueue.h linux.aio/include/linux/tqueue.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/tqueue.h Mon Jan 14 21:10:07 2002 +++ linux.aio/include/linux/tqueue.h Wed Jan 23 01:41:06 2002 @@ -67,6 +67,7 @@ #define TQ_ACTIVE(q) (!list_empty(&q)) extern task_queue tq_timer, tq_immediate, tq_disk; +extern struct tq_struct run_disk_tq; /* * To implement your own list of active bottom halfs, use the following diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/types.h linux.aio/include/linux/types.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/types.h Mon Jan 14 21:10:07 2002 +++ linux.aio/include/linux/types.h Wed Jan 23 01:41:06 2002 @@ -127,4 +127,9 @@ char f_fpack[6]; }; +/* kernel typedefs -- they belong here. */ +#ifdef __KERNEL__ +typedef struct kvec_cb kvec_cb_t; +#endif /* __KERNEL__ */ + #endif /* _LINUX_TYPES_H */ diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/vsyscall.h linux.aio/include/linux/vsyscall.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/vsyscall.h Wed Dec 31 19:00:00 1969 +++ linux.aio/include/linux/vsyscall.h Wed Jan 23 01:41:06 2002 @@ -0,0 +1,20 @@ +#ifndef _LINUX__VSYSCALL_H +#define _LINUX__VSYSCALL_H + +struct vsys_cpudata { + unsigned long context_switches; + unsigned long tv_sec; + unsigned long tsc_low, tsc_high; + unsigned long cycles_per_sec; +}; + +union vsys_union { + struct vsys_cpudata data; + char pad[128]; +}; + +extern union vsys_union vsys_cpudata[256] __attribute__((section(".data.vsyscall"))); + +#define vsys_data(cpu) (&vsys_cpudata[cpu].data) + +#endif /*ndef _LINUX__VSYSCALL_H*/ diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/wait.h linux.aio/include/linux/wait.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/wait.h Mon Jan 14 21:10:07 2002 +++ linux.aio/include/linux/wait.h Wed Jan 23 01:41:06 2002 @@ -28,17 +28,20 @@ #define WAITQUEUE_DEBUG 0 #endif +typedef struct __wait_queue wait_queue_t; +typedef void (*wait_queue_func_t)(wait_queue_t *wait); + struct __wait_queue { unsigned int flags; #define WQ_FLAG_EXCLUSIVE 0x01 struct task_struct * task; struct list_head task_list; + wait_queue_func_t func; #if WAITQUEUE_DEBUG long __magic; long __waker; #endif }; -typedef struct __wait_queue wait_queue_t; /* * 'dual' spinlock architecture. Can be switched between spinlock_t and @@ -137,6 +140,7 @@ #endif #define __WAITQUEUE_INITIALIZER(name, tsk) { \ + func: NULL, \ task: tsk, \ task_list: { NULL, NULL }, \ __WAITQUEUE_DEBUG_INIT(name)} @@ -174,6 +178,22 @@ #endif q->flags = 0; q->task = p; + q->func = NULL; +#if WAITQUEUE_DEBUG + q->__magic = (long)&q->__magic; +#endif +} + +static inline void init_waitqueue_func_entry(wait_queue_t *q, + wait_queue_func_t func) +{ +#if WAITQUEUE_DEBUG + if (!q || !p) + WQ_BUG(); +#endif + q->flags = 0; + q->task = NULL; + q->func = func; #if WAITQUEUE_DEBUG q->__magic = (long)&q->__magic; #endif @@ -231,6 +251,20 @@ list_del(&old->task_list); } +#define add_wait_queue_cond(q, wait, cond) \ + ({ \ + unsigned long flags; \ + int _raced = 0; \ + wq_write_lock_irqsave(&(q)->lock, flags); \ + (wait)->flags = 0; \ + if (cond) \ + __add_wait_queue((q), (wait)); \ + else \ + _raced = 1; \ + wq_write_unlock_irqrestore(&(q)->lock, flags); \ + _raced; \ + }) + #endif /* __KERNEL__ */ #endif diff -urN /md0/kernels/2.4/v2.4.18-pre3/include/linux/worktodo.h linux.aio/include/linux/worktodo.h --- /md0/kernels/2.4/v2.4.18-pre3/include/linux/worktodo.h Wed Dec 31 19:00:00 1969 +++ linux.aio/include/linux/worktodo.h Wed Jan 23 01:41:06 2002 @@ -0,0 +1,76 @@ +/* + * Written by Benjamin LaHaise. + * + * Copyright 2000-2001 Red Hat, Inc. + * + * #include "gpl.h" + * + * Basic design idea from Jeff Merkey. + * Stack based on ideas from Ingo Molnar. + */ +#ifndef __LINUX__WORKTODO_H +#define __LINUX__WORKTODO_H + +#ifndef _LINUX_WAIT_H +#include +#endif +#ifndef _LINUX_TQUEUE_H +#include +#endif + +struct wtd_stack { + void (*fn)(void *data); + void *data; +}; + +struct worktodo { + wait_queue_t wait; + struct tq_struct tq; + + void *data; /* for use by the wtd_ primatives */ + + int sp; + struct wtd_stack stack[3]; +}; + +/* FIXME NOTE: factor from kernel/context.c */ +#define wtd_init(wtd, routine) do { \ + INIT_TQUEUE(&(wtd)->tq, (routine), (wtd)); \ + (wtd)->data = 0; \ + (wtd)->sp = 0; \ +} while (0) + +#define wtd_queue(wtd) schedule_task(&(wtd)->tq) + +#define wtd_push(wtd, action, wtddata) \ +do { \ + (wtd)->stack[(wtd)->sp].fn = (wtd)->tq.routine; \ + (wtd)->stack[(wtd)->sp++].data = (wtd)->tq.data;\ + (wtd)->tq.routine = action; \ + (wtd)->tq.data = wtddata; \ +} while (0) + +static inline void wtd_pop(struct worktodo *wtd) +{ + if (wtd->sp) { + wtd->sp--; + wtd->tq.routine = wtd->stack[wtd->sp].fn; + wtd->tq.data = wtd->stack[wtd->sp].data; + } +} + +#define wtd_set_action(wtd, action, wtddata) INIT_TQUEUE(&(wtd)->tq, action, wtddata) + +struct page; +extern void wtd_wait_page(struct worktodo *wtd, struct page *page); +extern void wtd_lock_page(struct worktodo *wtd, struct page *page); +struct buffer_head; +extern void wtd_wait_on_buffer(struct worktodo *wtd, struct buffer_head *bh); + +#if 0 /* not implemented yet */ +extern void wtd_down(struct worktodo *wtd, struct semaphore *sem); +extern void wtd_down_write(struct worktodo *wtd, struct rw_semaphore *sem); +extern void wtd_down_read(struct worktodo *wtd, struct rw_semaphore *sem); +#endif + +#endif /* __LINUX__WORKTODO_H */ diff -urN /md0/kernels/2.4/v2.4.18-pre3/kernel/context.c linux.aio/kernel/context.c --- /md0/kernels/2.4/v2.4.18-pre3/kernel/context.c Thu Nov 1 16:40:03 2001 +++ linux.aio/kernel/context.c Wed Jan 23 01:41:06 2002 @@ -95,11 +95,17 @@ for (;;) { set_task_state(curtask, TASK_INTERRUPTIBLE); add_wait_queue(&context_task_wq, &wait); - if (TQ_ACTIVE(tq_context)) + if (spin_is_locked(&tqueue_lock) || TQ_ACTIVE(tq_context)) set_task_state(curtask, TASK_RUNNING); - schedule(); + else + schedule(); remove_wait_queue(&context_task_wq, &wait); run_task_queue(&tq_context); + while (TQ_ACTIVE(tq_context)) { + if (current->need_resched) + schedule(); + run_task_queue(&tq_context); + } wake_up(&context_task_done); if (signal_pending(curtask)) { while (waitpid(-1, (unsigned int *)0, __WALL|WNOHANG) > 0) diff -urN /md0/kernels/2.4/v2.4.18-pre3/kernel/fork.c linux.aio/kernel/fork.c --- /md0/kernels/2.4/v2.4.18-pre3/kernel/fork.c Mon Jan 14 21:06:00 2002 +++ linux.aio/kernel/fork.c Wed Jan 23 01:41:06 2002 @@ -46,6 +46,16 @@ wq_write_unlock_irqrestore(&q->lock, flags); } +void add_wait_queue_exclusive_lifo(wait_queue_head_t *q, wait_queue_t * wait) +{ + unsigned long flags; + + wq_write_lock_irqsave(&q->lock, flags); + wait->flags = WQ_FLAG_EXCLUSIVE; + __add_wait_queue(q, wait); + wq_write_unlock_irqrestore(&q->lock, flags); +} + void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t * wait) { unsigned long flags; diff -urN /md0/kernels/2.4/v2.4.18-pre3/kernel/sched.c linux.aio/kernel/sched.c --- /md0/kernels/2.4/v2.4.18-pre3/kernel/sched.c Tue Jan 1 14:09:35 2002 +++ linux.aio/kernel/sched.c Wed Jan 23 01:41:06 2002 @@ -705,33 +705,44 @@ } /* - * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just wake everything - * up. If it's an exclusive wakeup (nr_exclusive == small +ve number) then we wake all the - * non-exclusive tasks and one exclusive task. + * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just + * wake everything up. If it's an exclusive wakeup (nr_exclusive == small + * +ve number) then we wake all the non-exclusive tasks and one exclusive task. * * There are circumstances in which we can try to wake a task which has already - * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns zero - * in this (rare) case, and we handle it by contonuing to scan the queue. + * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns + * zero in this (rare) case, and we handle it by contonuing to scan the queue. */ static inline void __wake_up_common (wait_queue_head_t *q, unsigned int mode, int nr_exclusive, const int sync) { - struct list_head *tmp; + struct list_head *tmp, *next; struct task_struct *p; CHECK_MAGIC_WQHEAD(q); WQ_CHECK_LIST_HEAD(&q->task_list); - list_for_each(tmp,&q->task_list) { + list_for_each_safe(tmp, next, &q->task_list) { unsigned int state; - wait_queue_t *curr = list_entry(tmp, wait_queue_t, task_list); + wait_queue_t *curr = list_entry(tmp, wait_queue_t, task_list); + wait_queue_func_t func; CHECK_MAGIC(curr->__magic); + func = curr->func; + if (func) { + unsigned flags = curr->flags; + func(curr); + if ((flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive) + break; + continue; + } p = curr->task; state = p->state; if (state & mode) { WQ_NOTE_WAKER(curr); - if (try_to_wake_up(p, sync) && (curr->flags&WQ_FLAG_EXCLUSIVE) && !--nr_exclusive) + if (try_to_wake_up(p, sync) && + (curr->flags & WQ_FLAG_EXCLUSIVE) && + !--nr_exclusive) break; } } diff -urN /md0/kernels/2.4/v2.4.18-pre3/kernel/softirq.c linux.aio/kernel/softirq.c --- /md0/kernels/2.4/v2.4.18-pre3/kernel/softirq.c Mon Nov 12 17:49:51 2001 +++ linux.aio/kernel/softirq.c Wed Jan 23 01:41:06 2002 @@ -354,6 +354,7 @@ data = p->data; wmb(); p->sync = 0; + smp_mb(); if (f) f(data); } diff -urN /md0/kernels/2.4/v2.4.18-pre3/mm/Makefile linux.aio/mm/Makefile --- /md0/kernels/2.4/v2.4.18-pre3/mm/Makefile Mon Nov 12 17:49:51 2001 +++ linux.aio/mm/Makefile Wed Jan 23 01:41:06 2002 @@ -17,5 +17,6 @@ shmem.o obj-$(CONFIG_HIGHMEM) += highmem.o +obj-y += wtd.o include $(TOPDIR)/Rules.make diff -urN /md0/kernels/2.4/v2.4.18-pre3/mm/filemap.c linux.aio/mm/filemap.c --- /md0/kernels/2.4/v2.4.18-pre3/mm/filemap.c Mon Jan 14 21:06:00 2002 +++ linux.aio/mm/filemap.c Wed Jan 23 01:41:06 2002 @@ -30,6 +30,8 @@ #include #include +#include +#include /* * Shared mappings implemented 30.11.1994. It's not fully working yet, @@ -801,7 +803,6 @@ if (!test_and_clear_bit(PG_locked, &(page)->flags)) BUG(); smp_mb__after_clear_bit(); - if (waitqueue_active(&(page)->wait)) wake_up(&(page)->wait); } @@ -1145,7 +1146,7 @@ static void generic_file_readahead(int reada_ok, struct file * filp, struct inode * inode, - struct page * page) + struct page * page, int flags) { unsigned long end_index; unsigned long index = page->index; @@ -1275,7 +1276,7 @@ * This is really ugly. But the goto's actually try to clarify some * of the logic when it comes to error handling etc. */ -void do_generic_file_read(struct file * filp, loff_t *ppos, read_descriptor_t * desc, read_actor_t actor) +void do_generic_file_read(struct file * filp, loff_t *ppos, read_descriptor_t * desc, read_actor_t actor, int flags) { struct address_space *mapping = filp->f_dentry->d_inode->i_mapping; struct inode *inode = mapping->host; @@ -1334,13 +1335,17 @@ end_index = inode->i_size >> PAGE_CACHE_SHIFT; - if (index > end_index) + if (index > end_index) { + desc->error = 0; break; + } nr = PAGE_CACHE_SIZE; if (index == end_index) { nr = inode->i_size & ~PAGE_CACHE_MASK; - if (nr <= offset) + if (nr <= offset) { + desc->error = 0; break; + } } nr = nr - offset; @@ -1360,7 +1365,7 @@ if (!Page_Uptodate(page)) goto page_not_up_to_date; - generic_file_readahead(reada_ok, filp, inode, page); + generic_file_readahead(reada_ok, filp, inode, page, flags); page_ok: /* If users can be writing to this page using arbitrary * virtual addresses, take care about potential aliasing @@ -1400,13 +1405,24 @@ * Ok, the page was not immediately readable, so let's try to read ahead while we're at it.. */ page_not_up_to_date: - generic_file_readahead(reada_ok, filp, inode, page); + generic_file_readahead(reada_ok, filp, inode, page, flags); if (Page_Uptodate(page)) goto page_ok; /* Get exclusive access to the page ... */ - lock_page(page); + if (flags & F_ATOMIC) { + if (TryLockPage(page)) { + if (Page_Uptodate(page)) + goto page_ok; + printk("page_not_up_to_date: -EAGAIN\n"); + desc->error = -EWOULDBLOCKIO; + page_cache_release(page); + break; + } + printk("page_not_up_to_date: atomic trylock succeeded\n"); + } else + lock_page(page); /* Did it get unhashed before we got the lock? */ if (!page->mapping) { @@ -1430,11 +1446,12 @@ goto page_ok; /* Again, try some read-ahead while waiting for the page to finish.. */ - generic_file_readahead(reada_ok, filp, inode, page); - wait_on_page(page); + generic_file_readahead(reada_ok, filp, inode, page, flags); + if (!(flags & F_ATOMIC)) + wait_on_page(page); if (Page_Uptodate(page)) goto page_ok; - error = -EIO; + error = (flags & F_ATOMIC) ? -EWOULDBLOCKIO : -EIO; } /* UHHUH! A synchronous read error occurred. Report it */ @@ -1443,6 +1460,11 @@ break; no_cached_page: + if (flags & F_ATOMIC) { + spin_unlock(&pagecache_lock); + desc->error = -EWOULDBLOCKIO; + break; + } /* * Ok, it wasn't cached, so we need to create a new * page.. @@ -1595,6 +1617,11 @@ */ ssize_t generic_file_read(struct file * filp, char * buf, size_t count, loff_t *ppos) { + return generic_file_new_read(filp, buf, count, ppos, 0); +} + +ssize_t generic_file_new_read(struct file * filp, char * buf, size_t count, loff_t *ppos, int flags) +{ ssize_t retval; if ((ssize_t) count < 0) @@ -1614,7 +1641,7 @@ desc.count = count; desc.buf = buf; desc.error = 0; - do_generic_file_read(filp, ppos, &desc, file_read_actor); + do_generic_file_read(filp, ppos, &desc, file_read_actor, flags); retval = desc.written; if (!retval) @@ -1739,7 +1766,7 @@ desc.count = count; desc.buf = (char *) out_file; desc.error = 0; - do_generic_file_read(in_file, ppos, &desc, file_send_actor); + do_generic_file_read(in_file, ppos, &desc, file_send_actor, 0); retval = desc.written; if (!retval) @@ -3097,3 +3124,607 @@ panic("Failed to allocate page hash table\n"); memset((void *)page_hash_table, 0, PAGE_HASH_SIZE * sizeof(struct page *)); } + +/* address_space_map + * Maps a series of pages from the page cache into the given array. + */ +static int address_space_map(struct address_space *as, unsigned long index, + int nr, struct page **pages, + int *nr_newp, struct page **new_pages) +{ + struct page *cached_page = NULL; + int nr_new = 0; + int ret; + + ret = -EINVAL; + if (nr <= 0) + goto out; + + ret = 0; + + spin_lock(&pagecache_lock); + + while (nr > 0) { + struct page **hash = page_hash(as, index); + struct page *page; + + page = __find_page_nolock(as, index, *hash); + if (page) { + page_cache_get(page); +got_page: + pages[ret++] = page; + index++; + nr--; + continue; + } + + if (cached_page) { + __add_to_page_cache(cached_page, as, index, hash); + nr_new++; + *new_pages++ = page = cached_page; + cached_page = NULL; + goto got_page; + } + spin_unlock(&pagecache_lock); + + cached_page = page_cache_alloc(as); + if (!cached_page) + goto out; + + /* Okay, we now have an allocated page. Retry + * the search and add. */ + spin_lock(&pagecache_lock); + } + + spin_unlock(&pagecache_lock); + +out: + if (cached_page) + page_cache_release(cached_page); + + *nr_newp = nr_new; + return ret ? ret : -ENOMEM; +} + +struct iodesc { + struct worktodo wtd; + + struct page *good_page; /* the highest Uptodate page */ + int good_idx; + int err; + int did_read; + int rw; + + struct page **pages; + struct page **new_pages; + struct page **cur_pagep; + int nr_pages; + int nr_new_pages; + + struct address_space *as; + struct file *file; + kvec_cb_t cb; + + size_t size; + unsigned long transferred; + unsigned offset; + struct kveclet *veclet; + + struct kvec_dst src; + + int sync; + +#define READDESC_NR_DEF 3 + struct page *def_pages[READDESC_NR_DEF]; + struct page *def_new_pages[READDESC_NR_DEF]; +}; + +static void __iodesc_free(struct iodesc *io, int unlock) +{ + kvec_cb_t cb; + ssize_t res; + + if (unlock) { + unsigned i; + for (i=0; inr_pages; i++) { + struct page *page = io->pages[i]; + UnlockPage(page); + page_cache_release(page); + } + } else { + unsigned i; + for (i=0; inr_pages; i++) + page_cache_release(io->pages[i]); + } + + if (io->new_pages != io->def_new_pages) + kfree(io->new_pages); + if (io->pages != io->def_pages) + kfree(io->pages); + + cb = io->cb; + res = io->transferred ? io->transferred : io->err; + kfree(io); + + cb.fn(cb.data, cb.vec, res); +} + +/* By the time this function is called, all of the pages prior to + * the current good_idx have been released appropriately. The remaining + * duties are to release any remaining pages and to honour O_SYNC. + */ +static void __iodesc_finish_write(struct iodesc *io) +{ + pr_debug("__iodesc_finish_write(%p)\n", io); + + __iodesc_free(io, WRITE == io->rw); +} + +/* This is mostly ripped from generic_file_write */ +static int __iodesc_write_page(struct iodesc *io, struct page *page) +{ + char *kaddr = kmap(page); + unsigned long bytes; + unsigned long offset; + long status; + int done = 0; + + offset = io->offset; + kaddr += offset; + + bytes = PAGE_CACHE_SIZE - offset; + if (io->size < bytes) + bytes = io->size; + + pr_debug("__iodesc_write_page(%p (%lu), %lu %lu %lu)\n", page, page->index, offset, bytes); + + io->err = io->as->a_ops->prepare_write(io->file, page, + offset, offset + bytes); + if (unlikely(io->err)) { + printk("prepare_write: %d\n", io->err); + kunmap(page); + return 1; + } + + kvec_dst_map(&io->src); + memcpy_from_kvec_dst(kaddr, &io->src, bytes); + kvec_dst_unmap(&io->src); /* commit_write may block */ + + flush_dcache_page(page); + status = io->as->a_ops->commit_write(io->file, page, + offset, offset+bytes); + + /* We don't handle short writes */ + if (status > 0 && status != bytes) + done = 1; + + if (!status) + status = bytes; + else + printk("commit_write: %ld\n", status); + + if (likely(status > 0)) { + io->transferred += status; + io->size -= status; + io->offset = (offset + status) & (PAGE_CACHE_SIZE - 1); + + if (io->offset) + done = 1; + } else { + io->err = status; + done = 1; + } + + kunmap(page); + return done; +} + +void __iodesc_sync_wait_page(void *data) +{ + struct iodesc *io = data; + + do { + struct buffer_head *bh, *head = io->pages[io->good_idx]->buffers; + + if (!head) + continue; + + bh = head; + do { + if (buffer_locked(bh)) { + pr_debug("waiting on bh=%pi io=%p\n", bh, io); + wtd_wait_on_buffer(&io->wtd, bh); + return; + } + if (buffer_req(bh) && !buffer_uptodate(bh)) { + pr_debug("io err bh=%p (%p)\n", bh, io); + io->err = -EIO; + break; + } + } while ((bh = bh->b_this_page) != head); + } while (!io->err && ++io->good_idx < io->nr_pages) ; + + pr_debug("finish_write(%p)\n", io); + __iodesc_finish_write(io); +} + +static void __iodesc_do_write(void *data) +{ + struct iodesc *io = data; + unsigned i; + + up(&io->file->f_dentry->d_inode->i_sem); + + for (i=0; inr_pages; i++) { + if (__iodesc_write_page(io, io->pages[i])) + break; + } + + if (io->sync) { + io->good_idx = 0; + + pr_debug("writing out pages(%p)\n", io); + for (i=0; inr_pages; i++) { + if (io->pages[i]->buffers) + writeout_one_page(io->pages[i]); + } + + pr_debug("calling __iodesc_sync_wait_page(%p)\n", io); + wtd_set_action(&io->wtd, __iodesc_sync_wait_page, io); + __iodesc_sync_wait_page(io); + return; + } + + __iodesc_finish_write(io); +} + +static void __iodesc_write_lock_next_page(void *data) +{ + struct iodesc *io = data; + pr_debug("__iodesc_write_next_page(%p)\n", io); + + while (io->good_idx < io->nr_pages) { + io->good_page = io->pages[io->good_idx++]; + if (io->good_page == *io->cur_pagep) + io->cur_pagep++; + else { + wtd_lock_page(&io->wtd, io->good_page); + return; + } + } + + //Is this faster? __iodesc_do_write(io); + wtd_set_action(&io->wtd, __iodesc_do_write, io); + wtd_queue(&io->wtd); +} + +static void __generic_file_write_iodesc(struct iodesc *io) +{ + struct inode *inode = io->file->f_dentry->d_inode; + time_t now = CURRENT_TIME; + + remove_suid(inode); + if (inode->i_ctime != now || inode->i_mtime != now) { + inode->i_ctime = inode->i_mtime = now; + mark_inode_dirty_sync(inode); + } + + wtd_set_action(&io->wtd, __iodesc_write_lock_next_page, io); + io->sync = !!(io->file->f_flags & O_SYNC); + io->good_idx = 0; + io->cur_pagep = io->new_pages; + __iodesc_write_lock_next_page(io); +} + +static void __iodesc_read_finish(struct iodesc *io) +{ + struct page **src_pagep; + char *dst_addr, *src_addr; + int src_off; + size_t size; + size_t valid; + + struct kveclet *veclet = io->veclet; + struct page *dst_page = veclet->page; + int dst_len = veclet->length; + int dst_off = veclet->offset; + + + pr_debug("__iodesc_read_finish: good_idx = %d\n", io->good_idx); + if (io->good_idx <= 0) + goto no_data; + + size = io->size; + src_off = io->offset; + src_pagep = io->pages; + src_addr = kmap(*src_pagep); + + valid = (size_t)io->good_idx << PAGE_CACHE_SHIFT; + valid -= src_off; + pr_debug("size=%d valid=%d src_off=%d\n", size, valid, src_off); + + if (valid < size) + size = valid; + + dst_addr = kmap(veclet->page); + + while (size > 0) { + int this = PAGE_CACHE_SIZE - src_off; + if ((PAGE_SIZE - dst_off) < this) + this = PAGE_SIZE - dst_off; + if (size < this) + this = size; + pr_debug("this=%d src_off=%d dst_off=%d dst_len=%d\n", + this, src_off, dst_off, dst_len); + memcpy(dst_addr + dst_off, src_addr + src_off, this); + + src_off += this; + dst_off += this; + dst_len -= this; + size -= this; + io->transferred += this; + pr_debug("read_finish: this=%d transferred=%d\n", + this, io->transferred); + + if (size <= 0) + break; + + if (dst_len <= 0) { + kunmap(dst_page); + veclet++; + dst_page = veclet->page; + dst_off = veclet->offset; + dst_len = veclet->length; + dst_addr = kmap(dst_page); + } + + if (src_off >= PAGE_SIZE) { /* FIXME: PAGE_CACHE_SIZE */ + kunmap(*src_pagep); + pr_debug("page(%lu)->count = %d\n", + (*src_pagep)->index, + atomic_read(&(*src_pagep)->count)); + src_pagep++; + src_addr = kmap(*src_pagep); + src_off = 0; + } + } + kunmap(dst_page); + kunmap(*src_pagep); +no_data: + __iodesc_free(io, 0); +} + +static void __iodesc_make_uptodate(void *data) +{ + struct iodesc *io = data; + struct page *page = io->good_page; + int locked = 1; + + pr_debug("__iodesc_make_uptodate: io=%p index=%lu\n", io, page->index); + while (Page_Uptodate(page)) { +again: + pr_debug("page index %lu uptodate\n", page->index); + if (locked) { + UnlockPage(page); + locked = 0; + } + io->did_read = 0; + io->good_idx++; + if (io->good_idx >= io->nr_pages) { + __iodesc_read_finish(io); + return; + } + page = io->good_page = io->pages[io->good_idx]; + pr_debug("__iodesc_make_uptodate: index=%lu\n", page->index); + } + + if (!locked) { + wtd_lock_page(&io->wtd, page); + return; + } + + if (!io->did_read) { + /* We haven't tried reading this page before, give it a go. */ + printk("attempting to read %lu\n", page->index); + io->did_read = 1; + io->err = page->mapping->a_ops->readpage(io->file, page); + if (!io->err) { + if (Page_Uptodate(page)) + goto again; + wtd_lock_page(&io->wtd, page); + return; + } + } + + if (locked) + UnlockPage(page); + + /* We've already read this page before. Set err to EIO and quite */ + if (!io->err) + io->err = -EIO; + __iodesc_read_finish(io); +} + +static void __wtdgeneric_file_read_iodesc(void *data); + +static void __generic_file_read_iodesc(struct iodesc *io, int mayblock) +{ + int (*readpage)(struct file *, struct page *); + int i; + + wtd_set_action(&io->wtd, __iodesc_make_uptodate, io); + readpage = io->as->a_ops->readpage; + for (i=0; inr_new_pages; i++) { + int ret; + if (!mayblock) { + static int zoo; if (zoo++ < 5) printk("read sleep\n"); + wtd_set_action(&io->wtd, __wtdgeneric_file_read_iodesc, io); + wtd_queue(&io->wtd); + return; + } + ret = readpage(io->file, io->new_pages[i]); + if (ret) + printk(KERN_DEBUG "__generic_file_read_kiovec: readpage(%lu) = %d\n", io->new_pages[i]->index, ret); + } + + for (i=0; inr_pages; i++) { + struct page *page = io->pages[i]; + if (Page_Uptodate(page)) { + pr_debug("__generic_file_read_iodesc: %lu is uptodate\n", page->index); + continue; + } + + if (!mayblock) { + static int zoo; if (zoo++ < 5) printk("read sleep\n"); + wtd_set_action(&io->wtd, __wtdgeneric_file_read_iodesc, io); + wtd_queue(&io->wtd); + return; + } + if (!TryLockPage(page)) { + int ret = readpage(io->file, page); + if (ret) + printk(KERN_DEBUG "__generic_file_read_iodesc: readpage(%lu): %d\n", page->index, ret); + } + + if (!Page_Uptodate(page) && io->good_idx == -1) { + pr_debug("first good_idx=%d (%lu)\n", i, page->index); + io->good_idx = i; + io->good_page = page; + } + } + + /* Whee, all the pages are uptodate! */ + if (!io->good_page) { + static int zoo; if (!mayblock && zoo++ < 5) printk("all uptodate\n"); + pr_debug("all pages uptodate!\n"); + io->good_idx = io->nr_pages; + __iodesc_read_finish(io); + return; + } + + pr_debug("locking good_page\n"); + wtd_lock_page(&io->wtd, io->good_page); + return; +} + +static void __wtdgeneric_file_read_iodesc(void *data) +{ + struct iodesc *io = data; + __generic_file_read_iodesc(io, 1); +} + +static int generic_file_rw_kvec(struct file *file, int rw, kvec_cb_t cb, + size_t size, loff_t pos); + +int generic_file_kvec_read(struct file *file, kvec_cb_t cb, size_t size, loff_t pos) +{ + return generic_file_rw_kvec(file, READ, cb, size, pos); +} + +int generic_file_kvec_write(struct file *file, kvec_cb_t cb, size_t size, loff_t pos) +{ + return generic_file_rw_kvec(file, WRITE, cb, size, pos); +} + +int generic_file_rw_kvec(struct file *file, int rw, kvec_cb_t cb, + size_t size, loff_t pos) +{ + struct inode *inode = file->f_dentry->d_inode; + struct address_space *as = inode->i_mapping; + unsigned long index; + unsigned long eindex; + unsigned long nr_pages; + struct iodesc *io = NULL; + int ret; + + ret = -EINVAL; + if (rw != READ && rw != WRITE) + goto out; + + ret = -ENOMEM; + io = kmalloc(sizeof(*io), GFP_KERNEL); + if (!io) + goto out; + + memset(io, 0, sizeof(*io)); + io->size = size; + + if (READ == rw) { + pr_debug("pos=%Ld i_size=%Ld\n", pos, inode->i_size); + + if (pos > inode->i_size) + size = 0; + else if ((pos + size) > inode->i_size) + size = inode->i_size - pos; + + if (io->size < size) + size = io->size; + else if (size < io->size) + io->size = size; + + pr_debug("io->size=%d size=%d\n", io->size, size); + } + + index = pos >> PAGE_CACHE_SHIFT; + eindex = (pos + size - 1) >> PAGE_CACHE_SHIFT; + nr_pages = eindex - index + 1; + + pr_debug("nr_pages: %lu\n", nr_pages); + + io->good_idx = -1; + io->good_page = NULL; + io->did_read = 0; + io->err = 0; + io->rw = rw; + io->as = as; + io->offset = (unsigned long)pos & (PAGE_CACHE_SIZE - 1); + io->file = file; + io->cb = cb; + kvec_dst_init(&io->src, KM_USER0); + kvec_dst_set(&io->src, cb.vec->veclet); + io->veclet = cb.vec->veclet; + if (nr_pages < READDESC_NR_DEF) { + io->pages = io->def_pages; + io->new_pages = io->def_new_pages; + } else { + io->pages = kmalloc(sizeof(*io->pages) * (nr_pages + 1), GFP_KERNEL); + if (!io->pages) + goto out_io; + + io->new_pages = kmalloc(sizeof(*io->new_pages) * (nr_pages + 1), GFP_KERNEL); + if (!io->new_pages) + goto out_pages; + } + + /* FIXME: make the down a WTD_op */ + if (rw == WRITE) + down(&io->file->f_dentry->d_inode->i_sem); + + ret = address_space_map(as, index, nr_pages, io->pages, + &io->nr_new_pages, io->new_pages); + pr_debug("as_map: %d (%d new)\n", ret, io->nr_new_pages); + if (ret <= 0) + goto out_new_pages; + + io->nr_pages = ret; + io->pages[io->nr_pages] = NULL; + io->new_pages[io->nr_new_pages] = NULL; + + if (rw == READ) + __generic_file_read_iodesc(io, 0); + else if (rw == WRITE) + __generic_file_write_iodesc(io); + + return 0; + +out_new_pages: + if (io->new_pages != io->def_new_pages) + kfree(io->new_pages); +out_pages: + if (io->pages != io->def_pages) + kfree(io->pages); +out_io: + kfree(io); +out: + return ret; +} diff -urN /md0/kernels/2.4/v2.4.18-pre3/mm/memory.c linux.aio/mm/memory.c --- /md0/kernels/2.4/v2.4.18-pre3/mm/memory.c Tue Jan 1 14:09:35 2002 +++ linux.aio/mm/memory.c Wed Jan 23 01:41:06 2002 @@ -44,6 +44,8 @@ #include #include #include +#include +#include #include #include @@ -1443,3 +1445,207 @@ len, write, 0, NULL, NULL); return ret == len ? 0 : -1; } + +/* + * Force in an entire range of pages from the current process's user VA, + * and pin them in physical memory. + * FIXME: some architectures need to flush the cache based on user addresses + * here. Someone please provide a better macro than flush_cache_page. + */ + +#define dprintk(x...) +struct kvec *map_user_kvec(int rw, unsigned long ptr, size_t len) +{ + struct kvec *vec; + struct kveclet *veclet; + unsigned long end; + int err; + struct mm_struct * mm; + struct vm_area_struct * vma = 0; + int i; + int datain = (rw == READ); + unsigned nr_pages; + + end = ptr + len; + if (end < ptr) + return ERR_PTR(-EINVAL); + + nr_pages = (ptr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; + nr_pages -= ptr >> PAGE_SHIFT; + nr_pages ++; + vec = kmalloc(sizeof(struct kvec) + nr_pages * sizeof(struct kveclet), + GFP_KERNEL); + if (!vec) + return ERR_PTR(-ENOMEM); + vec->nr = 0; + vec->max_nr = nr_pages; + veclet = vec->veclet; + + /* Make sure the iobuf is not already mapped somewhere. */ + mm = current->mm; + dprintk ("map_user_kiobuf: begin\n"); + + down_read(&mm->mmap_sem); + + err = -EFAULT; + + i = 0; + + /* + * First of all, try to fault in all of the necessary pages + */ + while (ptr < end) { + struct page *map; + veclet->offset = ptr & ~PAGE_MASK; + veclet->length = PAGE_SIZE - veclet->offset; + if (len < veclet->length) + veclet->length = len; + ptr &= PAGE_MASK; + len -= veclet->length; + + if (!vma || ptr >= vma->vm_end) { + vma = find_vma(current->mm, ptr); + if (!vma) + goto out_unlock; + if (vma->vm_start > ptr) { + if (!(vma->vm_flags & VM_GROWSDOWN)) + goto out_unlock; + if (expand_stack(vma, ptr)) + goto out_unlock; + } + if (((datain) && (!(vma->vm_flags & VM_WRITE))) || + (!(vma->vm_flags & VM_READ))) { + err = -EACCES; + goto out_unlock; + } + } + spin_lock(&mm->page_table_lock); + while (!(map = follow_page(mm, ptr, datain))) { + int ret; + + spin_unlock(&mm->page_table_lock); + ret = handle_mm_fault(current->mm, vma, ptr, datain); + if (ret <= 0) { + if (!ret) + goto out_unlock; + else { + err = -ENOMEM; + goto out_unlock; + } + } + spin_lock(&mm->page_table_lock); + } + map = get_page_map(map); + if (map) { + flush_dcache_page(map); + atomic_inc(&map->count); + } else + printk (KERN_INFO "Mapped page missing [%d]\n", i); + spin_unlock(&mm->page_table_lock); + veclet->page = map; + veclet++; + + ptr += PAGE_SIZE; + vec->nr = ++i; + } + + veclet->page = NULL; /* dummy for the prefetch in free_kvec */ + veclet->length = 0; /* bug checking ;-) */ + + up_read(&mm->mmap_sem); + dprintk ("map_user_kiobuf: end OK\n"); + return vec; + + out_unlock: + up_read(&mm->mmap_sem); + unmap_kvec(vec, 0); + printk(KERN_DEBUG "map_user_kvec: err(%d)\n", err); + kfree(vec); + return ERR_PTR(err); +} + +/* + * Unmap all of the pages referenced by a kiobuf. We release the pages, + * and unlock them if they were locked. + */ + +void unmap_kvec (struct kvec *vec, int dirtied) +{ + struct kveclet *veclet = vec->veclet; + struct kveclet *end = vec->veclet + vec->nr; + struct page *map = veclet->page; + + prefetchw(map); + for (; vecletpage) { + prefetchw(veclet[1].page); + if (likely(map != NULL) && !PageReserved(map)) { + if (dirtied) { + SetPageDirty(map); + flush_dcache_page(map); /* FIXME */ + } + __free_page(map); + } + } + + vec->nr = 0; +} + +void free_kvec(struct kvec *vec) +{ + kfree(vec); +} + +/* kvec memory copy helper: appends len bytes in from to dst. + */ +void memcpy_to_kvec_dst(struct kvec_dst *dst, const char *from, long len) +{ + if (unlikely(len < 0)) + BUG(); + do { + int cnt = len; + if (dst->space < cnt) + cnt = dst->space; + + memcpy(dst->dst, from, cnt); + from += cnt; + dst->space -= cnt; + dst->dst += cnt; + len -= cnt; + if (!dst->space && len) { + kvec_dst_unmap(dst); + dst->let++; + dst->offset = 0; + kvec_dst_map(dst); + if (unlikely(!dst->space)) + BUG(); + } + } while (len); +} + +/* kvec memory copy helper: copies and consumes len bytes in from to dst. + */ +void memcpy_from_kvec_dst(char *to, struct kvec_dst *from, long len) +{ + if (unlikely(len < 0)) + BUG(); + do { + int cnt = len; + if (from->space < cnt) + cnt = from->space; + + memcpy(to, from->dst, cnt); + to += cnt; + from->space -= cnt; + from->dst += cnt; + len -= cnt; + if (unlikely(!from->space && len)) { + kvec_dst_unmap(from); + from->let++; + from->offset = 0; + kvec_dst_map(from); + if (unlikely(!from->space)) + BUG(); + } + } while (len); +} + diff -urN /md0/kernels/2.4/v2.4.18-pre3/mm/mmap.c linux.aio/mm/mmap.c --- /md0/kernels/2.4/v2.4.18-pre3/mm/mmap.c Mon Nov 12 17:49:51 2001 +++ linux.aio/mm/mmap.c Wed Jan 23 01:41:06 2002 @@ -1104,6 +1104,7 @@ { struct vm_area_struct * mpnt; + exit_aio(mm); release_segments(mm); spin_lock(&mm->page_table_lock); mpnt = mm->mmap; diff -urN /md0/kernels/2.4/v2.4.18-pre3/mm/wtd.c linux.aio/mm/wtd.c --- /md0/kernels/2.4/v2.4.18-pre3/mm/wtd.c Wed Dec 31 19:00:00 1969 +++ linux.aio/mm/wtd.c Wed Jan 23 01:41:06 2002 @@ -0,0 +1,72 @@ +#include +#include +#include + +static void __wtd_lock_page_waiter(wait_queue_t *wait) +{ + struct worktodo *wtd = (struct worktodo *)wait; + struct page *page = (struct page *)wtd->data; + + if (!TryLockPage(page)) { + __remove_wait_queue(&page->wait, &wtd->wait); + wtd_queue(wtd); + } else + schedule_task(&run_disk_tq); +} + +void wtd_lock_page(struct worktodo *wtd, struct page *page) +{ + if (TryLockPage(page)) { + wtd->data = page; + init_waitqueue_func_entry(&wtd->wait, __wtd_lock_page_waiter); + + /* Wakeups may race with TryLockPage, so try again within the wait + * queue spinlock. + */ + if (!add_wait_queue_cond(&page->wait, &wtd->wait, TryLockPage(page))) { + /* Page is still locked. Kick the disk queue... */ + run_task_queue(&tq_disk); + return; + } + } + + wtd->tq.routine(wtd->tq.data); +} + +static void __wtd_bh_waiter(wait_queue_t *wait) +{ + struct worktodo *wtd = (struct worktodo *)wait; + struct buffer_head *bh = (struct buffer_head *)wtd->data; + + if (!buffer_locked(bh)) { + __remove_wait_queue(&bh->b_wait, &wtd->wait); + wtd_queue(wtd); + } else { + schedule_task(&run_disk_tq); + } +} + +void wtd_wait_on_buffer(struct worktodo *wtd, struct buffer_head *bh) +{ + if (!buffer_locked(bh)) { + wtd->tq.routine(wtd->tq.data); + return; + } + wtd->data = bh; + init_waitqueue_func_entry(&wtd->wait, __wtd_bh_waiter); + if (add_wait_queue_cond(&bh->b_wait, &wtd->wait, buffer_locked(bh))) + wtd->tq.routine(wtd->tq.data); + else + run_task_queue(&tq_disk); +} + +void do_run_tq_disk(void *data) +{ + run_task_queue(&tq_disk); +} + +struct tq_struct run_disk_tq = { + routine: do_run_tq_disk, + data: NULL +}; + diff -urN /md0/kernels/2.4/v2.4.18-pre3/net/khttpd/datasending.c linux.aio/net/khttpd/datasending.c --- /md0/kernels/2.4/v2.4.18-pre3/net/khttpd/datasending.c Mon Sep 24 02:16:05 2001 +++ linux.aio/net/khttpd/datasending.c Wed Jan 23 01:41:06 2002 @@ -127,7 +127,7 @@ desc.count = ReadSize; desc.buf = (char *) CurrentRequest->sock; desc.error = 0; - do_generic_file_read(CurrentRequest->filp, ppos, &desc, sock_send_actor); + do_generic_file_read(CurrentRequest->filp, ppos, &desc, sock_send_actor, 0); if (desc.written>0) { CurrentRequest->BytesSent += desc.written; diff -urN /md0/kernels/2.4/v2.4.18-pre3/ulib/Makefile linux.aio/ulib/Makefile --- /md0/kernels/2.4/v2.4.18-pre3/ulib/Makefile Wed Dec 31 19:00:00 1969 +++ linux.aio/ulib/Makefile Wed Jan 23 01:41:06 2002 @@ -0,0 +1,30 @@ +all: libredhat-kernel.so + +ASFLAGS=-D__KERNEL__ -D__ASSEMBLY__ -I../include -nostdlib -nostartfiles +CFLAGS=-D__KERNEL__ -I../include -nostdlib -nostartfiles + +so_objs=vsysaddr.o kso_init.o + +vsysaddr.S: ../System.map stub.S Makefile + rm -f vsysaddr.S + echo '#include "stub.S"' >vsysaddr.S + awk -- "/^bfff.* vsys_/ { print \"dynamic_syscall(\"\$$3 \",0x\" \$$1 \")\"; }" <../System.map >>vsysaddr.S + +vsysaddr.o: vsysaddr.S + +kso_init.o: ../include/linux/compile.h + +libredhat-kernel.so.1: $(so_objs) libredhat-kernel.map + gcc -nostdlib -nostartfiles -shared -Wl,--version-script=libredhat-kernel.map -Wl,-soname=libredhat-kernel.so.1 -o $@ $(so_objs) + cp $@ $@.save + strip $@ + +libredhat-kernel.so: libredhat-kernel.so.1 + ln -sf $< $@ + +clean: + rm -f *.o libredhat-kernel.so myln libredhat-kernel.so.1 vsysaddr.S libredhat-kernel.so.1.save + +# test app +myln: myln.c libredhat-kernel.so Makefile + cc -g -o myln myln.c -L. -lredhat-kernel diff -urN /md0/kernels/2.4/v2.4.18-pre3/ulib/kso_init.c linux.aio/ulib/kso_init.c --- /md0/kernels/2.4/v2.4.18-pre3/ulib/kso_init.c Wed Dec 31 19:00:00 1969 +++ linux.aio/ulib/kso_init.c Wed Jan 23 01:41:06 2002 @@ -0,0 +1,48 @@ +#include +#include +#include +#include +#include +#include + +char libredhat_kernel_enosys = 1; /* the asm in stub.S depends on this */ + +long _init(void) +{ + static char unique[] = { LINUX_UNIQUE_BYTES }; + int errno; + long addr; + int fd; + int i; + + _syscall6(int, mmap2, unsigned long, addr, unsigned long, len, + unsigned long, prot, unsigned long, flags, + unsigned long, fd, unsigned long, pgoff) + _syscall2(long, munmap, unsigned long, addr, size_t, len) + _syscall2(int, open, const char *, name, int, flags) + _syscall1(int, close, int, fd) + + if (sizeof(unique) != 16) + return -1; + + fd = open("/dev/vsys", O_RDONLY); + if (-1 == fd) + return -1; + + addr = mmap2(0, VSYSCALL_SIZE, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0); + if (-1 == addr) + return -1; + + close(fd); + + for (i=0; i +#include +#include +#include +#include + +int main () +{ + long ctx = 0; + extern long vsys_io_setup(long, long *); + unsigned char *bob = (void*)&vsys_io_setup; + long ret; + int i; + printf("%p\n", bob); + //printf("%p\n", mmap(0, 65536, PROT_READ | PROT_EXEC, MAP_SHARED, + // open("/dev/vsys", O_RDONLY), 0)); + //for (i=0; i<16; i++) + // printf(" %02x\n", bob[i]); + //printf("\n"); + + ret = vsys_io_setup(100, &ctx); + + printf("ret=%ld, ctx=0x%lx\n", ret, ctx); + return 0; +} diff -urN /md0/kernels/2.4/v2.4.18-pre3/ulib/stub.S linux.aio/ulib/stub.S --- /md0/kernels/2.4/v2.4.18-pre3/ulib/stub.S Wed Dec 31 19:00:00 1969 +++ linux.aio/ulib/stub.S Wed Jan 23 01:41:06 2002 @@ -0,0 +1,19 @@ +/* stub.S */ +#include +#include + + .text + +#define dynamic_syscall(x,a) \ + .globl x ;\ + .type x, @function ;\ + .align 16 ;\ + x: ;\ + cmpb $0,libredhat_kernel_enosys ;\ + jne 1f ;\ + ljmp $__USER_CS, $a ;\ + 1: ;\ + movl $-ENOSYS,%eax ;\ + ret ;\ + .size x,.-x +