File:  [DragonFly] / src / sys / vm / vm_fault.c
Revision 1.13: download - view: text, annotated - select for diffs
Mon Mar 29 17:30:23 2004 UTC (10 years, 6 months ago) by drhodus
Branches: MAIN
CVS tags: HEAD
Move vm_fault_quick() out from the machine specific location
as the function is now cpu agnostic.

    1: /*
    2:  * Copyright (c) 1991, 1993
    3:  *	The Regents of the University of California.  All rights reserved.
    4:  * Copyright (c) 1994 John S. Dyson
    5:  * All rights reserved.
    6:  * Copyright (c) 1994 David Greenman
    7:  * All rights reserved.
    8:  *
    9:  *
   10:  * This code is derived from software contributed to Berkeley by
   11:  * The Mach Operating System project at Carnegie-Mellon University.
   12:  *
   13:  * Redistribution and use in source and binary forms, with or without
   14:  * modification, are permitted provided that the following conditions
   15:  * are met:
   16:  * 1. Redistributions of source code must retain the above copyright
   17:  *    notice, this list of conditions and the following disclaimer.
   18:  * 2. Redistributions in binary form must reproduce the above copyright
   19:  *    notice, this list of conditions and the following disclaimer in the
   20:  *    documentation and/or other materials provided with the distribution.
   21:  * 3. All advertising materials mentioning features or use of this software
   22:  *    must display the following acknowledgement:
   23:  *	This product includes software developed by the University of
   24:  *	California, Berkeley and its contributors.
   25:  * 4. Neither the name of the University nor the names of its contributors
   26:  *    may be used to endorse or promote products derived from this software
   27:  *    without specific prior written permission.
   28:  *
   29:  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   30:  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   31:  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   32:  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   33:  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   34:  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   35:  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   36:  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   37:  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   38:  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   39:  * SUCH DAMAGE.
   40:  *
   41:  *	from: @(#)vm_fault.c	8.4 (Berkeley) 1/12/94
   42:  *
   43:  *
   44:  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
   45:  * All rights reserved.
   46:  *
   47:  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
   48:  *
   49:  * Permission to use, copy, modify and distribute this software and
   50:  * its documentation is hereby granted, provided that both the copyright
   51:  * notice and this permission notice appear in all copies of the
   52:  * software, derivative works or modified versions, and any portions
   53:  * thereof, and that both notices appear in supporting documentation.
   54:  *
   55:  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
   56:  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
   57:  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
   58:  *
   59:  * Carnegie Mellon requests users of this software to return to
   60:  *
   61:  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
   62:  *  School of Computer Science
   63:  *  Carnegie Mellon University
   64:  *  Pittsburgh PA 15213-3890
   65:  *
   66:  * any improvements or extensions that they make and grant Carnegie the
   67:  * rights to redistribute these changes.
   68:  *
   69:  * $FreeBSD: src/sys/vm/vm_fault.c,v 1.108.2.8 2002/02/26 05:49:27 silby Exp $
   70:  * $DragonFly: src/sys/vm/vm_fault.c,v 1.13 2004/03/29 17:30:23 drhodus Exp $
   71:  */
   72: 
   73: /*
   74:  *	Page fault handling module.
   75:  */
   76: 
   77: #include <sys/param.h>
   78: #include <sys/systm.h>
   79: #include <sys/proc.h>
   80: #include <sys/vnode.h>
   81: #include <sys/resourcevar.h>
   82: #include <sys/vmmeter.h>
   83: 
   84: #include <vm/vm.h>
   85: #include <vm/vm_param.h>
   86: #include <sys/lock.h>
   87: #include <vm/pmap.h>
   88: #include <vm/vm_map.h>
   89: #include <vm/vm_object.h>
   90: #include <vm/vm_page.h>
   91: #include <vm/vm_pageout.h>
   92: #include <vm/vm_kern.h>
   93: #include <vm/vm_pager.h>
   94: #include <vm/vnode_pager.h>
   95: #include <vm/vm_extern.h>
   96: #include <vm/vm_page2.h>
   97: 
   98: static int vm_fault_additional_pages (vm_page_t, int,
   99: 					  int, vm_page_t *, int *);
  100: 
  101: #define VM_FAULT_READ_AHEAD 8
  102: #define VM_FAULT_READ_BEHIND 7
  103: #define VM_FAULT_READ (VM_FAULT_READ_AHEAD+VM_FAULT_READ_BEHIND+1)
  104: 
  105: struct faultstate {
  106: 	vm_page_t m;
  107: 	vm_object_t object;
  108: 	vm_pindex_t pindex;
  109: 	vm_page_t first_m;
  110: 	vm_object_t	first_object;
  111: 	vm_pindex_t first_pindex;
  112: 	vm_map_t map;
  113: 	vm_map_entry_t entry;
  114: 	int lookup_still_valid;
  115: 	struct vnode *vp;
  116: };
  117: 
  118: static __inline void
  119: release_page(struct faultstate *fs)
  120: {
  121: 	vm_page_wakeup(fs->m);
  122: 	vm_page_deactivate(fs->m);
  123: 	fs->m = NULL;
  124: }
  125: 
  126: static __inline void
  127: unlock_map(struct faultstate *fs)
  128: {
  129: 	if (fs->lookup_still_valid) {
  130: 		vm_map_lookup_done(fs->map, fs->entry, 0);
  131: 		fs->lookup_still_valid = FALSE;
  132: 	}
  133: }
  134: 
  135: static void
  136: _unlock_things(struct faultstate *fs, int dealloc)
  137: {
  138: 	vm_object_pip_wakeup(fs->object);
  139: 	if (fs->object != fs->first_object) {
  140: 		vm_page_free(fs->first_m);
  141: 		vm_object_pip_wakeup(fs->first_object);
  142: 		fs->first_m = NULL;
  143: 	}
  144: 	if (dealloc) {
  145: 		vm_object_deallocate(fs->first_object);
  146: 	}
  147: 	unlock_map(fs);	
  148: 	if (fs->vp != NULL) { 
  149: 		vput(fs->vp);
  150: 		fs->vp = NULL;
  151: 	}
  152: }
  153: 
  154: #define unlock_things(fs) _unlock_things(fs, 0)
  155: #define unlock_and_deallocate(fs) _unlock_things(fs, 1)
  156: 
  157: /*
  158:  * TRYPAGER - used by vm_fault to calculate whether the pager for the
  159:  *	      current object *might* contain the page.
  160:  *
  161:  *	      default objects are zero-fill, there is no real pager.
  162:  */
  163: 
  164: #define TRYPAGER	(fs.object->type != OBJT_DEFAULT && \
  165: 			(((fault_flags & VM_FAULT_WIRE_MASK) == 0) || wired))
  166: 
  167: /*
  168:  *	vm_fault:
  169:  *
  170:  *	Handle a page fault occurring at the given address,
  171:  *	requiring the given permissions, in the map specified.
  172:  *	If successful, the page is inserted into the
  173:  *	associated physical map.
  174:  *
  175:  *	NOTE: the given address should be truncated to the
  176:  *	proper page address.
  177:  *
  178:  *	KERN_SUCCESS is returned if the page fault is handled; otherwise,
  179:  *	a standard error specifying why the fault is fatal is returned.
  180:  *
  181:  *
  182:  *	The map in question must be referenced, and remains so.
  183:  *	Caller may hold no locks.
  184:  */
  185: int
  186: vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags)
  187: {
  188: 	vm_prot_t prot;
  189: 	int result;
  190: 	boolean_t wired;
  191: 	int map_generation;
  192: 	vm_object_t next_object;
  193: 	vm_page_t marray[VM_FAULT_READ];
  194: 	int hardfault;
  195: 	int faultcount;
  196: 	struct faultstate fs;
  197: 
  198: 	mycpu->gd_cnt.v_vm_faults++;
  199: 	hardfault = 0;
  200: 
  201: RetryFault:;
  202: 
  203: 	/*
  204: 	 * Find the backing store object and offset into it to begin the
  205: 	 * search.
  206: 	 */
  207: 	fs.map = map;
  208: 	if ((result = vm_map_lookup(&fs.map, vaddr,
  209: 		fault_type, &fs.entry, &fs.first_object,
  210: 		&fs.first_pindex, &prot, &wired)) != KERN_SUCCESS) {
  211: 		if ((result != KERN_PROTECTION_FAILURE) ||
  212: 			((fault_flags & VM_FAULT_WIRE_MASK) != VM_FAULT_USER_WIRE)) {
  213: 			return result;
  214: 		}
  215: 
  216: 		/*
  217:    		 * If we are user-wiring a r/w segment, and it is COW, then
  218:    		 * we need to do the COW operation.  Note that we don't COW
  219:    		 * currently RO sections now, because it is NOT desirable
  220:    		 * to COW .text.  We simply keep .text from ever being COW'ed
  221:    		 * and take the heat that one cannot debug wired .text sections.
  222:    		 */
  223: 		result = vm_map_lookup(&fs.map, vaddr,
  224: 			VM_PROT_READ|VM_PROT_WRITE|VM_PROT_OVERRIDE_WRITE,
  225: 			&fs.entry, &fs.first_object, &fs.first_pindex, &prot, &wired);
  226: 		if (result != KERN_SUCCESS) {
  227: 			return result;
  228: 		}
  229: 
  230: 		/*
  231: 		 * If we don't COW now, on a user wire, the user will never
  232: 		 * be able to write to the mapping.  If we don't make this
  233: 		 * restriction, the bookkeeping would be nearly impossible.
  234: 		 */
  235: 		if ((fs.entry->protection & VM_PROT_WRITE) == 0)
  236: 			fs.entry->max_protection &= ~VM_PROT_WRITE;
  237: 	}
  238: 
  239: 	map_generation = fs.map->timestamp;
  240: 
  241: 	if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
  242: 		panic("vm_fault: fault on nofault entry, addr: %lx",
  243: 		    (u_long)vaddr);
  244: 	}
  245: 
  246: 	/*
  247: 	 * Make a reference to this object to prevent its disposal while we
  248: 	 * are messing with it.  Once we have the reference, the map is free
  249: 	 * to be diddled.  Since objects reference their shadows (and copies),
  250: 	 * they will stay around as well.
  251: 	 *
  252: 	 * Bump the paging-in-progress count to prevent size changes (e.g.
  253: 	 * truncation operations) during I/O.  This must be done after
  254: 	 * obtaining the vnode lock in order to avoid possible deadlocks.
  255: 	 */
  256: 	vm_object_reference(fs.first_object);
  257: 	fs.vp = vnode_pager_lock(fs.first_object);
  258: 	vm_object_pip_add(fs.first_object, 1);
  259: 
  260: 	if ((fault_type & VM_PROT_WRITE) &&
  261: 		(fs.first_object->type == OBJT_VNODE)) {
  262: 		vm_freeze_copyopts(fs.first_object,
  263: 			fs.first_pindex, fs.first_pindex + 1);
  264: 	}
  265: 
  266: 	fs.lookup_still_valid = TRUE;
  267: 
  268: 	if (wired)
  269: 		fault_type = prot;
  270: 
  271: 	fs.first_m = NULL;
  272: 
  273: 	/*
  274: 	 * Search for the page at object/offset.
  275: 	 */
  276: 
  277: 	fs.object = fs.first_object;
  278: 	fs.pindex = fs.first_pindex;
  279: 
  280: 	while (TRUE) {
  281: 		/*
  282: 		 * If the object is dead, we stop here
  283: 		 */
  284: 
  285: 		if (fs.object->flags & OBJ_DEAD) {
  286: 			unlock_and_deallocate(&fs);
  287: 			return (KERN_PROTECTION_FAILURE);
  288: 		}
  289: 
  290: 		/*
  291: 		 * See if page is resident
  292: 		 */
  293: 			
  294: 		fs.m = vm_page_lookup(fs.object, fs.pindex);
  295: 		if (fs.m != NULL) {
  296: 			int queue, s;
  297: 			/*
  298: 			 * Wait/Retry if the page is busy.  We have to do this
  299: 			 * if the page is busy via either PG_BUSY or 
  300: 			 * vm_page_t->busy because the vm_pager may be using
  301: 			 * vm_page_t->busy for pageouts ( and even pageins if
  302: 			 * it is the vnode pager ), and we could end up trying
  303: 			 * to pagein and pageout the same page simultaneously.
  304: 			 *
  305: 			 * We can theoretically allow the busy case on a read
  306: 			 * fault if the page is marked valid, but since such
  307: 			 * pages are typically already pmap'd, putting that
  308: 			 * special case in might be more effort then it is 
  309: 			 * worth.  We cannot under any circumstances mess
  310: 			 * around with a vm_page_t->busy page except, perhaps,
  311: 			 * to pmap it.
  312: 			 */
  313: 			if ((fs.m->flags & PG_BUSY) || fs.m->busy) {
  314: 				unlock_things(&fs);
  315: 				(void)vm_page_sleep_busy(fs.m, TRUE, "vmpfw");
  316: 				mycpu->gd_cnt.v_intrans++;
  317: 				vm_object_deallocate(fs.first_object);
  318: 				goto RetryFault;
  319: 			}
  320: 
  321: 			queue = fs.m->queue;
  322: 			s = splvm();
  323: 			vm_page_unqueue_nowakeup(fs.m);
  324: 			splx(s);
  325: 
  326: 			if ((queue - fs.m->pc) == PQ_CACHE && vm_page_count_severe()) {
  327: 				vm_page_activate(fs.m);
  328: 				unlock_and_deallocate(&fs);
  329: 				VM_WAITPFAULT;
  330: 				goto RetryFault;
  331: 			}
  332: 
  333: 			/*
  334: 			 * Mark page busy for other processes, and the 
  335: 			 * pagedaemon.  If it still isn't completely valid
  336: 			 * (readable), jump to readrest, else break-out ( we
  337: 			 * found the page ).
  338: 			 */
  339: 
  340: 			vm_page_busy(fs.m);
  341: 			if (((fs.m->valid & VM_PAGE_BITS_ALL) != VM_PAGE_BITS_ALL) &&
  342: 				fs.m->object != kernel_object && fs.m->object != kmem_object) {
  343: 				goto readrest;
  344: 			}
  345: 
  346: 			break;
  347: 		}
  348: 
  349: 		/*
  350: 		 * Page is not resident, If this is the search termination
  351: 		 * or the pager might contain the page, allocate a new page.
  352: 		 */
  353: 
  354: 		if (TRYPAGER || fs.object == fs.first_object) {
  355: 			if (fs.pindex >= fs.object->size) {
  356: 				unlock_and_deallocate(&fs);
  357: 				return (KERN_PROTECTION_FAILURE);
  358: 			}
  359: 
  360: 			/*
  361: 			 * Allocate a new page for this object/offset pair.
  362: 			 */
  363: 			fs.m = NULL;
  364: 			if (!vm_page_count_severe()) {
  365: 				fs.m = vm_page_alloc(fs.object, fs.pindex,
  366: 				    (fs.vp || fs.object->backing_object)? VM_ALLOC_NORMAL: VM_ALLOC_NORMAL | VM_ALLOC_ZERO);
  367: 			}
  368: 			if (fs.m == NULL) {
  369: 				unlock_and_deallocate(&fs);
  370: 				VM_WAITPFAULT;
  371: 				goto RetryFault;
  372: 			}
  373: 		}
  374: 
  375: readrest:
  376: 		/*
  377: 		 * We have found a valid page or we have allocated a new page.
  378: 		 * The page thus may not be valid or may not be entirely 
  379: 		 * valid.
  380: 		 *
  381: 		 * Attempt to fault-in the page if there is a chance that the
  382: 		 * pager has it, and potentially fault in additional pages
  383: 		 * at the same time.
  384: 		 */
  385: 
  386: 		if (TRYPAGER) {
  387: 			int rv;
  388: 			int reqpage;
  389: 			int ahead, behind;
  390: 			u_char behavior = vm_map_entry_behavior(fs.entry);
  391: 
  392: 			if (behavior == MAP_ENTRY_BEHAV_RANDOM) {
  393: 				ahead = 0;
  394: 				behind = 0;
  395: 			} else {
  396: 				behind = (vaddr - fs.entry->start) >> PAGE_SHIFT;
  397: 				if (behind > VM_FAULT_READ_BEHIND)
  398: 					behind = VM_FAULT_READ_BEHIND;
  399: 
  400: 				ahead = ((fs.entry->end - vaddr) >> PAGE_SHIFT) - 1;
  401: 				if (ahead > VM_FAULT_READ_AHEAD)
  402: 					ahead = VM_FAULT_READ_AHEAD;
  403: 			}
  404: 
  405: 			if ((fs.first_object->type != OBJT_DEVICE) &&
  406: 			    (behavior == MAP_ENTRY_BEHAV_SEQUENTIAL ||
  407:                                 (behavior != MAP_ENTRY_BEHAV_RANDOM &&
  408:                                 fs.pindex >= fs.entry->lastr &&
  409:                                 fs.pindex < fs.entry->lastr + VM_FAULT_READ))
  410: 			) {
  411: 				vm_pindex_t firstpindex, tmppindex;
  412: 
  413: 				if (fs.first_pindex < 2 * VM_FAULT_READ)
  414: 					firstpindex = 0;
  415: 				else
  416: 					firstpindex = fs.first_pindex - 2 * VM_FAULT_READ;
  417: 
  418: 				/*
  419: 				 * note: partially valid pages cannot be 
  420: 				 * included in the lookahead - NFS piecemeal
  421: 				 * writes will barf on it badly.
  422: 				 */
  423: 
  424: 				for(tmppindex = fs.first_pindex - 1;
  425: 					tmppindex >= firstpindex;
  426: 					--tmppindex) {
  427: 					vm_page_t mt;
  428: 					mt = vm_page_lookup( fs.first_object, tmppindex);
  429: 					if (mt == NULL || (mt->valid != VM_PAGE_BITS_ALL))
  430: 						break;
  431: 					if (mt->busy ||
  432: 						(mt->flags & (PG_BUSY | PG_FICTITIOUS | PG_UNMANAGED)) ||
  433: 						mt->hold_count ||
  434: 						mt->wire_count) 
  435: 						continue;
  436: 					if (mt->dirty == 0)
  437: 						vm_page_test_dirty(mt);
  438: 					if (mt->dirty) {
  439: 						vm_page_protect(mt, VM_PROT_NONE);
  440: 						vm_page_deactivate(mt);
  441: 					} else {
  442: 						vm_page_cache(mt);
  443: 					}
  444: 				}
  445: 
  446: 				ahead += behind;
  447: 				behind = 0;
  448: 			}
  449: 
  450: 			/*
  451: 			 * now we find out if any other pages should be paged
  452: 			 * in at this time this routine checks to see if the
  453: 			 * pages surrounding this fault reside in the same
  454: 			 * object as the page for this fault.  If they do,
  455: 			 * then they are faulted in also into the object.  The
  456: 			 * array "marray" returned contains an array of
  457: 			 * vm_page_t structs where one of them is the
  458: 			 * vm_page_t passed to the routine.  The reqpage
  459: 			 * return value is the index into the marray for the
  460: 			 * vm_page_t passed to the routine.
  461: 			 *
  462: 			 * fs.m plus the additional pages are PG_BUSY'd.
  463: 			 */
  464: 			faultcount = vm_fault_additional_pages(
  465: 			    fs.m, behind, ahead, marray, &reqpage);
  466: 
  467: 			/*
  468: 			 * update lastr imperfectly (we do not know how much
  469: 			 * getpages will actually read), but good enough.
  470: 			 */
  471: 			fs.entry->lastr = fs.pindex + faultcount - behind;
  472: 
  473: 			/*
  474: 			 * Call the pager to retrieve the data, if any, after
  475: 			 * releasing the lock on the map.  We hold a ref on
  476: 			 * fs.object and the pages are PG_BUSY'd.
  477: 			 */
  478: 			unlock_map(&fs);
  479: 
  480: 			rv = faultcount ?
  481: 			    vm_pager_get_pages(fs.object, marray, faultcount,
  482: 				reqpage) : VM_PAGER_FAIL;
  483: 
  484: 			if (rv == VM_PAGER_OK) {
  485: 				/*
  486: 				 * Found the page. Leave it busy while we play
  487: 				 * with it.
  488: 				 */
  489: 
  490: 				/*
  491: 				 * Relookup in case pager changed page. Pager
  492: 				 * is responsible for disposition of old page
  493: 				 * if moved.
  494: 				 */
  495: 				fs.m = vm_page_lookup(fs.object, fs.pindex);
  496: 				if(!fs.m) {
  497: 					unlock_and_deallocate(&fs);
  498: 					goto RetryFault;
  499: 				}
  500: 
  501: 				hardfault++;
  502: 				break; /* break to PAGE HAS BEEN FOUND */
  503: 			}
  504: 			/*
  505: 			 * Remove the bogus page (which does not exist at this
  506: 			 * object/offset); before doing so, we must get back
  507: 			 * our object lock to preserve our invariant.
  508: 			 *
  509: 			 * Also wake up any other process that may want to bring
  510: 			 * in this page.
  511: 			 *
  512: 			 * If this is the top-level object, we must leave the
  513: 			 * busy page to prevent another process from rushing
  514: 			 * past us, and inserting the page in that object at
  515: 			 * the same time that we are.
  516: 			 */
  517: 
  518: 			if (rv == VM_PAGER_ERROR)
  519: 				printf("vm_fault: pager read error, pid %d (%s)\n",
  520: 				    curproc->p_pid, curproc->p_comm);
  521: 			/*
  522: 			 * Data outside the range of the pager or an I/O error
  523: 			 */
  524: 			/*
  525: 			 * XXX - the check for kernel_map is a kludge to work
  526: 			 * around having the machine panic on a kernel space
  527: 			 * fault w/ I/O error.
  528: 			 */
  529: 			if (((fs.map != kernel_map) && (rv == VM_PAGER_ERROR)) ||
  530: 				(rv == VM_PAGER_BAD)) {
  531: 				vm_page_free(fs.m);
  532: 				fs.m = NULL;
  533: 				unlock_and_deallocate(&fs);
  534: 				return ((rv == VM_PAGER_ERROR) ? KERN_FAILURE : KERN_PROTECTION_FAILURE);
  535: 			}
  536: 			if (fs.object != fs.first_object) {
  537: 				vm_page_free(fs.m);
  538: 				fs.m = NULL;
  539: 				/*
  540: 				 * XXX - we cannot just fall out at this
  541: 				 * point, m has been freed and is invalid!
  542: 				 */
  543: 			}
  544: 		}
  545: 
  546: 		/*
  547: 		 * We get here if the object has default pager (or unwiring) 
  548: 		 * or the pager doesn't have the page.
  549: 		 */
  550: 		if (fs.object == fs.first_object)
  551: 			fs.first_m = fs.m;
  552: 
  553: 		/*
  554: 		 * Move on to the next object.  Lock the next object before
  555: 		 * unlocking the current one.
  556: 		 */
  557: 
  558: 		fs.pindex += OFF_TO_IDX(fs.object->backing_object_offset);
  559: 		next_object = fs.object->backing_object;
  560: 		if (next_object == NULL) {
  561: 			/*
  562: 			 * If there's no object left, fill the page in the top
  563: 			 * object with zeros.
  564: 			 */
  565: 			if (fs.object != fs.first_object) {
  566: 				vm_object_pip_wakeup(fs.object);
  567: 
  568: 				fs.object = fs.first_object;
  569: 				fs.pindex = fs.first_pindex;
  570: 				fs.m = fs.first_m;
  571: 			}
  572: 			fs.first_m = NULL;
  573: 
  574: 			/*
  575: 			 * Zero the page if necessary and mark it valid.
  576: 			 */
  577: 			if ((fs.m->flags & PG_ZERO) == 0) {
  578: 				vm_page_zero_fill(fs.m);
  579: 			} else {
  580: 				mycpu->gd_cnt.v_ozfod++;
  581: 			}
  582: 			mycpu->gd_cnt.v_zfod++;
  583: 			fs.m->valid = VM_PAGE_BITS_ALL;
  584: 			break;	/* break to PAGE HAS BEEN FOUND */
  585: 		} else {
  586: 			if (fs.object != fs.first_object) {
  587: 				vm_object_pip_wakeup(fs.object);
  588: 			}
  589: 			KASSERT(fs.object != next_object, ("object loop %p", next_object));
  590: 			fs.object = next_object;
  591: 			vm_object_pip_add(fs.object, 1);
  592: 		}
  593: 	}
  594: 
  595: 	KASSERT((fs.m->flags & PG_BUSY) != 0,
  596: 	    ("vm_fault: not busy after main loop"));
  597: 
  598: 	/*
  599: 	 * PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
  600: 	 * is held.]
  601: 	 */
  602: 
  603: 	/*
  604: 	 * If the page is being written, but isn't already owned by the
  605: 	 * top-level object, we have to copy it into a new page owned by the
  606: 	 * top-level object.
  607: 	 */
  608: 
  609: 	if (fs.object != fs.first_object) {
  610: 		/*
  611: 		 * We only really need to copy if we want to write it.
  612: 		 */
  613: 
  614: 		if (fault_type & VM_PROT_WRITE) {
  615: 			/*
  616: 			 * This allows pages to be virtually copied from a 
  617: 			 * backing_object into the first_object, where the 
  618: 			 * backing object has no other refs to it, and cannot
  619: 			 * gain any more refs.  Instead of a bcopy, we just 
  620: 			 * move the page from the backing object to the 
  621: 			 * first object.  Note that we must mark the page 
  622: 			 * dirty in the first object so that it will go out 
  623: 			 * to swap when needed.
  624: 			 */
  625: 			if (map_generation == fs.map->timestamp &&
  626: 				/*
  627: 				 * Only one shadow object
  628: 				 */
  629: 				(fs.object->shadow_count == 1) &&
  630: 				/*
  631: 				 * No COW refs, except us
  632: 				 */
  633: 				(fs.object->ref_count == 1) &&
  634: 				/*
  635: 				 * No one else can look this object up
  636: 				 */
  637: 				(fs.object->handle == NULL) &&
  638: 				/*
  639: 				 * No other ways to look the object up
  640: 				 */
  641: 				((fs.object->type == OBJT_DEFAULT) ||
  642: 				 (fs.object->type == OBJT_SWAP)) &&
  643: 				/*
  644: 				 * We don't chase down the shadow chain
  645: 				 */
  646: 				(fs.object == fs.first_object->backing_object) &&
  647: 
  648: 				/*
  649: 				 * grab the lock if we need to
  650: 				 */
  651: 				(fs.lookup_still_valid ||
  652: 				 lockmgr(&fs.map->lock, LK_EXCLUSIVE|LK_NOWAIT, NULL, curthread) == 0)
  653: 			    ) {
  654: 				
  655: 				fs.lookup_still_valid = 1;
  656: 				/*
  657: 				 * get rid of the unnecessary page
  658: 				 */
  659: 				vm_page_protect(fs.first_m, VM_PROT_NONE);
  660: 				vm_page_free(fs.first_m);
  661: 				fs.first_m = NULL;
  662: 
  663: 				/*
  664: 				 * grab the page and put it into the 
  665: 				 * process'es object.  The page is 
  666: 				 * automatically made dirty.
  667: 				 */
  668: 				vm_page_rename(fs.m, fs.first_object, fs.first_pindex);
  669: 				fs.first_m = fs.m;
  670: 				vm_page_busy(fs.first_m);
  671: 				fs.m = NULL;
  672: 				mycpu->gd_cnt.v_cow_optim++;
  673: 			} else {
  674: 				/*
  675: 				 * Oh, well, lets copy it.
  676: 				 */
  677: 				vm_page_copy(fs.m, fs.first_m);
  678: 			}
  679: 
  680: 			if (fs.m) {
  681: 				/*
  682: 				 * We no longer need the old page or object.
  683: 				 */
  684: 				release_page(&fs);
  685: 			}
  686: 
  687: 			/*
  688: 			 * fs.object != fs.first_object due to above 
  689: 			 * conditional
  690: 			 */
  691: 
  692: 			vm_object_pip_wakeup(fs.object);
  693: 
  694: 			/*
  695: 			 * Only use the new page below...
  696: 			 */
  697: 
  698: 			mycpu->gd_cnt.v_cow_faults++;
  699: 			fs.m = fs.first_m;
  700: 			fs.object = fs.first_object;
  701: 			fs.pindex = fs.first_pindex;
  702: 
  703: 		} else {
  704: 			prot &= ~VM_PROT_WRITE;
  705: 		}
  706: 	}
  707: 
  708: 	/*
  709: 	 * We must verify that the maps have not changed since our last
  710: 	 * lookup.
  711: 	 */
  712: 
  713: 	if (!fs.lookup_still_valid &&
  714: 		(fs.map->timestamp != map_generation)) {
  715: 		vm_object_t retry_object;
  716: 		vm_pindex_t retry_pindex;
  717: 		vm_prot_t retry_prot;
  718: 
  719: 		/*
  720: 		 * Since map entries may be pageable, make sure we can take a
  721: 		 * page fault on them.
  722: 		 */
  723: 
  724: 		/*
  725: 		 * Unlock vnode before the lookup to avoid deadlock.   E.G.
  726: 		 * avoid a deadlock between the inode and exec_map that can
  727: 		 * occur due to locks being obtained in different orders.
  728: 		 */
  729: 
  730: 		if (fs.vp != NULL) {
  731: 			vput(fs.vp);
  732: 			fs.vp = NULL;
  733: 		}
  734: 		
  735: 		if (fs.map->infork) {
  736: 			release_page(&fs);
  737: 			unlock_and_deallocate(&fs);
  738: 			goto RetryFault;
  739: 		}
  740: 
  741: 		/*
  742: 		 * To avoid trying to write_lock the map while another process
  743: 		 * has it read_locked (in vm_map_wire), we do not try for
  744: 		 * write permission.  If the page is still writable, we will
  745: 		 * get write permission.  If it is not, or has been marked
  746: 		 * needs_copy, we enter the mapping without write permission,
  747: 		 * and will merely take another fault.
  748: 		 */
  749: 		result = vm_map_lookup(&fs.map, vaddr, fault_type & ~VM_PROT_WRITE,
  750: 		    &fs.entry, &retry_object, &retry_pindex, &retry_prot, &wired);
  751: 		map_generation = fs.map->timestamp;
  752: 
  753: 		/*
  754: 		 * If we don't need the page any longer, put it on the active
  755: 		 * list (the easiest thing to do here).  If no one needs it,
  756: 		 * pageout will grab it eventually.
  757: 		 */
  758: 
  759: 		if (result != KERN_SUCCESS) {
  760: 			release_page(&fs);
  761: 			unlock_and_deallocate(&fs);
  762: 			return (result);
  763: 		}
  764: 		fs.lookup_still_valid = TRUE;
  765: 
  766: 		if ((retry_object != fs.first_object) ||
  767: 		    (retry_pindex != fs.first_pindex)) {
  768: 			release_page(&fs);
  769: 			unlock_and_deallocate(&fs);
  770: 			goto RetryFault;
  771: 		}
  772: 		/*
  773: 		 * Check whether the protection has changed or the object has
  774: 		 * been copied while we left the map unlocked. Changing from
  775: 		 * read to write permission is OK - we leave the page
  776: 		 * write-protected, and catch the write fault. Changing from
  777: 		 * write to read permission means that we can't mark the page
  778: 		 * write-enabled after all.
  779: 		 */
  780: 		prot &= retry_prot;
  781: 	}
  782: 
  783: 	/*
  784: 	 * Put this page into the physical map. We had to do the unlock above
  785: 	 * because pmap_enter may cause other faults.   We don't put the page
  786: 	 * back on the active queue until later so that the page-out daemon
  787: 	 * won't find us (yet).
  788: 	 */
  789: 
  790: 	if (prot & VM_PROT_WRITE) {
  791: 		vm_page_flag_set(fs.m, PG_WRITEABLE);
  792: 		vm_object_set_writeable_dirty(fs.m->object);
  793: 
  794: 		/*
  795: 		 * If the fault is a write, we know that this page is being
  796: 		 * written NOW so dirty it explicitly to save on 
  797: 		 * pmap_is_modified() calls later.
  798: 		 *
  799: 		 * If this is a NOSYNC mmap we do not want to set PG_NOSYNC
  800: 		 * if the page is already dirty to prevent data written with
  801: 		 * the expectation of being synced from not being synced.
  802: 		 * Likewise if this entry does not request NOSYNC then make
  803: 		 * sure the page isn't marked NOSYNC.  Applications sharing
  804: 		 * data should use the same flags to avoid ping ponging.
  805: 		 *
  806: 		 * Also tell the backing pager, if any, that it should remove
  807: 		 * any swap backing since the page is now dirty.
  808: 		 */
  809: 		if (fs.entry->eflags & MAP_ENTRY_NOSYNC) {
  810: 			if (fs.m->dirty == 0)
  811: 				vm_page_flag_set(fs.m, PG_NOSYNC);
  812: 		} else {
  813: 			vm_page_flag_clear(fs.m, PG_NOSYNC);
  814: 		}
  815: 		if (fault_flags & VM_FAULT_DIRTY) {
  816: 			int s;
  817: 			vm_page_dirty(fs.m);
  818: 			s = splvm();
  819: 			vm_pager_page_unswapped(fs.m);
  820: 			splx(s);
  821: 		}
  822: 	}
  823: 
  824: 	/*
  825: 	 * Page had better still be busy
  826: 	 */
  827: 
  828: 	KASSERT(fs.m->flags & PG_BUSY,
  829: 		("vm_fault: page %p not busy!", fs.m));
  830: 
  831: 	unlock_things(&fs);
  832: 
  833: 	/*
  834: 	 * Sanity check: page must be completely valid or it is not fit to
  835: 	 * map into user space.  vm_pager_get_pages() ensures this.
  836: 	 */
  837: 
  838: 	if (fs.m->valid != VM_PAGE_BITS_ALL) {
  839: 		vm_page_zero_invalid(fs.m, TRUE);
  840: 		printf("Warning: page %p partially invalid on fault\n", fs.m);
  841: 	}
  842: 
  843: 	pmap_enter(fs.map->pmap, vaddr, fs.m, prot, wired);
  844: 
  845: 	if (((fault_flags & VM_FAULT_WIRE_MASK) == 0) && (wired == 0)) {
  846: 		pmap_prefault(fs.map->pmap, vaddr, fs.entry);
  847: 	}
  848: 
  849: 	vm_page_flag_clear(fs.m, PG_ZERO);
  850: 	vm_page_flag_set(fs.m, PG_MAPPED|PG_REFERENCED);
  851: 	if (fault_flags & VM_FAULT_HOLD)
  852: 		vm_page_hold(fs.m);
  853: 
  854: 	/*
  855: 	 * If the page is not wired down, then put it where the pageout daemon
  856: 	 * can find it.
  857: 	 */
  858: 
  859: 	if (fault_flags & VM_FAULT_WIRE_MASK) {
  860: 		if (wired)
  861: 			vm_page_wire(fs.m);
  862: 		else
  863: 			vm_page_unwire(fs.m, 1);
  864: 	} else {
  865: 		vm_page_activate(fs.m);
  866: 	}
  867: 
  868: 	if (curproc && (curproc->p_flag & P_INMEM) && curproc->p_stats) {
  869: 		if (hardfault) {
  870: 			curproc->p_stats->p_ru.ru_majflt++;
  871: 		} else {
  872: 			curproc->p_stats->p_ru.ru_minflt++;
  873: 		}
  874: 	}
  875: 
  876: 	/*
  877: 	 * Unlock everything, and return
  878: 	 */
  879: 
  880: 	vm_page_wakeup(fs.m);
  881: 	vm_object_deallocate(fs.first_object);
  882: 
  883: 	return (KERN_SUCCESS);
  884: 
  885: }
  886: 
  887: /*
  888:  * quick version of vm_fault
  889:  */
  890: int
  891: vm_fault_quick(caddr_t v, int prot)
  892: {
  893: 	int r;
  894: 
  895: 	if (prot & VM_PROT_WRITE)
  896: 		r = subyte(v, fubyte(v));
  897: 	else
  898: 		r = fubyte(v);
  899: 	return(r);
  900: }
  901: 
  902: /*
  903:  *	vm_fault_wire:
  904:  *
  905:  *	Wire down a range of virtual addresses in a map.
  906:  */
  907: int
  908: vm_fault_wire(vm_map_t map, vm_offset_t start, vm_offset_t end)
  909: {
  910: 
  911: 	vm_offset_t va;
  912: 	pmap_t pmap;
  913: 	int rv;
  914: 
  915: 	pmap = vm_map_pmap(map);
  916: 
  917: 	/*
  918: 	 * Inform the physical mapping system that the range of addresses may
  919: 	 * not fault, so that page tables and such can be locked down as well.
  920: 	 */
  921: 
  922: 	pmap_pageable(pmap, start, end, FALSE);
  923: 
  924: 	/*
  925: 	 * We simulate a fault to get the page and enter it in the physical
  926: 	 * map.
  927: 	 */
  928: 
  929: 	for (va = start; va < end; va += PAGE_SIZE) {
  930: 		rv = vm_fault(map, va, VM_PROT_READ|VM_PROT_WRITE,
  931: 			VM_FAULT_CHANGE_WIRING);
  932: 		if (rv) {
  933: 			if (va != start)
  934: 				vm_fault_unwire(map, start, va);
  935: 			return (rv);
  936: 		}
  937: 	}
  938: 	return (KERN_SUCCESS);
  939: }
  940: 
  941: /*
  942:  *	vm_fault_user_wire:
  943:  *
  944:  *	Wire down a range of virtual addresses in a map.  This
  945:  *	is for user mode though, so we only ask for read access
  946:  *	on currently read only sections.
  947:  */
  948: int
  949: vm_fault_user_wire(vm_map_t map, vm_offset_t start, vm_offset_t end)
  950: {
  951: 
  952: 	vm_offset_t va;
  953: 	pmap_t pmap;
  954: 	int rv;
  955: 
  956: 	pmap = vm_map_pmap(map);
  957: 
  958: 	/*
  959: 	 * Inform the physical mapping system that the range of addresses may
  960: 	 * not fault, so that page tables and such can be locked down as well.
  961: 	 */
  962: 
  963: 	pmap_pageable(pmap, start, end, FALSE);
  964: 
  965: 	/*
  966: 	 * We simulate a fault to get the page and enter it in the physical
  967: 	 * map.
  968: 	 */
  969: 	for (va = start; va < end; va += PAGE_SIZE) {
  970: 		rv = vm_fault(map, va, VM_PROT_READ, VM_FAULT_USER_WIRE);
  971: 		if (rv) {
  972: 			if (va != start)
  973: 				vm_fault_unwire(map, start, va);
  974: 			return (rv);
  975: 		}
  976: 	}
  977: 	return (KERN_SUCCESS);
  978: }
  979: 
  980: 
  981: /*
  982:  *	vm_fault_unwire:
  983:  *
  984:  *	Unwire a range of virtual addresses in a map.
  985:  */
  986: void
  987: vm_fault_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end)
  988: {
  989: 
  990: 	vm_offset_t va;
  991: 	vm_paddr_t pa;
  992: 	pmap_t pmap;
  993: 
  994: 	pmap = vm_map_pmap(map);
  995: 
  996: 	/*
  997: 	 * Since the pages are wired down, we must be able to get their
  998: 	 * mappings from the physical map system.
  999: 	 */
 1000: 
 1001: 	for (va = start; va < end; va += PAGE_SIZE) {
 1002: 		pa = pmap_extract(pmap, va);
 1003: 		if (pa != 0) {
 1004: 			pmap_change_wiring(pmap, va, FALSE);
 1005: 			vm_page_unwire(PHYS_TO_VM_PAGE(pa), 1);
 1006: 		}
 1007: 	}
 1008: 
 1009: 	/*
 1010: 	 * Inform the physical mapping system that the range of addresses may
 1011: 	 * fault, so that page tables and such may be unwired themselves.
 1012: 	 */
 1013: 
 1014: 	pmap_pageable(pmap, start, end, TRUE);
 1015: 
 1016: }
 1017: 
 1018: /*
 1019:  *	Routine:
 1020:  *		vm_fault_copy_entry
 1021:  *	Function:
 1022:  *		Copy all of the pages from a wired-down map entry to another.
 1023:  *
 1024:  *	In/out conditions:
 1025:  *		The source and destination maps must be locked for write.
 1026:  *		The source map entry must be wired down (or be a sharing map
 1027:  *		entry corresponding to a main map entry that is wired down).
 1028:  */
 1029: 
 1030: void
 1031: vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map,
 1032:     vm_map_entry_t dst_entry, vm_map_entry_t src_entry)
 1033: {
 1034: 	vm_object_t dst_object;
 1035: 	vm_object_t src_object;
 1036: 	vm_ooffset_t dst_offset;
 1037: 	vm_ooffset_t src_offset;
 1038: 	vm_prot_t prot;
 1039: 	vm_offset_t vaddr;
 1040: 	vm_page_t dst_m;
 1041: 	vm_page_t src_m;
 1042: 
 1043: #ifdef	lint
 1044: 	src_map++;
 1045: #endif	/* lint */
 1046: 
 1047: 	src_object = src_entry->object.vm_object;
 1048: 	src_offset = src_entry->offset;
 1049: 
 1050: 	/*
 1051: 	 * Create the top-level object for the destination entry. (Doesn't
 1052: 	 * actually shadow anything - we copy the pages directly.)
 1053: 	 */
 1054: 	dst_object = vm_object_allocate(OBJT_DEFAULT,
 1055: 	    (vm_size_t) OFF_TO_IDX(dst_entry->end - dst_entry->start));
 1056: 
 1057: 	dst_entry->object.vm_object = dst_object;
 1058: 	dst_entry->offset = 0;
 1059: 
 1060: 	prot = dst_entry->max_protection;
 1061: 
 1062: 	/*
 1063: 	 * Loop through all of the pages in the entry's range, copying each
 1064: 	 * one from the source object (it should be there) to the destination
 1065: 	 * object.
 1066: 	 */
 1067: 	for (vaddr = dst_entry->start, dst_offset = 0;
 1068: 	    vaddr < dst_entry->end;
 1069: 	    vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
 1070: 
 1071: 		/*
 1072: 		 * Allocate a page in the destination object
 1073: 		 */
 1074: 		do {
 1075: 			dst_m = vm_page_alloc(dst_object,
 1076: 				OFF_TO_IDX(dst_offset), VM_ALLOC_NORMAL);
 1077: 			if (dst_m == NULL) {
 1078: 				VM_WAIT;
 1079: 			}
 1080: 		} while (dst_m == NULL);
 1081: 
 1082: 		/*
 1083: 		 * Find the page in the source object, and copy it in.
 1084: 		 * (Because the source is wired down, the page will be in
 1085: 		 * memory.)
 1086: 		 */
 1087: 		src_m = vm_page_lookup(src_object,
 1088: 			OFF_TO_IDX(dst_offset + src_offset));
 1089: 		if (src_m == NULL)
 1090: 			panic("vm_fault_copy_wired: page missing");
 1091: 
 1092: 		vm_page_copy(src_m, dst_m);
 1093: 
 1094: 		/*
 1095: 		 * Enter it in the pmap...
 1096: 		 */
 1097: 
 1098: 		vm_page_flag_clear(dst_m, PG_ZERO);
 1099: 		pmap_enter(dst_map->pmap, vaddr, dst_m, prot, FALSE);
 1100: 		vm_page_flag_set(dst_m, PG_WRITEABLE|PG_MAPPED);
 1101: 
 1102: 		/*
 1103: 		 * Mark it no longer busy, and put it on the active list.
 1104: 		 */
 1105: 		vm_page_activate(dst_m);
 1106: 		vm_page_wakeup(dst_m);
 1107: 	}
 1108: }
 1109: 
 1110: 
 1111: /*
 1112:  * This routine checks around the requested page for other pages that
 1113:  * might be able to be faulted in.  This routine brackets the viable
 1114:  * pages for the pages to be paged in.
 1115:  *
 1116:  * Inputs:
 1117:  *	m, rbehind, rahead
 1118:  *
 1119:  * Outputs:
 1120:  *  marray (array of vm_page_t), reqpage (index of requested page)
 1121:  *
 1122:  * Return value:
 1123:  *  number of pages in marray
 1124:  */
 1125: static int
 1126: vm_fault_additional_pages(vm_page_t m, int rbehind, int rahead,
 1127:     vm_page_t *marray, int *reqpage)
 1128: {
 1129: 	int i,j;
 1130: 	vm_object_t object;
 1131: 	vm_pindex_t pindex, startpindex, endpindex, tpindex;
 1132: 	vm_page_t rtm;
 1133: 	int cbehind, cahead;
 1134: 
 1135: 	object = m->object;
 1136: 	pindex = m->pindex;
 1137: 
 1138: 	/*
 1139: 	 * we don't fault-ahead for device pager
 1140: 	 */
 1141: 	if (object->type == OBJT_DEVICE) {
 1142: 		*reqpage = 0;
 1143: 		marray[0] = m;
 1144: 		return 1;
 1145: 	}
 1146: 
 1147: 	/*
 1148: 	 * if the requested page is not available, then give up now
 1149: 	 */
 1150: 
 1151: 	if (!vm_pager_has_page(object, pindex, &cbehind, &cahead)) {
 1152: 		return 0;
 1153: 	}
 1154: 
 1155: 	if ((cbehind == 0) && (cahead == 0)) {
 1156: 		*reqpage = 0;
 1157: 		marray[0] = m;
 1158: 		return 1;
 1159: 	}
 1160: 
 1161: 	if (rahead > cahead) {
 1162: 		rahead = cahead;
 1163: 	}
 1164: 
 1165: 	if (rbehind > cbehind) {
 1166: 		rbehind = cbehind;
 1167: 	}
 1168: 
 1169: 	/*
 1170: 	 * try to do any readahead that we might have free pages for.
 1171: 	 */
 1172: 	if ((rahead + rbehind) >
 1173: 		((vmstats.v_free_count + vmstats.v_cache_count) - vmstats.v_free_reserved)) {
 1174: 		pagedaemon_wakeup();
 1175: 		marray[0] = m;
 1176: 		*reqpage = 0;
 1177: 		return 1;
 1178: 	}
 1179: 
 1180: 	/*
 1181: 	 * scan backward for the read behind pages -- in memory 
 1182: 	 */
 1183: 	if (pindex > 0) {
 1184: 		if (rbehind > pindex) {
 1185: 			rbehind = pindex;
 1186: 			startpindex = 0;
 1187: 		} else {
 1188: 			startpindex = pindex - rbehind;
 1189: 		}
 1190: 
 1191: 		for ( tpindex = pindex - 1; tpindex >= startpindex; tpindex -= 1) {
 1192: 			if (vm_page_lookup( object, tpindex)) {
 1193: 				startpindex = tpindex + 1;
 1194: 				break;
 1195: 			}
 1196: 			if (tpindex == 0)
 1197: 				break;
 1198: 		}
 1199: 
 1200: 		for(i = 0, tpindex = startpindex; tpindex < pindex; i++, tpindex++) {
 1201: 
 1202: 			rtm = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
 1203: 			if (rtm == NULL) {
 1204: 				for (j = 0; j < i; j++) {
 1205: 					vm_page_free(marray[j]);
 1206: 				}
 1207: 				marray[0] = m;
 1208: 				*reqpage = 0;
 1209: 				return 1;
 1210: 			}
 1211: 
 1212: 			marray[i] = rtm;
 1213: 		}
 1214: 	} else {
 1215: 		startpindex = 0;
 1216: 		i = 0;
 1217: 	}
 1218: 
 1219: 	marray[i] = m;
 1220: 	/* page offset of the required page */
 1221: 	*reqpage = i;
 1222: 
 1223: 	tpindex = pindex + 1;
 1224: 	i++;
 1225: 
 1226: 	/*
 1227: 	 * scan forward for the read ahead pages
 1228: 	 */
 1229: 	endpindex = tpindex + rahead;
 1230: 	if (endpindex > object->size)
 1231: 		endpindex = object->size;
 1232: 
 1233: 	for( ; tpindex < endpindex; i++, tpindex++) {
 1234: 
 1235: 		if (vm_page_lookup(object, tpindex)) {
 1236: 			break;
 1237: 		}
 1238: 
 1239: 		rtm = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
 1240: 		if (rtm == NULL) {
 1241: 			break;
 1242: 		}
 1243: 
 1244: 		marray[i] = rtm;
 1245: 	}
 1246: 
 1247: 	/* return number of bytes of pages */
 1248: 	return i;
 1249: }