* Copyright (c) 1991 Regents of the University of California.
* This code is derived from software contributed to Berkeley by
* The Mach Operating System project at Carnegie-Mellon University.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the University of
* California, Berkeley and its contributors.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* from: @(#)vm_fault.c 7.6 (Berkeley) 5/7/91
* $Id: vm_fault.c,v 1.6 1993/11/07 17:54:09 wollman Exp $
* Copyright (c) 1987, 1990 Carnegie-Mellon University.
* Authors: Avadis Tevanian, Jr., Michael Wayne Young
* Permission to use, copy, modify and distribute this software and
* its documentation is hereby granted, provided that both the copyright
* notice and this permission notice appear in all copies of the
* software, derivative works or modified versions, and any portions
* thereof, and that both notices appear in supporting documentation.
* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
* FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
* Carnegie Mellon requests users of this software to return to
* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
* School of Computer Science
* Carnegie Mellon University
* Pittsburgh PA 15213-3890
* any improvements or extensions that they make and grant Carnegie the
* rights to redistribute these changes.
* Page fault handling module.
vm_statistics_data_t vm_stat
;
* Handle a page fault occuring at the given address,
* requiring the given permissions, in the map specified.
* If successful, the page is inserted into the
* associated physical map.
* NOTE: the given address should be truncated to the
* KERN_SUCCESS is returned if the page fault is handled; otherwise,
* a standard error specifying why the fault is fatal is returned.
* The map in question must be referenced, and remains so.
* Caller may hold no locks.
vm_fault(map
, vaddr
, fault_type
, change_wiring
)
vm_object_t first_object
;
vm_offset_t first_offset
;
register vm_object_t object
;
register vm_offset_t offset
;
boolean_t lookup_still_valid
;
vm_stat
.faults
++; /* needs lock XXX */
vm_page_unlock_queues(); \
#define RELEASE_PAGE(m) { \
vm_page_unlock_queues(); \
if (lookup_still_valid) { \
vm_map_lookup_done(map, entry); \
lookup_still_valid = FALSE; \
#define UNLOCK_THINGS { \
object->paging_in_progress--; \
vm_object_unlock(object); \
if (object != first_object) { \
vm_object_lock(first_object); \
first_object->paging_in_progress--; \
vm_object_unlock(first_object); \
#define UNLOCK_AND_DEALLOCATE { \
vm_object_deallocate(first_object); \
* Find the backing store object and offset into
* it to begin the search.
if ((result
= vm_map_lookup(&map
, vaddr
, fault_type
, &entry
,
&first_object
, &first_offset
,
&prot
, &wired
, &su
)) != KERN_SUCCESS
) {
lookup_still_valid
= TRUE
;
* Make a reference to this object to
* prevent its disposal while we are messing with
* it. Once we have the reference, the map is free
* to be diddled. Since objects reference their
* shadows (and copies), they will stay around as well.
vm_object_lock(first_object
);
first_object
->ref_count
++;
first_object
->paging_in_progress
++;
* INVARIANTS (through entire routine):
* 1) At all times, we must either have the object
* lock or a busy page in some object to prevent
* some other thread from trying to bring in
* Note that we cannot hold any locks during the
* pager access or when waiting for memory, so
* we use a busy page then.
* Note also that we aren't as concerned about
* more than one thead attempting to pager_data_unlock
* the same page at once, so we don't hold the page
* as busy then, but do record the highest unlock
* value so far. [Unlock requests may also be delivered
* 2) Once we have a busy page, we must remove it from
* the pageout queues, so that the pageout daemon
* 3) To prevent another thread from racing us down the
* shadow chain and entering a new page in the top
* object before we do, we must keep a busy page in
* the top object while following the shadow chain.
* 4) We must increment paging_in_progress on any object
* for which we have a busy page, to prevent
* vm_object_collapse from removing the busy page
* Search for the page at object/offset.
* See whether this page is resident
m
= vm_page_lookup(object
, offset
);
* If the page is being brought in,
* wait for it and then retry.
PAGE_ASSERT_WAIT(m
, !change_wiring
);
wait_result
= current_thread()->wait_result
;
vm_object_deallocate(first_object
);
if (wait_result
!= THREAD_AWAKENED
)
PAGE_ASSERT_WAIT(m
, !change_wiring
);
thread_wakeup((int)&vm_pages_needed
);/* XXX! */
vm_object_deallocate(first_object
);
panic("vm_fault: absent");
* If the desired access to this page has
* been locked out, request that it be unlocked.
if (fault_type
& m
->page_lock
) {
if ((fault_type
& m
->unlock_request
) != fault_type
)
panic("vm_fault: pager_data_unlock");
PAGE_ASSERT_WAIT(m
, !change_wiring
);
wait_result
= current_thread()->wait_result
;
vm_object_deallocate(first_object
);
if (wait_result
!= THREAD_AWAKENED
)
if ((fault_type
& m
->unlock_request
) != fault_type
)
panic("vm_fault: pager_data_unlock");
PAGE_ASSERT_WAIT(m
, !change_wiring
);
thread_wakeup((int)&vm_pages_needed
); /* XXX */
vm_object_deallocate(first_object
);
* Remove the page from the pageout daemon's
* reach while we play with it.
queue_remove(&vm_page_queue_inactive
, m
,
vm_page_inactive_count
--;
queue_remove(&vm_page_queue_active
, m
,
* Mark page busy for other threads.
if (((object
->pager
!= NULL
) &&
(!change_wiring
|| wired
))
|| (object
== first_object
)) {
* Allocate a new page for this object/offset
m
= vm_page_alloc(object
, offset
);
if ((object
->pager
!= NULL
) &&
(!change_wiring
|| wired
)) {
* Now that we have a busy page, we can
* release the object lock.
vm_object_unlock(object
);
* Call the pager to retrieve the data, if any,
* after releasing the lock on the map.
rv
= vm_pager_get(object
->pager
, m
, TRUE
);
* Leave it busy while we play with it.
* Relookup in case pager changed page.
* Pager is responsible for disposition
m
= vm_page_lookup(object
, offset
);
pmap_clear_modify(VM_PAGE_TO_PHYS(m
));
* Remove the bogus page (which does not
* exist at this object/offset); before
* doing so, we must get back our object
* lock to preserve our invariant.
* Also wake up any other thread that may want
* If this is the top-level object, we must
* leave the busy page to prevent another
* thread from rushing past us, and inserting
* the page in that object at the same time
* Data outside the range of the pager; an error
if (rv
== VM_PAGER_BAD
) {
return(KERN_PROTECTION_FAILURE
); /* XXX */
if (object
!= first_object
) {
* XXX - we cannot just fall out at this
* point, m has been freed and is invalid!
* We get here if the object has no pager (or unwiring)
* or the pager doesn't have the page.
if (object
== first_object
)
* Move on to the next object. Lock the next
* object before unlocking the current one.
offset
+= object
->shadow_offset
;
next_object
= object
->shadow
;
if (next_object
== NULL
) {
* If there's no object left, fill the page
* in the top object with zeros.
if (object
!= first_object
) {
object
->paging_in_progress
--;
vm_object_unlock(object
);
vm_stat
.zero_fill_count
++;
vm_object_lock(next_object
);
if (object
!= first_object
)
object
->paging_in_progress
--;
vm_object_unlock(object
);
object
->paging_in_progress
++;
if (m
->absent
|| m
->active
|| m
->inactive
|| !m
->busy
)
panic("vm_fault: absent or active or inactive or not busy after main loop");
* [Loop invariant still holds -- the object lock
old_m
= m
; /* save page that would be copied */
* If the page is being written, but isn't
* already owned by the top-level object,
* we have to copy it into a new page owned
* by the top-level object.
if (object
!= first_object
) {
* We only really need to copy if we
if (fault_type
& VM_PROT_WRITE
) {
* If we try to collapse first_object at this
* point, we may deadlock when we try to get
* the lock on an intermediate object (since we
* have the bottom object locked). We can't
* unlock the bottom object, because the page
* we found may move (by collapse) if we do.
* Instead, we first copy the page. Then, when
* we have no more use for the bottom object,
* we unlock it and try to collapse.
* Note that we copy the page even if we didn't
* need to... that's the breaks.
* We already have an empty page in
vm_page_copy(m
, first_m
);
* If another map is truly sharing this
* page with us, we have to flush all
* uses of the original page, since we
* can't distinguish those which want the
* original from those which need the
* XXX If we know that only one map has
* access to this page, then we could
* avoid the pmap_page_protect() call.
pmap_page_protect(VM_PAGE_TO_PHYS(m
), VM_PROT_NONE
);
* We no longer need the old page or object.
object
->paging_in_progress
--;
vm_object_unlock(object
);
* Only use the new page below...
* Now that we've gotten the copy out of the
* way, let's try to collapse the top object.
* But we have to play ugly games with
* paging_in_progress to do that...
object
->paging_in_progress
--;
vm_object_collapse(object
);
object
->paging_in_progress
++;
prot
&= (~VM_PROT_WRITE
);
if (m
->active
|| m
->inactive
)
panic("vm_fault: active or inactive before copy object handling");
* If the page is being written, but hasn't been
* copied to the copy-object, we have to copy it there.
if (first_object
->copy
!= NULL
) {
vm_object_t copy_object
= first_object
->copy
;
* We only need to copy if we want to write it.
if ((fault_type
& VM_PROT_WRITE
) == 0) {
* Try to get the lock on the copy_object.
if (!vm_object_lock_try(copy_object
)) {
vm_object_unlock(object
);
/* should spin a bit here... */
* Make another reference to the copy-object,
* to keep it from disappearing during the
copy_object
->ref_count
++;
* Does the page exist in the copy?
copy_offset
= first_offset
- copy_object
->shadow_offset
;
copy_m
= vm_page_lookup(copy_object
, copy_offset
);
if (page_exists
= (copy_m
!= NULL
)) {
* If the page is being brought
* in, wait for it and then retry.
PAGE_ASSERT_WAIT(copy_m
, !change_wiring
);
copy_object
->ref_count
--;
vm_object_unlock(copy_object
);
wait_result
= current_thread()->wait_result
;
vm_object_deallocate(first_object
);
if (wait_result
!= THREAD_AWAKENED
)
* If the page is being brought
* in, wait for it and then retry.
PAGE_ASSERT_WAIT(copy_m
, !change_wiring
);
copy_object
->ref_count
--;
vm_object_unlock(copy_object
);
thread_wakeup((int)&vm_pages_needed
);
vm_object_deallocate(first_object
);
* If the page is not in memory (in the object)
* and the object has a pager, we have to check
* if the pager has the data in secondary
* If we don't allocate a (blank) page
* here... another thread could try
* to page it in, allocate a page, and
* then block on the busy page in its
* shadow (first_object). Then we'd
* trip over the busy page after we
* found that the copy_object's pager
* doesn't have the page...
copy_m
= vm_page_alloc(copy_object
,
* Wait for a page, then retry.
copy_object
->ref_count
--;
vm_object_unlock(copy_object
);
if (copy_object
->pager
!= NULL
) {
vm_object_unlock(object
);
vm_object_unlock(copy_object
);
page_exists
= vm_pager_has_page(
(copy_offset
+ copy_object
->paging_offset
));
vm_object_lock(copy_object
);
* Since the map is unlocked, someone
* else could have copied this object
* and put a different copy_object
* between the two. Or, the last
* reference to the copy-object (other
* than the one we have) may have
* disappeared - if that has happened,
* we don't need to make the copy.
if (copy_object
->shadow
!= object
||
copy_object
->ref_count
== 1) {
vm_object_unlock(copy_object
);
vm_object_deallocate(copy_object
);
* We didn't need the page
* Must copy page into copy-object.
* 1. The copied page must be marked 'dirty'
* so it will be paged out to the copy
* 2. If the old page was in use by any users
* of the copy-object, it must be removed
* from all pmaps. (We can't know which
pmap_page_protect(VM_PAGE_TO_PHYS(old_m
),
vm_page_activate(copy_m
); /* XXX */
* The reference count on copy_object must be
* at least 2: one for our extra reference,
* and at least one from the outside world
* (we checked that when we last locked
copy_object
->ref_count
--;
vm_object_unlock(copy_object
);
m
->copy_on_write
= FALSE
;
if (m
->active
|| m
->inactive
)
panic("vm_fault: active or inactive before retrying lookup");
* We must verify that the maps have not changed
if (!lookup_still_valid
) {
vm_object_t retry_object
;
vm_offset_t retry_offset
;
* Since map entries may be pageable, make sure we can
* take a page fault on them.
vm_object_unlock(object
);
* To avoid trying to write_lock the map while another
* thread has it read_locked (in vm_map_pageable), we
* do not try for write permission. If the page is
* still writable, we will get write permission. If it
* is not, or has been marked needs_copy, we enter the
* mapping without write permission, and will merely
result
= vm_map_lookup(&map
, vaddr
,
fault_type
& ~VM_PROT_WRITE
, &entry
,
&retry_object
, &retry_offset
, &retry_prot
,
* If we don't need the page any longer, put it on the
* active list (the easiest thing to do here). If no
* one needs it, pageout will grab it eventually.
if (result
!= KERN_SUCCESS
) {
lookup_still_valid
= TRUE
;
if ((retry_object
!= first_object
) ||
(retry_offset
!= first_offset
)) {
* Check whether the protection has changed or the object
* has been copied while we left the map unlocked.
* Changing from read to write permission is OK - we leave
* the page write-protected, and catch the write fault.
* Changing from write to read permission means that we
* can't mark the page write-enabled after all.
* (the various bits we're fiddling with here are locked by
/* XXX This distorts the meaning of the copy_on_write bit */
if (prot
& VM_PROT_WRITE
)
m
->copy_on_write
= FALSE
;
* It's critically important that a wired-down page be faulted
* only once in each map for which it is wired.
if (m
->active
|| m
->inactive
)
panic("vm_fault: active or inactive before pmap_enter");
vm_object_unlock(object
);
* Put this page into the physical map.
* We had to do the unlock above because pmap_enter
* may cause other faults. We don't put the
* page back on the active queue until later so
* that the page-out daemon won't find us (yet).
pmap_enter(map
->pmap
, vaddr
, VM_PAGE_TO_PHYS(m
),
prot
& ~(m
->page_lock
), wired
);
* If the page is not wired down, then put it where the
* pageout daemon can find it.
* Unlock everything, and return
* Wire down a range of virtual addresses in a map.
void vm_fault_wire(map
, start
, end
)
* Inform the physical mapping system that the
* range of addresses may not fault, so that
* page tables and such can be locked down as well.
pmap_pageable(pmap
, start
, end
, FALSE
);
* We simulate a fault to get the page and enter it
for (va
= start
; va
< end
; va
+= PAGE_SIZE
) {
(void) vm_fault(map
, va
, VM_PROT_NONE
, TRUE
);
* Unwire a range of virtual addresses in a map.
void vm_fault_unwire(map
, start
, end
)
register vm_offset_t va
, pa
;
* Since the pages are wired down, we must be able to
* get their mappings from the physical map system.
for (va
= start
; va
< end
; va
+= PAGE_SIZE
) {
pa
= pmap_extract(pmap
, va
);
if (pa
== (vm_offset_t
) 0) {
panic("unwire: page not in pmap");
pmap_change_wiring(pmap
, va
, FALSE
);
vm_page_unwire(PHYS_TO_VM_PAGE(pa
));
* Inform the physical mapping system that the range
* of addresses may fault, so that page tables and
* such may be unwired themselves.
pmap_pageable(pmap
, start
, end
, TRUE
);
* Copy all of the pages from a wired-down map entry to another.
* The source and destination maps must be locked for write.
* The source map entry must be wired down (or be a sharing map
* entry corresponding to a main map entry that is wired down).
void vm_fault_copy_entry(dst_map
, src_map
, dst_entry
, src_entry
)
vm_map_entry_t dst_entry
;
vm_map_entry_t src_entry
;
src_object
= src_entry
->object
.vm_object
;
src_offset
= src_entry
->offset
;
* Create the top-level object for the destination entry.
* (Doesn't actually shadow anything - we copy the pages
dst_object
= vm_object_allocate(
(vm_size_t
) (dst_entry
->end
- dst_entry
->start
));
dst_entry
->object
.vm_object
= dst_object
;
prot
= dst_entry
->max_protection
;
* Loop through all of the pages in the entry's range, copying
* each one from the source object (it should be there) to the
for (vaddr
= dst_entry
->start
, dst_offset
= 0;
vaddr
+= PAGE_SIZE
, dst_offset
+= PAGE_SIZE
) {
* Allocate a page in the destination object
vm_object_lock(dst_object
);
dst_m
= vm_page_alloc(dst_object
, dst_offset
);
vm_object_unlock(dst_object
);
vm_object_lock(dst_object
);
* Find the page in the source object, and copy it in.
* (Because the source is wired down, the page will be
vm_object_lock(src_object
);
src_m
= vm_page_lookup(src_object
, dst_offset
+ src_offset
);
panic("vm_fault_copy_wired: page missing");
vm_page_copy(src_m
, dst_m
);
* Enter it in the pmap...
vm_object_unlock(src_object
);
vm_object_unlock(dst_object
);
pmap_enter(dst_map
->pmap
, vaddr
, VM_PAGE_TO_PHYS(dst_m
),
* Mark it no longer busy, and put it on the active list.
vm_object_lock(dst_object
);
vm_object_unlock(dst_object
);