Commit | Line | Data |
---|---|---|
cf5b1e7a WJ |
1 | /* |
2 | * Copyright (c) 1991 Regents of the University of California. | |
3 | * All rights reserved. | |
4 | * | |
5 | * This code is derived from software contributed to Berkeley by | |
6 | * The Mach Operating System project at Carnegie-Mellon University. | |
7 | * | |
8 | * Redistribution and use in source and binary forms, with or without | |
9 | * modification, are permitted provided that the following conditions | |
10 | * are met: | |
11 | * 1. Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * 2. Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in the | |
15 | * documentation and/or other materials provided with the distribution. | |
16 | * 3. All advertising materials mentioning features or use of this software | |
17 | * must display the following acknowledgement: | |
18 | * This product includes software developed by the University of | |
19 | * California, Berkeley and its contributors. | |
20 | * 4. Neither the name of the University nor the names of its contributors | |
21 | * may be used to endorse or promote products derived from this software | |
22 | * without specific prior written permission. | |
23 | * | |
24 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND | |
25 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
26 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
27 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | |
28 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |
29 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | |
30 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | |
31 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | |
32 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |
33 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
34 | * SUCH DAMAGE. | |
35 | * | |
36 | * @(#)vm_fault.c 7.6 (Berkeley) 5/7/91 | |
37 | * | |
38 | * | |
39 | * Copyright (c) 1987, 1990 Carnegie-Mellon University. | |
40 | * All rights reserved. | |
41 | * | |
42 | * Authors: Avadis Tevanian, Jr., Michael Wayne Young | |
43 | * | |
44 | * Permission to use, copy, modify and distribute this software and | |
45 | * its documentation is hereby granted, provided that both the copyright | |
46 | * notice and this permission notice appear in all copies of the | |
47 | * software, derivative works or modified versions, and any portions | |
48 | * thereof, and that both notices appear in supporting documentation. | |
49 | * | |
50 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" | |
51 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND | |
52 | * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. | |
53 | * | |
54 | * Carnegie Mellon requests users of this software to return to | |
55 | * | |
56 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU | |
57 | * School of Computer Science | |
58 | * Carnegie Mellon University | |
59 | * Pittsburgh PA 15213-3890 | |
60 | * | |
61 | * any improvements or extensions that they make and grant Carnegie the | |
62 | * rights to redistribute these changes. | |
63 | */ | |
64 | ||
65 | static char rcsid[] = "$Header: /usr/bill/working/sys/vm/RCS/vm_fault.c,v 1.2 92/01/21 21:58:17 william Exp $"; | |
66 | ||
67 | /* | |
68 | * Page fault handling module. | |
69 | */ | |
70 | ||
71 | #include "param.h" | |
72 | ||
73 | #include "vm.h" | |
74 | #include "vm_page.h" | |
75 | #include "vm_pageout.h" | |
76 | ||
77 | /* | |
78 | * vm_fault: | |
79 | * | |
80 | * Handle a page fault occuring at the given address, | |
81 | * requiring the given permissions, in the map specified. | |
82 | * If successful, the page is inserted into the | |
83 | * associated physical map. | |
84 | * | |
85 | * NOTE: the given address should be truncated to the | |
86 | * proper page address. | |
87 | * | |
88 | * KERN_SUCCESS is returned if the page fault is handled; otherwise, | |
89 | * a standard error specifying why the fault is fatal is returned. | |
90 | * | |
91 | * | |
92 | * The map in question must be referenced, and remains so. | |
93 | * Caller may hold no locks. | |
94 | */ | |
95 | vm_fault(map, vaddr, fault_type, change_wiring) | |
96 | vm_map_t map; | |
97 | vm_offset_t vaddr; | |
98 | vm_prot_t fault_type; | |
99 | boolean_t change_wiring; | |
100 | { | |
101 | vm_object_t first_object; | |
102 | vm_offset_t first_offset; | |
103 | vm_map_entry_t entry; | |
104 | register vm_object_t object; | |
105 | register vm_offset_t offset; | |
106 | register vm_page_t m; | |
107 | vm_page_t first_m; | |
108 | vm_prot_t prot; | |
109 | int result; | |
110 | boolean_t wired; | |
111 | boolean_t su; | |
112 | boolean_t lookup_still_valid; | |
113 | boolean_t page_exists; | |
114 | vm_page_t old_m; | |
115 | vm_object_t next_object; | |
116 | ||
117 | vm_stat.faults++; /* needs lock XXX */ | |
118 | /* | |
119 | * Recovery actions | |
120 | */ | |
121 | #define FREE_PAGE(m) { \ | |
122 | PAGE_WAKEUP(m); \ | |
123 | vm_page_lock_queues(); \ | |
124 | vm_page_free(m); \ | |
125 | vm_page_unlock_queues(); \ | |
126 | } | |
127 | ||
128 | #define RELEASE_PAGE(m) { \ | |
129 | PAGE_WAKEUP(m); \ | |
130 | vm_page_lock_queues(); \ | |
131 | vm_page_activate(m); \ | |
132 | vm_page_unlock_queues(); \ | |
133 | } | |
134 | ||
135 | #define UNLOCK_MAP { \ | |
136 | if (lookup_still_valid) { \ | |
137 | vm_map_lookup_done(map, entry); \ | |
138 | lookup_still_valid = FALSE; \ | |
139 | } \ | |
140 | } | |
141 | ||
142 | #define UNLOCK_THINGS { \ | |
143 | object->paging_in_progress--; \ | |
144 | vm_object_unlock(object); \ | |
145 | if (object != first_object) { \ | |
146 | vm_object_lock(first_object); \ | |
147 | FREE_PAGE(first_m); \ | |
148 | first_object->paging_in_progress--; \ | |
149 | vm_object_unlock(first_object); \ | |
150 | } \ | |
151 | UNLOCK_MAP; \ | |
152 | } | |
153 | ||
154 | #define UNLOCK_AND_DEALLOCATE { \ | |
155 | UNLOCK_THINGS; \ | |
156 | vm_object_deallocate(first_object); \ | |
157 | } | |
158 | ||
159 | RetryFault: ; | |
160 | ||
161 | /* | |
162 | * Find the backing store object and offset into | |
163 | * it to begin the search. | |
164 | */ | |
165 | ||
166 | if ((result = vm_map_lookup(&map, vaddr, fault_type, &entry, | |
167 | &first_object, &first_offset, | |
168 | &prot, &wired, &su)) != KERN_SUCCESS) { | |
169 | return(result); | |
170 | } | |
171 | lookup_still_valid = TRUE; | |
172 | ||
173 | if (wired) | |
174 | fault_type = prot; | |
175 | ||
176 | first_m = NULL; | |
177 | ||
178 | /* | |
179 | * Make a reference to this object to | |
180 | * prevent its disposal while we are messing with | |
181 | * it. Once we have the reference, the map is free | |
182 | * to be diddled. Since objects reference their | |
183 | * shadows (and copies), they will stay around as well. | |
184 | */ | |
185 | ||
186 | vm_object_lock(first_object); | |
187 | ||
188 | first_object->ref_count++; | |
189 | first_object->paging_in_progress++; | |
190 | ||
191 | /* | |
192 | * INVARIANTS (through entire routine): | |
193 | * | |
194 | * 1) At all times, we must either have the object | |
195 | * lock or a busy page in some object to prevent | |
196 | * some other thread from trying to bring in | |
197 | * the same page. | |
198 | * | |
199 | * Note that we cannot hold any locks during the | |
200 | * pager access or when waiting for memory, so | |
201 | * we use a busy page then. | |
202 | * | |
203 | * Note also that we aren't as concerned about | |
204 | * more than one thead attempting to pager_data_unlock | |
205 | * the same page at once, so we don't hold the page | |
206 | * as busy then, but do record the highest unlock | |
207 | * value so far. [Unlock requests may also be delivered | |
208 | * out of order.] | |
209 | * | |
210 | * 2) Once we have a busy page, we must remove it from | |
211 | * the pageout queues, so that the pageout daemon | |
212 | * will not grab it away. | |
213 | * | |
214 | * 3) To prevent another thread from racing us down the | |
215 | * shadow chain and entering a new page in the top | |
216 | * object before we do, we must keep a busy page in | |
217 | * the top object while following the shadow chain. | |
218 | * | |
219 | * 4) We must increment paging_in_progress on any object | |
220 | * for which we have a busy page, to prevent | |
221 | * vm_object_collapse from removing the busy page | |
222 | * without our noticing. | |
223 | */ | |
224 | ||
225 | /* | |
226 | * Search for the page at object/offset. | |
227 | */ | |
228 | ||
229 | object = first_object; | |
230 | offset = first_offset; | |
231 | ||
232 | /* | |
233 | * See whether this page is resident | |
234 | */ | |
235 | ||
236 | while (TRUE) { | |
237 | m = vm_page_lookup(object, offset); | |
238 | if (m != NULL) { | |
239 | /* | |
240 | * If the page is being brought in, | |
241 | * wait for it and then retry. | |
242 | */ | |
243 | if (m->busy) { | |
244 | #ifdef DOTHREADS | |
245 | int wait_result; | |
246 | ||
247 | PAGE_ASSERT_WAIT(m, !change_wiring); | |
248 | UNLOCK_THINGS; | |
249 | thread_block(); | |
250 | wait_result = current_thread()->wait_result; | |
251 | vm_object_deallocate(first_object); | |
252 | if (wait_result != THREAD_AWAKENED) | |
253 | return(KERN_SUCCESS); | |
254 | goto RetryFault; | |
255 | #else | |
256 | PAGE_ASSERT_WAIT(m, !change_wiring); | |
257 | UNLOCK_THINGS; | |
258 | thread_wakeup(&vm_pages_needed); /* XXX! */ | |
259 | thread_block(); | |
260 | vm_object_deallocate(first_object); | |
261 | goto RetryFault; | |
262 | #endif | |
263 | } | |
264 | ||
265 | if (m->absent) | |
266 | panic("vm_fault: absent"); | |
267 | ||
268 | /* | |
269 | * If the desired access to this page has | |
270 | * been locked out, request that it be unlocked. | |
271 | */ | |
272 | ||
273 | if (fault_type & m->page_lock) { | |
274 | #ifdef DOTHREADS | |
275 | int wait_result; | |
276 | ||
277 | if ((fault_type & m->unlock_request) != fault_type) | |
278 | panic("vm_fault: pager_data_unlock"); | |
279 | ||
280 | PAGE_ASSERT_WAIT(m, !change_wiring); | |
281 | UNLOCK_THINGS; | |
282 | thread_block(); | |
283 | wait_result = current_thread()->wait_result; | |
284 | vm_object_deallocate(first_object); | |
285 | if (wait_result != THREAD_AWAKENED) | |
286 | return(KERN_SUCCESS); | |
287 | goto RetryFault; | |
288 | #else | |
289 | if ((fault_type & m->unlock_request) != fault_type) | |
290 | panic("vm_fault: pager_data_unlock"); | |
291 | ||
292 | PAGE_ASSERT_WAIT(m, !change_wiring); | |
293 | UNLOCK_THINGS; | |
294 | thread_wakeup(&vm_pages_needed); /* XXX */ | |
295 | thread_block(); | |
296 | vm_object_deallocate(first_object); | |
297 | goto RetryFault; | |
298 | #endif | |
299 | } | |
300 | ||
301 | /* | |
302 | * Remove the page from the pageout daemon's | |
303 | * reach while we play with it. | |
304 | */ | |
305 | ||
306 | vm_page_lock_queues(); | |
307 | if (m->inactive) { | |
308 | queue_remove(&vm_page_queue_inactive, m, | |
309 | vm_page_t, pageq); | |
310 | m->inactive = FALSE; | |
311 | vm_page_inactive_count--; | |
312 | vm_stat.reactivations++; | |
313 | } | |
314 | ||
315 | if (m->active) { | |
316 | queue_remove(&vm_page_queue_active, m, | |
317 | vm_page_t, pageq); | |
318 | m->active = FALSE; | |
319 | vm_page_active_count--; | |
320 | } | |
321 | vm_page_unlock_queues(); | |
322 | ||
323 | /* | |
324 | * Mark page busy for other threads. | |
325 | */ | |
326 | m->busy = TRUE; | |
327 | m->absent = FALSE; | |
328 | break; | |
329 | } | |
330 | ||
331 | if (((object->pager != NULL) && | |
332 | (!change_wiring || wired)) | |
333 | || (object == first_object)) { | |
334 | ||
335 | /* | |
336 | * Allocate a new page for this object/offset | |
337 | * pair. | |
338 | */ | |
339 | ||
340 | m = vm_page_alloc(object, offset); | |
341 | ||
342 | if (m == NULL) { | |
343 | UNLOCK_AND_DEALLOCATE; | |
344 | VM_WAIT; | |
345 | goto RetryFault; | |
346 | } | |
347 | } | |
348 | ||
349 | if ((object->pager != NULL) && | |
350 | (!change_wiring || wired)) { | |
351 | int rv; | |
352 | ||
353 | /* | |
354 | * Now that we have a busy page, we can | |
355 | * release the object lock. | |
356 | */ | |
357 | vm_object_unlock(object); | |
358 | ||
359 | /* | |
360 | * Call the pager to retrieve the data, if any, | |
361 | * after releasing the lock on the map. | |
362 | */ | |
363 | UNLOCK_MAP; | |
364 | ||
365 | rv = vm_pager_get(object->pager, m, TRUE); | |
366 | if (rv == VM_PAGER_OK) { | |
367 | /* | |
368 | * Found the page. | |
369 | * Leave it busy while we play with it. | |
370 | */ | |
371 | vm_object_lock(object); | |
372 | ||
373 | /* | |
374 | * Relookup in case pager changed page. | |
375 | * Pager is responsible for disposition | |
376 | * of old page if moved. | |
377 | */ | |
378 | m = vm_page_lookup(object, offset); | |
379 | ||
380 | vm_stat.pageins++; | |
381 | m->fake = FALSE; | |
382 | pmap_clear_modify(VM_PAGE_TO_PHYS(m)); | |
383 | break; | |
384 | } | |
385 | ||
386 | /* | |
387 | * Remove the bogus page (which does not | |
388 | * exist at this object/offset); before | |
389 | * doing so, we must get back our object | |
390 | * lock to preserve our invariant. | |
391 | * | |
392 | * Also wake up any other thread that may want | |
393 | * to bring in this page. | |
394 | * | |
395 | * If this is the top-level object, we must | |
396 | * leave the busy page to prevent another | |
397 | * thread from rushing past us, and inserting | |
398 | * the page in that object at the same time | |
399 | * that we are. | |
400 | */ | |
401 | ||
402 | vm_object_lock(object); | |
403 | /* | |
404 | * Data outside the range of the pager; an error | |
405 | */ | |
406 | if (rv == VM_PAGER_BAD) { | |
407 | FREE_PAGE(m); | |
408 | UNLOCK_AND_DEALLOCATE; | |
409 | return(KERN_PROTECTION_FAILURE); /* XXX */ | |
410 | } | |
411 | if (object != first_object) { | |
412 | FREE_PAGE(m); | |
413 | /* | |
414 | * XXX - we cannot just fall out at this | |
415 | * point, m has been freed and is invalid! | |
416 | */ | |
417 | } | |
418 | } | |
419 | ||
420 | /* | |
421 | * We get here if the object has no pager (or unwiring) | |
422 | * or the pager doesn't have the page. | |
423 | */ | |
424 | if (object == first_object) | |
425 | first_m = m; | |
426 | ||
427 | /* | |
428 | * Move on to the next object. Lock the next | |
429 | * object before unlocking the current one. | |
430 | */ | |
431 | ||
432 | offset += object->shadow_offset; | |
433 | next_object = object->shadow; | |
434 | if (next_object == NULL) { | |
435 | /* | |
436 | * If there's no object left, fill the page | |
437 | * in the top object with zeros. | |
438 | */ | |
439 | if (object != first_object) { | |
440 | object->paging_in_progress--; | |
441 | vm_object_unlock(object); | |
442 | ||
443 | object = first_object; | |
444 | offset = first_offset; | |
445 | m = first_m; | |
446 | vm_object_lock(object); | |
447 | } | |
448 | first_m = NULL; | |
449 | ||
450 | vm_page_zero_fill(m); | |
451 | vm_stat.zero_fill_count++; | |
452 | m->fake = FALSE; | |
453 | m->absent = FALSE; | |
454 | break; | |
455 | } | |
456 | else { | |
457 | vm_object_lock(next_object); | |
458 | if (object != first_object) | |
459 | object->paging_in_progress--; | |
460 | vm_object_unlock(object); | |
461 | object = next_object; | |
462 | object->paging_in_progress++; | |
463 | } | |
464 | } | |
465 | ||
466 | if (m->absent || m->active || m->inactive || !m->busy) | |
467 | panic("vm_fault: absent or active or inactive or not busy after main loop"); | |
468 | ||
469 | /* | |
470 | * PAGE HAS BEEN FOUND. | |
471 | * [Loop invariant still holds -- the object lock | |
472 | * is held.] | |
473 | */ | |
474 | ||
475 | old_m = m; /* save page that would be copied */ | |
476 | ||
477 | /* | |
478 | * If the page is being written, but isn't | |
479 | * already owned by the top-level object, | |
480 | * we have to copy it into a new page owned | |
481 | * by the top-level object. | |
482 | */ | |
483 | ||
484 | if (object != first_object) { | |
485 | /* | |
486 | * We only really need to copy if we | |
487 | * want to write it. | |
488 | */ | |
489 | ||
490 | if (fault_type & VM_PROT_WRITE) { | |
491 | ||
492 | /* | |
493 | * If we try to collapse first_object at this | |
494 | * point, we may deadlock when we try to get | |
495 | * the lock on an intermediate object (since we | |
496 | * have the bottom object locked). We can't | |
497 | * unlock the bottom object, because the page | |
498 | * we found may move (by collapse) if we do. | |
499 | * | |
500 | * Instead, we first copy the page. Then, when | |
501 | * we have no more use for the bottom object, | |
502 | * we unlock it and try to collapse. | |
503 | * | |
504 | * Note that we copy the page even if we didn't | |
505 | * need to... that's the breaks. | |
506 | */ | |
507 | ||
508 | /* | |
509 | * We already have an empty page in | |
510 | * first_object - use it. | |
511 | */ | |
512 | ||
513 | vm_page_copy(m, first_m); | |
514 | first_m->fake = FALSE; | |
515 | first_m->absent = FALSE; | |
516 | ||
517 | /* | |
518 | * If another map is truly sharing this | |
519 | * page with us, we have to flush all | |
520 | * uses of the original page, since we | |
521 | * can't distinguish those which want the | |
522 | * original from those which need the | |
523 | * new copy. | |
524 | * | |
525 | * XXX If we know that only one map has | |
526 | * access to this page, then we could | |
527 | * avoid the pmap_page_protect() call. | |
528 | */ | |
529 | ||
530 | vm_page_lock_queues(); | |
531 | vm_page_deactivate(m); | |
532 | pmap_page_protect(VM_PAGE_TO_PHYS(m), VM_PROT_NONE); | |
533 | vm_page_unlock_queues(); | |
534 | ||
535 | /* | |
536 | * We no longer need the old page or object. | |
537 | */ | |
538 | PAGE_WAKEUP(m); | |
539 | object->paging_in_progress--; | |
540 | vm_object_unlock(object); | |
541 | ||
542 | /* | |
543 | * Only use the new page below... | |
544 | */ | |
545 | ||
546 | vm_stat.cow_faults++; | |
547 | m = first_m; | |
548 | object = first_object; | |
549 | offset = first_offset; | |
550 | ||
551 | /* | |
552 | * Now that we've gotten the copy out of the | |
553 | * way, let's try to collapse the top object. | |
554 | */ | |
555 | vm_object_lock(object); | |
556 | /* | |
557 | * But we have to play ugly games with | |
558 | * paging_in_progress to do that... | |
559 | */ | |
560 | object->paging_in_progress--; | |
561 | vm_object_collapse(object); | |
562 | object->paging_in_progress++; | |
563 | } | |
564 | else { | |
565 | prot &= (~VM_PROT_WRITE); | |
566 | m->copy_on_write = TRUE; | |
567 | } | |
568 | } | |
569 | ||
570 | if (m->active || m->inactive) | |
571 | panic("vm_fault: active or inactive before copy object handling"); | |
572 | ||
573 | /* | |
574 | * If the page is being written, but hasn't been | |
575 | * copied to the copy-object, we have to copy it there. | |
576 | */ | |
577 | RetryCopy: | |
578 | if (first_object->copy != NULL) { | |
579 | vm_object_t copy_object = first_object->copy; | |
580 | vm_offset_t copy_offset; | |
581 | vm_page_t copy_m; | |
582 | ||
583 | /* | |
584 | * We only need to copy if we want to write it. | |
585 | */ | |
586 | if ((fault_type & VM_PROT_WRITE) == 0) { | |
587 | prot &= ~VM_PROT_WRITE; | |
588 | m->copy_on_write = TRUE; | |
589 | } | |
590 | else { | |
591 | /* | |
592 | * Try to get the lock on the copy_object. | |
593 | */ | |
594 | if (!vm_object_lock_try(copy_object)) { | |
595 | vm_object_unlock(object); | |
596 | /* should spin a bit here... */ | |
597 | vm_object_lock(object); | |
598 | goto RetryCopy; | |
599 | } | |
600 | ||
601 | /* | |
602 | * Make another reference to the copy-object, | |
603 | * to keep it from disappearing during the | |
604 | * copy. | |
605 | */ | |
606 | copy_object->ref_count++; | |
607 | ||
608 | /* | |
609 | * Does the page exist in the copy? | |
610 | */ | |
611 | copy_offset = first_offset | |
612 | - copy_object->shadow_offset; | |
613 | copy_m = vm_page_lookup(copy_object, copy_offset); | |
614 | if (page_exists = (copy_m != NULL)) { | |
615 | if (copy_m->busy) { | |
616 | #ifdef DOTHREADS | |
617 | int wait_result; | |
618 | ||
619 | /* | |
620 | * If the page is being brought | |
621 | * in, wait for it and then retry. | |
622 | */ | |
623 | PAGE_ASSERT_WAIT(copy_m, !change_wiring); | |
624 | RELEASE_PAGE(m); | |
625 | copy_object->ref_count--; | |
626 | vm_object_unlock(copy_object); | |
627 | UNLOCK_THINGS; | |
628 | thread_block(); | |
629 | wait_result = current_thread()->wait_result; | |
630 | vm_object_deallocate(first_object); | |
631 | if (wait_result != THREAD_AWAKENED) | |
632 | return(KERN_SUCCESS); | |
633 | goto RetryFault; | |
634 | #else | |
635 | /* | |
636 | * If the page is being brought | |
637 | * in, wait for it and then retry. | |
638 | */ | |
639 | PAGE_ASSERT_WAIT(copy_m, !change_wiring); | |
640 | RELEASE_PAGE(m); | |
641 | copy_object->ref_count--; | |
642 | vm_object_unlock(copy_object); | |
643 | UNLOCK_THINGS; | |
644 | thread_wakeup(&vm_pages_needed); /* XXX */ | |
645 | thread_block(); | |
646 | vm_object_deallocate(first_object); | |
647 | goto RetryFault; | |
648 | #endif | |
649 | } | |
650 | } | |
651 | ||
652 | /* | |
653 | * If the page is not in memory (in the object) | |
654 | * and the object has a pager, we have to check | |
655 | * if the pager has the data in secondary | |
656 | * storage. | |
657 | */ | |
658 | if (!page_exists) { | |
659 | ||
660 | /* | |
661 | * If we don't allocate a (blank) page | |
662 | * here... another thread could try | |
663 | * to page it in, allocate a page, and | |
664 | * then block on the busy page in its | |
665 | * shadow (first_object). Then we'd | |
666 | * trip over the busy page after we | |
667 | * found that the copy_object's pager | |
668 | * doesn't have the page... | |
669 | */ | |
670 | copy_m = vm_page_alloc(copy_object, | |
671 | copy_offset); | |
672 | if (copy_m == NULL) { | |
673 | /* | |
674 | * Wait for a page, then retry. | |
675 | */ | |
676 | RELEASE_PAGE(m); | |
677 | copy_object->ref_count--; | |
678 | vm_object_unlock(copy_object); | |
679 | UNLOCK_AND_DEALLOCATE; | |
680 | VM_WAIT; | |
681 | goto RetryFault; | |
682 | } | |
683 | ||
684 | if (copy_object->pager != NULL) { | |
685 | vm_object_unlock(object); | |
686 | vm_object_unlock(copy_object); | |
687 | UNLOCK_MAP; | |
688 | ||
689 | page_exists = vm_pager_has_page( | |
690 | copy_object->pager, | |
691 | (copy_offset + copy_object->paging_offset)); | |
692 | ||
693 | vm_object_lock(copy_object); | |
694 | ||
695 | /* | |
696 | * Since the map is unlocked, someone | |
697 | * else could have copied this object | |
698 | * and put a different copy_object | |
699 | * between the two. Or, the last | |
700 | * reference to the copy-object (other | |
701 | * than the one we have) may have | |
702 | * disappeared - if that has happened, | |
703 | * we don't need to make the copy. | |
704 | */ | |
705 | if (copy_object->shadow != object || | |
706 | copy_object->ref_count == 1) { | |
707 | /* | |
708 | * Gaah... start over! | |
709 | */ | |
710 | FREE_PAGE(copy_m); | |
711 | vm_object_unlock(copy_object); | |
712 | vm_object_deallocate(copy_object); | |
713 | /* may block */ | |
714 | vm_object_lock(object); | |
715 | goto RetryCopy; | |
716 | } | |
717 | vm_object_lock(object); | |
718 | ||
719 | if (page_exists) { | |
720 | /* | |
721 | * We didn't need the page | |
722 | */ | |
723 | FREE_PAGE(copy_m); | |
724 | } | |
725 | } | |
726 | } | |
727 | if (!page_exists) { | |
728 | /* | |
729 | * Must copy page into copy-object. | |
730 | */ | |
731 | vm_page_copy(m, copy_m); | |
732 | copy_m->fake = FALSE; | |
733 | copy_m->absent = FALSE; | |
734 | ||
735 | /* | |
736 | * Things to remember: | |
737 | * 1. The copied page must be marked 'dirty' | |
738 | * so it will be paged out to the copy | |
739 | * object. | |
740 | * 2. If the old page was in use by any users | |
741 | * of the copy-object, it must be removed | |
742 | * from all pmaps. (We can't know which | |
743 | * pmaps use it.) | |
744 | */ | |
745 | vm_page_lock_queues(); | |
746 | pmap_page_protect(VM_PAGE_TO_PHYS(old_m), | |
747 | VM_PROT_NONE); | |
748 | copy_m->clean = FALSE; | |
749 | vm_page_activate(copy_m); /* XXX */ | |
750 | vm_page_unlock_queues(); | |
751 | ||
752 | PAGE_WAKEUP(copy_m); | |
753 | } | |
754 | /* | |
755 | * The reference count on copy_object must be | |
756 | * at least 2: one for our extra reference, | |
757 | * and at least one from the outside world | |
758 | * (we checked that when we last locked | |
759 | * copy_object). | |
760 | */ | |
761 | copy_object->ref_count--; | |
762 | vm_object_unlock(copy_object); | |
763 | m->copy_on_write = FALSE; | |
764 | } | |
765 | } | |
766 | ||
767 | if (m->active || m->inactive) | |
768 | panic("vm_fault: active or inactive before retrying lookup"); | |
769 | ||
770 | /* | |
771 | * We must verify that the maps have not changed | |
772 | * since our last lookup. | |
773 | */ | |
774 | ||
775 | if (!lookup_still_valid) { | |
776 | vm_object_t retry_object; | |
777 | vm_offset_t retry_offset; | |
778 | vm_prot_t retry_prot; | |
779 | ||
780 | /* | |
781 | * Since map entries may be pageable, make sure we can | |
782 | * take a page fault on them. | |
783 | */ | |
784 | vm_object_unlock(object); | |
785 | ||
786 | /* | |
787 | * To avoid trying to write_lock the map while another | |
788 | * thread has it read_locked (in vm_map_pageable), we | |
789 | * do not try for write permission. If the page is | |
790 | * still writable, we will get write permission. If it | |
791 | * is not, or has been marked needs_copy, we enter the | |
792 | * mapping without write permission, and will merely | |
793 | * take another fault. | |
794 | */ | |
795 | result = vm_map_lookup(&map, vaddr, | |
796 | fault_type & ~VM_PROT_WRITE, &entry, | |
797 | &retry_object, &retry_offset, &retry_prot, | |
798 | &wired, &su); | |
799 | ||
800 | vm_object_lock(object); | |
801 | ||
802 | /* | |
803 | * If we don't need the page any longer, put it on the | |
804 | * active list (the easiest thing to do here). If no | |
805 | * one needs it, pageout will grab it eventually. | |
806 | */ | |
807 | ||
808 | if (result != KERN_SUCCESS) { | |
809 | RELEASE_PAGE(m); | |
810 | UNLOCK_AND_DEALLOCATE; | |
811 | return(result); | |
812 | } | |
813 | ||
814 | lookup_still_valid = TRUE; | |
815 | ||
816 | if ((retry_object != first_object) || | |
817 | (retry_offset != first_offset)) { | |
818 | RELEASE_PAGE(m); | |
819 | UNLOCK_AND_DEALLOCATE; | |
820 | goto RetryFault; | |
821 | } | |
822 | ||
823 | /* | |
824 | * Check whether the protection has changed or the object | |
825 | * has been copied while we left the map unlocked. | |
826 | * Changing from read to write permission is OK - we leave | |
827 | * the page write-protected, and catch the write fault. | |
828 | * Changing from write to read permission means that we | |
829 | * can't mark the page write-enabled after all. | |
830 | */ | |
831 | prot &= retry_prot; | |
832 | if (m->copy_on_write) | |
833 | prot &= ~VM_PROT_WRITE; | |
834 | } | |
835 | ||
836 | /* | |
837 | * (the various bits we're fiddling with here are locked by | |
838 | * the object's lock) | |
839 | */ | |
840 | ||
841 | /* XXX This distorts the meaning of the copy_on_write bit */ | |
842 | ||
843 | if (prot & VM_PROT_WRITE) | |
844 | m->copy_on_write = FALSE; | |
845 | ||
846 | /* | |
847 | * It's critically important that a wired-down page be faulted | |
848 | * only once in each map for which it is wired. | |
849 | */ | |
850 | ||
851 | if (m->active || m->inactive) | |
852 | panic("vm_fault: active or inactive before pmap_enter"); | |
853 | ||
854 | vm_object_unlock(object); | |
855 | ||
856 | /* | |
857 | * Put this page into the physical map. | |
858 | * We had to do the unlock above because pmap_enter | |
859 | * may cause other faults. We don't put the | |
860 | * page back on the active queue until later so | |
861 | * that the page-out daemon won't find us (yet). | |
862 | */ | |
863 | ||
864 | pmap_enter(map->pmap, vaddr, VM_PAGE_TO_PHYS(m), | |
865 | prot & ~(m->page_lock), wired); | |
866 | ||
867 | /* | |
868 | * If the page is not wired down, then put it where the | |
869 | * pageout daemon can find it. | |
870 | */ | |
871 | vm_object_lock(object); | |
872 | vm_page_lock_queues(); | |
873 | if (change_wiring) { | |
874 | if (wired) | |
875 | vm_page_wire(m); | |
876 | else | |
877 | vm_page_unwire(m); | |
878 | } | |
879 | else | |
880 | vm_page_activate(m); | |
881 | vm_page_unlock_queues(); | |
882 | ||
883 | /* | |
884 | * Unlock everything, and return | |
885 | */ | |
886 | ||
887 | PAGE_WAKEUP(m); | |
888 | UNLOCK_AND_DEALLOCATE; | |
889 | ||
890 | return(KERN_SUCCESS); | |
891 | ||
892 | } | |
893 | ||
894 | /* | |
895 | * vm_fault_wire: | |
896 | * | |
897 | * Wire down a range of virtual addresses in a map. | |
898 | */ | |
899 | void vm_fault_wire(map, start, end) | |
900 | vm_map_t map; | |
901 | vm_offset_t start, end; | |
902 | { | |
903 | ||
904 | register vm_offset_t va; | |
905 | register pmap_t pmap; | |
906 | ||
907 | pmap = vm_map_pmap(map); | |
908 | ||
909 | /* | |
910 | * Inform the physical mapping system that the | |
911 | * range of addresses may not fault, so that | |
912 | * page tables and such can be locked down as well. | |
913 | */ | |
914 | ||
915 | pmap_pageable(pmap, start, end, FALSE); | |
916 | ||
917 | /* | |
918 | * We simulate a fault to get the page and enter it | |
919 | * in the physical map. | |
920 | */ | |
921 | ||
922 | for (va = start; va < end; va += PAGE_SIZE) { | |
923 | (void) vm_fault(map, va, VM_PROT_NONE, TRUE); | |
924 | } | |
925 | } | |
926 | ||
927 | ||
928 | /* | |
929 | * vm_fault_unwire: | |
930 | * | |
931 | * Unwire a range of virtual addresses in a map. | |
932 | */ | |
933 | void vm_fault_unwire(map, start, end) | |
934 | vm_map_t map; | |
935 | vm_offset_t start, end; | |
936 | { | |
937 | ||
938 | register vm_offset_t va, pa; | |
939 | register pmap_t pmap; | |
940 | ||
941 | pmap = vm_map_pmap(map); | |
942 | ||
943 | /* | |
944 | * Since the pages are wired down, we must be able to | |
945 | * get their mappings from the physical map system. | |
946 | */ | |
947 | ||
948 | vm_page_lock_queues(); | |
949 | ||
950 | for (va = start; va < end; va += PAGE_SIZE) { | |
951 | pa = pmap_extract(pmap, va); | |
952 | if (pa == (vm_offset_t) 0) { | |
953 | panic("unwire: page not in pmap"); | |
954 | } | |
955 | pmap_change_wiring(pmap, va, FALSE); | |
956 | vm_page_unwire(PHYS_TO_VM_PAGE(pa)); | |
957 | } | |
958 | vm_page_unlock_queues(); | |
959 | ||
960 | /* | |
961 | * Inform the physical mapping system that the range | |
962 | * of addresses may fault, so that page tables and | |
963 | * such may be unwired themselves. | |
964 | */ | |
965 | ||
966 | pmap_pageable(pmap, start, end, TRUE); | |
967 | ||
968 | } | |
969 | ||
970 | /* | |
971 | * Routine: | |
972 | * vm_fault_copy_entry | |
973 | * Function: | |
974 | * Copy all of the pages from a wired-down map entry to another. | |
975 | * | |
976 | * In/out conditions: | |
977 | * The source and destination maps must be locked for write. | |
978 | * The source map entry must be wired down (or be a sharing map | |
979 | * entry corresponding to a main map entry that is wired down). | |
980 | */ | |
981 | ||
982 | void vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry) | |
983 | vm_map_t dst_map; | |
984 | vm_map_t src_map; | |
985 | vm_map_entry_t dst_entry; | |
986 | vm_map_entry_t src_entry; | |
987 | { | |
988 | ||
989 | vm_object_t dst_object; | |
990 | vm_object_t src_object; | |
991 | vm_offset_t dst_offset; | |
992 | vm_offset_t src_offset; | |
993 | vm_prot_t prot; | |
994 | vm_offset_t vaddr; | |
995 | vm_page_t dst_m; | |
996 | vm_page_t src_m; | |
997 | ||
998 | #ifdef lint | |
999 | src_map++; | |
1000 | #endif lint | |
1001 | ||
1002 | src_object = src_entry->object.vm_object; | |
1003 | src_offset = src_entry->offset; | |
1004 | ||
1005 | /* | |
1006 | * Create the top-level object for the destination entry. | |
1007 | * (Doesn't actually shadow anything - we copy the pages | |
1008 | * directly.) | |
1009 | */ | |
1010 | dst_object = vm_object_allocate( | |
1011 | (vm_size_t) (dst_entry->end - dst_entry->start)); | |
1012 | ||
1013 | dst_entry->object.vm_object = dst_object; | |
1014 | dst_entry->offset = 0; | |
1015 | ||
1016 | prot = dst_entry->max_protection; | |
1017 | ||
1018 | /* | |
1019 | * Loop through all of the pages in the entry's range, copying | |
1020 | * each one from the source object (it should be there) to the | |
1021 | * destination object. | |
1022 | */ | |
1023 | for (vaddr = dst_entry->start, dst_offset = 0; | |
1024 | vaddr < dst_entry->end; | |
1025 | vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) { | |
1026 | ||
1027 | /* | |
1028 | * Allocate a page in the destination object | |
1029 | */ | |
1030 | vm_object_lock(dst_object); | |
1031 | do { | |
1032 | dst_m = vm_page_alloc(dst_object, dst_offset); | |
1033 | if (dst_m == NULL) { | |
1034 | vm_object_unlock(dst_object); | |
1035 | VM_WAIT; | |
1036 | vm_object_lock(dst_object); | |
1037 | } | |
1038 | } while (dst_m == NULL); | |
1039 | ||
1040 | /* | |
1041 | * Find the page in the source object, and copy it in. | |
1042 | * (Because the source is wired down, the page will be | |
1043 | * in memory.) | |
1044 | */ | |
1045 | vm_object_lock(src_object); | |
1046 | src_m = vm_page_lookup(src_object, dst_offset + src_offset); | |
1047 | if (src_m == NULL) | |
1048 | panic("vm_fault_copy_wired: page missing"); | |
1049 | ||
1050 | vm_page_copy(src_m, dst_m); | |
1051 | ||
1052 | /* | |
1053 | * Enter it in the pmap... | |
1054 | */ | |
1055 | vm_object_unlock(src_object); | |
1056 | vm_object_unlock(dst_object); | |
1057 | ||
1058 | pmap_enter(dst_map->pmap, vaddr, VM_PAGE_TO_PHYS(dst_m), | |
1059 | prot, FALSE); | |
1060 | ||
1061 | /* | |
1062 | * Mark it no longer busy, and put it on the active list. | |
1063 | */ | |
1064 | vm_object_lock(dst_object); | |
1065 | vm_page_lock_queues(); | |
1066 | vm_page_activate(dst_m); | |
1067 | vm_page_unlock_queues(); | |
1068 | PAGE_WAKEUP(dst_m); | |
1069 | vm_object_unlock(dst_object); | |
1070 | } | |
1071 | ||
1072 | } |