Commit | Line | Data |
---|---|---|
175f072e | 1 | /* |
75a29a52 KM |
2 | * Copyright (c) 1991 Regents of the University of California. |
3 | * All rights reserved. | |
4 | * | |
5 | * This code is derived from software contributed to Berkeley by | |
6 | * The Mach Operating System project at Carnegie-Mellon University. | |
7 | * | |
8 | * %sccs.include.redist.c% | |
9 | * | |
34716da1 | 10 | * @(#)vm_fault.c 7.9 (Berkeley) %G% |
75a29a52 KM |
11 | * |
12 | * | |
13 | * Copyright (c) 1987, 1990 Carnegie-Mellon University. | |
175f072e KM |
14 | * All rights reserved. |
15 | * | |
6f0815d1 KM |
16 | * Authors: Avadis Tevanian, Jr., Michael Wayne Young |
17 | * | |
18 | * Permission to use, copy, modify and distribute this software and | |
19 | * its documentation is hereby granted, provided that both the copyright | |
20 | * notice and this permission notice appear in all copies of the | |
21 | * software, derivative works or modified versions, and any portions | |
22 | * thereof, and that both notices appear in supporting documentation. | |
23 | * | |
24 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" | |
25 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND | |
26 | * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. | |
27 | * | |
75a29a52 | 28 | * Carnegie Mellon requests users of this software to return to |
39d47b56 | 29 | * |
75a29a52 KM |
30 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
31 | * School of Computer Science | |
32 | * Carnegie Mellon University | |
33 | * Pittsburgh PA 15213-3890 | |
175f072e | 34 | * |
75a29a52 KM |
35 | * any improvements or extensions that they make and grant Carnegie the |
36 | * rights to redistribute these changes. | |
175f072e KM |
37 | */ |
38 | ||
39 | /* | |
40 | * Page fault handling module. | |
41 | */ | |
42 | ||
43 | #include "param.h" | |
ffe0d082 MK |
44 | |
45 | #include "vm.h" | |
46 | #include "vm_page.h" | |
47 | #include "vm_pageout.h" | |
175f072e KM |
48 | |
49 | /* | |
50 | * vm_fault: | |
51 | * | |
52 | * Handle a page fault occuring at the given address, | |
53 | * requiring the given permissions, in the map specified. | |
54 | * If successful, the page is inserted into the | |
55 | * associated physical map. | |
56 | * | |
57 | * NOTE: the given address should be truncated to the | |
58 | * proper page address. | |
59 | * | |
60 | * KERN_SUCCESS is returned if the page fault is handled; otherwise, | |
61 | * a standard error specifying why the fault is fatal is returned. | |
62 | * | |
63 | * | |
64 | * The map in question must be referenced, and remains so. | |
65 | * Caller may hold no locks. | |
66 | */ | |
67 | vm_fault(map, vaddr, fault_type, change_wiring) | |
68 | vm_map_t map; | |
69 | vm_offset_t vaddr; | |
70 | vm_prot_t fault_type; | |
71 | boolean_t change_wiring; | |
72 | { | |
73 | vm_object_t first_object; | |
74 | vm_offset_t first_offset; | |
75 | vm_map_entry_t entry; | |
76 | register vm_object_t object; | |
77 | register vm_offset_t offset; | |
78 | register vm_page_t m; | |
79 | vm_page_t first_m; | |
80 | vm_prot_t prot; | |
81 | int result; | |
82 | boolean_t wired; | |
83 | boolean_t su; | |
84 | boolean_t lookup_still_valid; | |
85 | boolean_t page_exists; | |
86 | vm_page_t old_m; | |
87 | vm_object_t next_object; | |
88 | ||
f92ce83a | 89 | cnt.v_vm_faults++; /* needs lock XXX */ |
175f072e KM |
90 | /* |
91 | * Recovery actions | |
92 | */ | |
93 | #define FREE_PAGE(m) { \ | |
94 | PAGE_WAKEUP(m); \ | |
95 | vm_page_lock_queues(); \ | |
96 | vm_page_free(m); \ | |
97 | vm_page_unlock_queues(); \ | |
98 | } | |
99 | ||
100 | #define RELEASE_PAGE(m) { \ | |
101 | PAGE_WAKEUP(m); \ | |
102 | vm_page_lock_queues(); \ | |
103 | vm_page_activate(m); \ | |
104 | vm_page_unlock_queues(); \ | |
105 | } | |
106 | ||
107 | #define UNLOCK_MAP { \ | |
108 | if (lookup_still_valid) { \ | |
109 | vm_map_lookup_done(map, entry); \ | |
110 | lookup_still_valid = FALSE; \ | |
111 | } \ | |
112 | } | |
113 | ||
114 | #define UNLOCK_THINGS { \ | |
115 | object->paging_in_progress--; \ | |
116 | vm_object_unlock(object); \ | |
117 | if (object != first_object) { \ | |
118 | vm_object_lock(first_object); \ | |
119 | FREE_PAGE(first_m); \ | |
120 | first_object->paging_in_progress--; \ | |
121 | vm_object_unlock(first_object); \ | |
122 | } \ | |
123 | UNLOCK_MAP; \ | |
124 | } | |
125 | ||
126 | #define UNLOCK_AND_DEALLOCATE { \ | |
127 | UNLOCK_THINGS; \ | |
128 | vm_object_deallocate(first_object); \ | |
129 | } | |
130 | ||
131 | RetryFault: ; | |
132 | ||
133 | /* | |
134 | * Find the backing store object and offset into | |
135 | * it to begin the search. | |
136 | */ | |
137 | ||
138 | if ((result = vm_map_lookup(&map, vaddr, fault_type, &entry, | |
139 | &first_object, &first_offset, | |
140 | &prot, &wired, &su)) != KERN_SUCCESS) { | |
141 | return(result); | |
142 | } | |
143 | lookup_still_valid = TRUE; | |
144 | ||
145 | if (wired) | |
146 | fault_type = prot; | |
147 | ||
ffe0d082 | 148 | first_m = NULL; |
175f072e KM |
149 | |
150 | /* | |
151 | * Make a reference to this object to | |
152 | * prevent its disposal while we are messing with | |
153 | * it. Once we have the reference, the map is free | |
154 | * to be diddled. Since objects reference their | |
155 | * shadows (and copies), they will stay around as well. | |
156 | */ | |
157 | ||
158 | vm_object_lock(first_object); | |
159 | ||
160 | first_object->ref_count++; | |
161 | first_object->paging_in_progress++; | |
162 | ||
163 | /* | |
164 | * INVARIANTS (through entire routine): | |
165 | * | |
166 | * 1) At all times, we must either have the object | |
167 | * lock or a busy page in some object to prevent | |
168 | * some other thread from trying to bring in | |
169 | * the same page. | |
170 | * | |
171 | * Note that we cannot hold any locks during the | |
172 | * pager access or when waiting for memory, so | |
173 | * we use a busy page then. | |
174 | * | |
175 | * Note also that we aren't as concerned about | |
176 | * more than one thead attempting to pager_data_unlock | |
177 | * the same page at once, so we don't hold the page | |
178 | * as busy then, but do record the highest unlock | |
179 | * value so far. [Unlock requests may also be delivered | |
180 | * out of order.] | |
181 | * | |
182 | * 2) Once we have a busy page, we must remove it from | |
183 | * the pageout queues, so that the pageout daemon | |
184 | * will not grab it away. | |
185 | * | |
186 | * 3) To prevent another thread from racing us down the | |
187 | * shadow chain and entering a new page in the top | |
188 | * object before we do, we must keep a busy page in | |
189 | * the top object while following the shadow chain. | |
190 | * | |
191 | * 4) We must increment paging_in_progress on any object | |
192 | * for which we have a busy page, to prevent | |
193 | * vm_object_collapse from removing the busy page | |
194 | * without our noticing. | |
195 | */ | |
196 | ||
197 | /* | |
198 | * Search for the page at object/offset. | |
199 | */ | |
200 | ||
201 | object = first_object; | |
202 | offset = first_offset; | |
203 | ||
204 | /* | |
205 | * See whether this page is resident | |
206 | */ | |
207 | ||
208 | while (TRUE) { | |
209 | m = vm_page_lookup(object, offset); | |
ffe0d082 | 210 | if (m != NULL) { |
175f072e KM |
211 | /* |
212 | * If the page is being brought in, | |
213 | * wait for it and then retry. | |
214 | */ | |
215 | if (m->busy) { | |
216 | #ifdef DOTHREADS | |
217 | int wait_result; | |
218 | ||
219 | PAGE_ASSERT_WAIT(m, !change_wiring); | |
220 | UNLOCK_THINGS; | |
221 | thread_block(); | |
222 | wait_result = current_thread()->wait_result; | |
223 | vm_object_deallocate(first_object); | |
224 | if (wait_result != THREAD_AWAKENED) | |
225 | return(KERN_SUCCESS); | |
226 | goto RetryFault; | |
227 | #else | |
228 | PAGE_ASSERT_WAIT(m, !change_wiring); | |
229 | UNLOCK_THINGS; | |
230 | thread_block(); | |
231 | vm_object_deallocate(first_object); | |
232 | goto RetryFault; | |
233 | #endif | |
234 | } | |
235 | ||
236 | if (m->absent) | |
237 | panic("vm_fault: absent"); | |
238 | ||
239 | /* | |
240 | * If the desired access to this page has | |
241 | * been locked out, request that it be unlocked. | |
242 | */ | |
243 | ||
244 | if (fault_type & m->page_lock) { | |
245 | #ifdef DOTHREADS | |
246 | int wait_result; | |
247 | ||
248 | if ((fault_type & m->unlock_request) != fault_type) | |
249 | panic("vm_fault: pager_data_unlock"); | |
250 | ||
251 | PAGE_ASSERT_WAIT(m, !change_wiring); | |
252 | UNLOCK_THINGS; | |
253 | thread_block(); | |
254 | wait_result = current_thread()->wait_result; | |
255 | vm_object_deallocate(first_object); | |
256 | if (wait_result != THREAD_AWAKENED) | |
257 | return(KERN_SUCCESS); | |
258 | goto RetryFault; | |
259 | #else | |
260 | if ((fault_type & m->unlock_request) != fault_type) | |
261 | panic("vm_fault: pager_data_unlock"); | |
262 | ||
263 | PAGE_ASSERT_WAIT(m, !change_wiring); | |
264 | UNLOCK_THINGS; | |
265 | thread_block(); | |
266 | vm_object_deallocate(first_object); | |
267 | goto RetryFault; | |
268 | #endif | |
269 | } | |
270 | ||
271 | /* | |
272 | * Remove the page from the pageout daemon's | |
273 | * reach while we play with it. | |
274 | */ | |
275 | ||
276 | vm_page_lock_queues(); | |
277 | if (m->inactive) { | |
278 | queue_remove(&vm_page_queue_inactive, m, | |
279 | vm_page_t, pageq); | |
280 | m->inactive = FALSE; | |
f92ce83a KM |
281 | cnt.v_inactive_count--; |
282 | cnt.v_reactivated++; | |
175f072e KM |
283 | } |
284 | ||
285 | if (m->active) { | |
286 | queue_remove(&vm_page_queue_active, m, | |
287 | vm_page_t, pageq); | |
288 | m->active = FALSE; | |
f92ce83a | 289 | cnt.v_active_count--; |
175f072e KM |
290 | } |
291 | vm_page_unlock_queues(); | |
292 | ||
293 | /* | |
294 | * Mark page busy for other threads. | |
295 | */ | |
296 | m->busy = TRUE; | |
297 | m->absent = FALSE; | |
298 | break; | |
299 | } | |
300 | ||
ffe0d082 | 301 | if (((object->pager != NULL) && |
175f072e KM |
302 | (!change_wiring || wired)) |
303 | || (object == first_object)) { | |
304 | ||
305 | /* | |
306 | * Allocate a new page for this object/offset | |
307 | * pair. | |
308 | */ | |
309 | ||
310 | m = vm_page_alloc(object, offset); | |
311 | ||
ffe0d082 | 312 | if (m == NULL) { |
175f072e KM |
313 | UNLOCK_AND_DEALLOCATE; |
314 | VM_WAIT; | |
315 | goto RetryFault; | |
316 | } | |
317 | } | |
318 | ||
ffe0d082 | 319 | if ((object->pager != NULL) && |
175f072e KM |
320 | (!change_wiring || wired)) { |
321 | int rv; | |
322 | ||
323 | /* | |
324 | * Now that we have a busy page, we can | |
325 | * release the object lock. | |
326 | */ | |
327 | vm_object_unlock(object); | |
328 | ||
329 | /* | |
330 | * Call the pager to retrieve the data, if any, | |
331 | * after releasing the lock on the map. | |
332 | */ | |
333 | UNLOCK_MAP; | |
334 | ||
335 | rv = vm_pager_get(object->pager, m, TRUE); | |
336 | if (rv == VM_PAGER_OK) { | |
337 | /* | |
338 | * Found the page. | |
339 | * Leave it busy while we play with it. | |
340 | */ | |
341 | vm_object_lock(object); | |
342 | ||
343 | /* | |
344 | * Relookup in case pager changed page. | |
345 | * Pager is responsible for disposition | |
346 | * of old page if moved. | |
347 | */ | |
348 | m = vm_page_lookup(object, offset); | |
349 | ||
f92ce83a | 350 | cnt.v_pageins++; |
175f072e KM |
351 | m->fake = FALSE; |
352 | pmap_clear_modify(VM_PAGE_TO_PHYS(m)); | |
353 | break; | |
354 | } | |
355 | ||
356 | /* | |
357 | * Remove the bogus page (which does not | |
358 | * exist at this object/offset); before | |
359 | * doing so, we must get back our object | |
360 | * lock to preserve our invariant. | |
361 | * | |
362 | * Also wake up any other thread that may want | |
363 | * to bring in this page. | |
364 | * | |
365 | * If this is the top-level object, we must | |
366 | * leave the busy page to prevent another | |
367 | * thread from rushing past us, and inserting | |
368 | * the page in that object at the same time | |
369 | * that we are. | |
370 | */ | |
371 | ||
372 | vm_object_lock(object); | |
373 | /* | |
374 | * Data outside the range of the pager; an error | |
375 | */ | |
376 | if (rv == VM_PAGER_BAD) { | |
377 | FREE_PAGE(m); | |
378 | UNLOCK_AND_DEALLOCATE; | |
379 | return(KERN_PROTECTION_FAILURE); /* XXX */ | |
380 | } | |
381 | if (object != first_object) { | |
382 | FREE_PAGE(m); | |
383 | /* | |
384 | * XXX - we cannot just fall out at this | |
385 | * point, m has been freed and is invalid! | |
386 | */ | |
387 | } | |
388 | } | |
389 | ||
390 | /* | |
391 | * We get here if the object has no pager (or unwiring) | |
392 | * or the pager doesn't have the page. | |
393 | */ | |
394 | if (object == first_object) | |
395 | first_m = m; | |
396 | ||
397 | /* | |
398 | * Move on to the next object. Lock the next | |
399 | * object before unlocking the current one. | |
400 | */ | |
401 | ||
402 | offset += object->shadow_offset; | |
403 | next_object = object->shadow; | |
ffe0d082 | 404 | if (next_object == NULL) { |
175f072e KM |
405 | /* |
406 | * If there's no object left, fill the page | |
407 | * in the top object with zeros. | |
408 | */ | |
409 | if (object != first_object) { | |
410 | object->paging_in_progress--; | |
411 | vm_object_unlock(object); | |
412 | ||
413 | object = first_object; | |
414 | offset = first_offset; | |
415 | m = first_m; | |
416 | vm_object_lock(object); | |
417 | } | |
ffe0d082 | 418 | first_m = NULL; |
175f072e KM |
419 | |
420 | vm_page_zero_fill(m); | |
f92ce83a | 421 | cnt.v_zfod++; |
175f072e KM |
422 | m->fake = FALSE; |
423 | m->absent = FALSE; | |
424 | break; | |
425 | } | |
426 | else { | |
427 | vm_object_lock(next_object); | |
428 | if (object != first_object) | |
429 | object->paging_in_progress--; | |
430 | vm_object_unlock(object); | |
431 | object = next_object; | |
432 | object->paging_in_progress++; | |
433 | } | |
434 | } | |
435 | ||
436 | if (m->absent || m->active || m->inactive || !m->busy) | |
437 | panic("vm_fault: absent or active or inactive or not busy after main loop"); | |
438 | ||
439 | /* | |
440 | * PAGE HAS BEEN FOUND. | |
441 | * [Loop invariant still holds -- the object lock | |
442 | * is held.] | |
443 | */ | |
444 | ||
445 | old_m = m; /* save page that would be copied */ | |
446 | ||
447 | /* | |
448 | * If the page is being written, but isn't | |
449 | * already owned by the top-level object, | |
450 | * we have to copy it into a new page owned | |
451 | * by the top-level object. | |
452 | */ | |
453 | ||
454 | if (object != first_object) { | |
455 | /* | |
456 | * We only really need to copy if we | |
457 | * want to write it. | |
458 | */ | |
459 | ||
460 | if (fault_type & VM_PROT_WRITE) { | |
461 | ||
462 | /* | |
463 | * If we try to collapse first_object at this | |
464 | * point, we may deadlock when we try to get | |
465 | * the lock on an intermediate object (since we | |
466 | * have the bottom object locked). We can't | |
467 | * unlock the bottom object, because the page | |
468 | * we found may move (by collapse) if we do. | |
469 | * | |
470 | * Instead, we first copy the page. Then, when | |
471 | * we have no more use for the bottom object, | |
472 | * we unlock it and try to collapse. | |
473 | * | |
474 | * Note that we copy the page even if we didn't | |
475 | * need to... that's the breaks. | |
476 | */ | |
477 | ||
478 | /* | |
479 | * We already have an empty page in | |
480 | * first_object - use it. | |
481 | */ | |
482 | ||
483 | vm_page_copy(m, first_m); | |
484 | first_m->fake = FALSE; | |
485 | first_m->absent = FALSE; | |
486 | ||
487 | /* | |
488 | * If another map is truly sharing this | |
489 | * page with us, we have to flush all | |
490 | * uses of the original page, since we | |
491 | * can't distinguish those which want the | |
492 | * original from those which need the | |
493 | * new copy. | |
29261da1 MH |
494 | * |
495 | * XXX If we know that only one map has | |
496 | * access to this page, then we could | |
497 | * avoid the pmap_page_protect() call. | |
175f072e KM |
498 | */ |
499 | ||
500 | vm_page_lock_queues(); | |
34716da1 | 501 | vm_page_activate(m); |
29261da1 | 502 | pmap_page_protect(VM_PAGE_TO_PHYS(m), VM_PROT_NONE); |
175f072e KM |
503 | vm_page_unlock_queues(); |
504 | ||
505 | /* | |
506 | * We no longer need the old page or object. | |
507 | */ | |
508 | PAGE_WAKEUP(m); | |
509 | object->paging_in_progress--; | |
510 | vm_object_unlock(object); | |
511 | ||
512 | /* | |
513 | * Only use the new page below... | |
514 | */ | |
515 | ||
f92ce83a | 516 | cnt.v_cow_faults++; |
175f072e KM |
517 | m = first_m; |
518 | object = first_object; | |
519 | offset = first_offset; | |
520 | ||
521 | /* | |
522 | * Now that we've gotten the copy out of the | |
523 | * way, let's try to collapse the top object. | |
524 | */ | |
525 | vm_object_lock(object); | |
526 | /* | |
527 | * But we have to play ugly games with | |
528 | * paging_in_progress to do that... | |
529 | */ | |
530 | object->paging_in_progress--; | |
531 | vm_object_collapse(object); | |
532 | object->paging_in_progress++; | |
533 | } | |
534 | else { | |
535 | prot &= (~VM_PROT_WRITE); | |
536 | m->copy_on_write = TRUE; | |
537 | } | |
538 | } | |
539 | ||
540 | if (m->active || m->inactive) | |
541 | panic("vm_fault: active or inactive before copy object handling"); | |
542 | ||
543 | /* | |
544 | * If the page is being written, but hasn't been | |
545 | * copied to the copy-object, we have to copy it there. | |
546 | */ | |
547 | RetryCopy: | |
ffe0d082 MK |
548 | if (first_object->copy != NULL) { |
549 | vm_object_t copy_object = first_object->copy; | |
550 | vm_offset_t copy_offset; | |
551 | vm_page_t copy_m; | |
175f072e KM |
552 | |
553 | /* | |
554 | * We only need to copy if we want to write it. | |
555 | */ | |
556 | if ((fault_type & VM_PROT_WRITE) == 0) { | |
557 | prot &= ~VM_PROT_WRITE; | |
558 | m->copy_on_write = TRUE; | |
559 | } | |
560 | else { | |
561 | /* | |
562 | * Try to get the lock on the copy_object. | |
563 | */ | |
564 | if (!vm_object_lock_try(copy_object)) { | |
565 | vm_object_unlock(object); | |
566 | /* should spin a bit here... */ | |
567 | vm_object_lock(object); | |
568 | goto RetryCopy; | |
569 | } | |
570 | ||
571 | /* | |
572 | * Make another reference to the copy-object, | |
573 | * to keep it from disappearing during the | |
574 | * copy. | |
575 | */ | |
576 | copy_object->ref_count++; | |
577 | ||
578 | /* | |
579 | * Does the page exist in the copy? | |
580 | */ | |
581 | copy_offset = first_offset | |
582 | - copy_object->shadow_offset; | |
583 | copy_m = vm_page_lookup(copy_object, copy_offset); | |
ffe0d082 | 584 | if (page_exists = (copy_m != NULL)) { |
175f072e KM |
585 | if (copy_m->busy) { |
586 | #ifdef DOTHREADS | |
587 | int wait_result; | |
588 | ||
589 | /* | |
590 | * If the page is being brought | |
591 | * in, wait for it and then retry. | |
592 | */ | |
593 | PAGE_ASSERT_WAIT(copy_m, !change_wiring); | |
594 | RELEASE_PAGE(m); | |
595 | copy_object->ref_count--; | |
596 | vm_object_unlock(copy_object); | |
597 | UNLOCK_THINGS; | |
598 | thread_block(); | |
599 | wait_result = current_thread()->wait_result; | |
600 | vm_object_deallocate(first_object); | |
601 | if (wait_result != THREAD_AWAKENED) | |
602 | return(KERN_SUCCESS); | |
603 | goto RetryFault; | |
604 | #else | |
605 | /* | |
606 | * If the page is being brought | |
607 | * in, wait for it and then retry. | |
608 | */ | |
609 | PAGE_ASSERT_WAIT(copy_m, !change_wiring); | |
610 | RELEASE_PAGE(m); | |
611 | copy_object->ref_count--; | |
612 | vm_object_unlock(copy_object); | |
613 | UNLOCK_THINGS; | |
614 | thread_block(); | |
615 | vm_object_deallocate(first_object); | |
616 | goto RetryFault; | |
617 | #endif | |
618 | } | |
619 | } | |
620 | ||
621 | /* | |
622 | * If the page is not in memory (in the object) | |
623 | * and the object has a pager, we have to check | |
624 | * if the pager has the data in secondary | |
625 | * storage. | |
626 | */ | |
627 | if (!page_exists) { | |
628 | ||
629 | /* | |
630 | * If we don't allocate a (blank) page | |
631 | * here... another thread could try | |
632 | * to page it in, allocate a page, and | |
633 | * then block on the busy page in its | |
634 | * shadow (first_object). Then we'd | |
635 | * trip over the busy page after we | |
636 | * found that the copy_object's pager | |
637 | * doesn't have the page... | |
638 | */ | |
639 | copy_m = vm_page_alloc(copy_object, | |
640 | copy_offset); | |
ffe0d082 | 641 | if (copy_m == NULL) { |
175f072e KM |
642 | /* |
643 | * Wait for a page, then retry. | |
644 | */ | |
645 | RELEASE_PAGE(m); | |
646 | copy_object->ref_count--; | |
647 | vm_object_unlock(copy_object); | |
648 | UNLOCK_AND_DEALLOCATE; | |
649 | VM_WAIT; | |
650 | goto RetryFault; | |
651 | } | |
652 | ||
ffe0d082 | 653 | if (copy_object->pager != NULL) { |
175f072e KM |
654 | vm_object_unlock(object); |
655 | vm_object_unlock(copy_object); | |
656 | UNLOCK_MAP; | |
657 | ||
658 | page_exists = vm_pager_has_page( | |
659 | copy_object->pager, | |
660 | (copy_offset + copy_object->paging_offset)); | |
661 | ||
662 | vm_object_lock(copy_object); | |
663 | ||
664 | /* | |
665 | * Since the map is unlocked, someone | |
666 | * else could have copied this object | |
667 | * and put a different copy_object | |
668 | * between the two. Or, the last | |
669 | * reference to the copy-object (other | |
670 | * than the one we have) may have | |
671 | * disappeared - if that has happened, | |
672 | * we don't need to make the copy. | |
673 | */ | |
674 | if (copy_object->shadow != object || | |
675 | copy_object->ref_count == 1) { | |
676 | /* | |
677 | * Gaah... start over! | |
678 | */ | |
679 | FREE_PAGE(copy_m); | |
680 | vm_object_unlock(copy_object); | |
681 | vm_object_deallocate(copy_object); | |
682 | /* may block */ | |
683 | vm_object_lock(object); | |
684 | goto RetryCopy; | |
685 | } | |
686 | vm_object_lock(object); | |
687 | ||
688 | if (page_exists) { | |
689 | /* | |
690 | * We didn't need the page | |
691 | */ | |
692 | FREE_PAGE(copy_m); | |
693 | } | |
694 | } | |
695 | } | |
696 | if (!page_exists) { | |
697 | /* | |
698 | * Must copy page into copy-object. | |
699 | */ | |
700 | vm_page_copy(m, copy_m); | |
701 | copy_m->fake = FALSE; | |
702 | copy_m->absent = FALSE; | |
703 | ||
704 | /* | |
705 | * Things to remember: | |
706 | * 1. The copied page must be marked 'dirty' | |
707 | * so it will be paged out to the copy | |
708 | * object. | |
709 | * 2. If the old page was in use by any users | |
710 | * of the copy-object, it must be removed | |
711 | * from all pmaps. (We can't know which | |
712 | * pmaps use it.) | |
713 | */ | |
714 | vm_page_lock_queues(); | |
29261da1 MH |
715 | pmap_page_protect(VM_PAGE_TO_PHYS(old_m), |
716 | VM_PROT_NONE); | |
175f072e KM |
717 | copy_m->clean = FALSE; |
718 | vm_page_activate(copy_m); /* XXX */ | |
719 | vm_page_unlock_queues(); | |
720 | ||
721 | PAGE_WAKEUP(copy_m); | |
722 | } | |
723 | /* | |
724 | * The reference count on copy_object must be | |
725 | * at least 2: one for our extra reference, | |
726 | * and at least one from the outside world | |
727 | * (we checked that when we last locked | |
728 | * copy_object). | |
729 | */ | |
730 | copy_object->ref_count--; | |
731 | vm_object_unlock(copy_object); | |
732 | m->copy_on_write = FALSE; | |
733 | } | |
734 | } | |
735 | ||
736 | if (m->active || m->inactive) | |
737 | panic("vm_fault: active or inactive before retrying lookup"); | |
738 | ||
739 | /* | |
740 | * We must verify that the maps have not changed | |
741 | * since our last lookup. | |
742 | */ | |
743 | ||
744 | if (!lookup_still_valid) { | |
745 | vm_object_t retry_object; | |
746 | vm_offset_t retry_offset; | |
747 | vm_prot_t retry_prot; | |
748 | ||
749 | /* | |
750 | * Since map entries may be pageable, make sure we can | |
751 | * take a page fault on them. | |
752 | */ | |
753 | vm_object_unlock(object); | |
754 | ||
755 | /* | |
756 | * To avoid trying to write_lock the map while another | |
757 | * thread has it read_locked (in vm_map_pageable), we | |
758 | * do not try for write permission. If the page is | |
759 | * still writable, we will get write permission. If it | |
760 | * is not, or has been marked needs_copy, we enter the | |
761 | * mapping without write permission, and will merely | |
762 | * take another fault. | |
763 | */ | |
764 | result = vm_map_lookup(&map, vaddr, | |
765 | fault_type & ~VM_PROT_WRITE, &entry, | |
766 | &retry_object, &retry_offset, &retry_prot, | |
767 | &wired, &su); | |
768 | ||
769 | vm_object_lock(object); | |
770 | ||
771 | /* | |
772 | * If we don't need the page any longer, put it on the | |
773 | * active list (the easiest thing to do here). If no | |
774 | * one needs it, pageout will grab it eventually. | |
775 | */ | |
776 | ||
777 | if (result != KERN_SUCCESS) { | |
778 | RELEASE_PAGE(m); | |
779 | UNLOCK_AND_DEALLOCATE; | |
780 | return(result); | |
781 | } | |
782 | ||
783 | lookup_still_valid = TRUE; | |
784 | ||
785 | if ((retry_object != first_object) || | |
786 | (retry_offset != first_offset)) { | |
787 | RELEASE_PAGE(m); | |
788 | UNLOCK_AND_DEALLOCATE; | |
789 | goto RetryFault; | |
790 | } | |
791 | ||
792 | /* | |
793 | * Check whether the protection has changed or the object | |
794 | * has been copied while we left the map unlocked. | |
795 | * Changing from read to write permission is OK - we leave | |
796 | * the page write-protected, and catch the write fault. | |
797 | * Changing from write to read permission means that we | |
798 | * can't mark the page write-enabled after all. | |
799 | */ | |
800 | prot &= retry_prot; | |
801 | if (m->copy_on_write) | |
802 | prot &= ~VM_PROT_WRITE; | |
803 | } | |
804 | ||
805 | /* | |
806 | * (the various bits we're fiddling with here are locked by | |
807 | * the object's lock) | |
808 | */ | |
809 | ||
810 | /* XXX This distorts the meaning of the copy_on_write bit */ | |
811 | ||
812 | if (prot & VM_PROT_WRITE) | |
813 | m->copy_on_write = FALSE; | |
814 | ||
815 | /* | |
816 | * It's critically important that a wired-down page be faulted | |
817 | * only once in each map for which it is wired. | |
818 | */ | |
819 | ||
820 | if (m->active || m->inactive) | |
821 | panic("vm_fault: active or inactive before pmap_enter"); | |
822 | ||
823 | vm_object_unlock(object); | |
824 | ||
825 | /* | |
826 | * Put this page into the physical map. | |
827 | * We had to do the unlock above because pmap_enter | |
828 | * may cause other faults. We don't put the | |
829 | * page back on the active queue until later so | |
830 | * that the page-out daemon won't find us (yet). | |
831 | */ | |
832 | ||
833 | pmap_enter(map->pmap, vaddr, VM_PAGE_TO_PHYS(m), | |
834 | prot & ~(m->page_lock), wired); | |
835 | ||
836 | /* | |
837 | * If the page is not wired down, then put it where the | |
838 | * pageout daemon can find it. | |
839 | */ | |
840 | vm_object_lock(object); | |
841 | vm_page_lock_queues(); | |
842 | if (change_wiring) { | |
843 | if (wired) | |
844 | vm_page_wire(m); | |
845 | else | |
846 | vm_page_unwire(m); | |
847 | } | |
848 | else | |
849 | vm_page_activate(m); | |
850 | vm_page_unlock_queues(); | |
851 | ||
852 | /* | |
853 | * Unlock everything, and return | |
854 | */ | |
855 | ||
856 | PAGE_WAKEUP(m); | |
857 | UNLOCK_AND_DEALLOCATE; | |
858 | ||
859 | return(KERN_SUCCESS); | |
860 | ||
861 | } | |
862 | ||
863 | /* | |
864 | * vm_fault_wire: | |
865 | * | |
866 | * Wire down a range of virtual addresses in a map. | |
867 | */ | |
868 | void vm_fault_wire(map, start, end) | |
869 | vm_map_t map; | |
870 | vm_offset_t start, end; | |
871 | { | |
872 | ||
873 | register vm_offset_t va; | |
874 | register pmap_t pmap; | |
875 | ||
876 | pmap = vm_map_pmap(map); | |
877 | ||
878 | /* | |
879 | * Inform the physical mapping system that the | |
880 | * range of addresses may not fault, so that | |
881 | * page tables and such can be locked down as well. | |
882 | */ | |
883 | ||
884 | pmap_pageable(pmap, start, end, FALSE); | |
885 | ||
886 | /* | |
887 | * We simulate a fault to get the page and enter it | |
888 | * in the physical map. | |
889 | */ | |
890 | ||
891 | for (va = start; va < end; va += PAGE_SIZE) { | |
892 | (void) vm_fault(map, va, VM_PROT_NONE, TRUE); | |
893 | } | |
894 | } | |
895 | ||
896 | ||
897 | /* | |
898 | * vm_fault_unwire: | |
899 | * | |
900 | * Unwire a range of virtual addresses in a map. | |
901 | */ | |
902 | void vm_fault_unwire(map, start, end) | |
903 | vm_map_t map; | |
904 | vm_offset_t start, end; | |
905 | { | |
906 | ||
907 | register vm_offset_t va, pa; | |
908 | register pmap_t pmap; | |
909 | ||
910 | pmap = vm_map_pmap(map); | |
911 | ||
912 | /* | |
913 | * Since the pages are wired down, we must be able to | |
914 | * get their mappings from the physical map system. | |
915 | */ | |
916 | ||
917 | vm_page_lock_queues(); | |
918 | ||
919 | for (va = start; va < end; va += PAGE_SIZE) { | |
920 | pa = pmap_extract(pmap, va); | |
921 | if (pa == (vm_offset_t) 0) { | |
922 | panic("unwire: page not in pmap"); | |
923 | } | |
924 | pmap_change_wiring(pmap, va, FALSE); | |
925 | vm_page_unwire(PHYS_TO_VM_PAGE(pa)); | |
926 | } | |
927 | vm_page_unlock_queues(); | |
928 | ||
929 | /* | |
930 | * Inform the physical mapping system that the range | |
931 | * of addresses may fault, so that page tables and | |
932 | * such may be unwired themselves. | |
933 | */ | |
934 | ||
935 | pmap_pageable(pmap, start, end, TRUE); | |
936 | ||
937 | } | |
938 | ||
939 | /* | |
940 | * Routine: | |
941 | * vm_fault_copy_entry | |
942 | * Function: | |
943 | * Copy all of the pages from a wired-down map entry to another. | |
944 | * | |
945 | * In/out conditions: | |
946 | * The source and destination maps must be locked for write. | |
947 | * The source map entry must be wired down (or be a sharing map | |
948 | * entry corresponding to a main map entry that is wired down). | |
949 | */ | |
950 | ||
951 | void vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry) | |
952 | vm_map_t dst_map; | |
953 | vm_map_t src_map; | |
954 | vm_map_entry_t dst_entry; | |
955 | vm_map_entry_t src_entry; | |
956 | { | |
957 | ||
958 | vm_object_t dst_object; | |
959 | vm_object_t src_object; | |
960 | vm_offset_t dst_offset; | |
961 | vm_offset_t src_offset; | |
962 | vm_prot_t prot; | |
963 | vm_offset_t vaddr; | |
964 | vm_page_t dst_m; | |
965 | vm_page_t src_m; | |
966 | ||
967 | #ifdef lint | |
968 | src_map++; | |
969 | #endif lint | |
970 | ||
971 | src_object = src_entry->object.vm_object; | |
972 | src_offset = src_entry->offset; | |
973 | ||
974 | /* | |
975 | * Create the top-level object for the destination entry. | |
976 | * (Doesn't actually shadow anything - we copy the pages | |
977 | * directly.) | |
978 | */ | |
979 | dst_object = vm_object_allocate( | |
980 | (vm_size_t) (dst_entry->end - dst_entry->start)); | |
981 | ||
982 | dst_entry->object.vm_object = dst_object; | |
983 | dst_entry->offset = 0; | |
984 | ||
985 | prot = dst_entry->max_protection; | |
986 | ||
987 | /* | |
988 | * Loop through all of the pages in the entry's range, copying | |
989 | * each one from the source object (it should be there) to the | |
990 | * destination object. | |
991 | */ | |
992 | for (vaddr = dst_entry->start, dst_offset = 0; | |
993 | vaddr < dst_entry->end; | |
994 | vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) { | |
995 | ||
996 | /* | |
997 | * Allocate a page in the destination object | |
998 | */ | |
999 | vm_object_lock(dst_object); | |
1000 | do { | |
1001 | dst_m = vm_page_alloc(dst_object, dst_offset); | |
ffe0d082 | 1002 | if (dst_m == NULL) { |
175f072e KM |
1003 | vm_object_unlock(dst_object); |
1004 | VM_WAIT; | |
1005 | vm_object_lock(dst_object); | |
1006 | } | |
ffe0d082 | 1007 | } while (dst_m == NULL); |
175f072e KM |
1008 | |
1009 | /* | |
1010 | * Find the page in the source object, and copy it in. | |
1011 | * (Because the source is wired down, the page will be | |
1012 | * in memory.) | |
1013 | */ | |
1014 | vm_object_lock(src_object); | |
1015 | src_m = vm_page_lookup(src_object, dst_offset + src_offset); | |
ffe0d082 | 1016 | if (src_m == NULL) |
175f072e KM |
1017 | panic("vm_fault_copy_wired: page missing"); |
1018 | ||
1019 | vm_page_copy(src_m, dst_m); | |
1020 | ||
1021 | /* | |
1022 | * Enter it in the pmap... | |
1023 | */ | |
1024 | vm_object_unlock(src_object); | |
1025 | vm_object_unlock(dst_object); | |
1026 | ||
1027 | pmap_enter(dst_map->pmap, vaddr, VM_PAGE_TO_PHYS(dst_m), | |
1028 | prot, FALSE); | |
1029 | ||
1030 | /* | |
1031 | * Mark it no longer busy, and put it on the active list. | |
1032 | */ | |
1033 | vm_object_lock(dst_object); | |
1034 | vm_page_lock_queues(); | |
1035 | vm_page_activate(dst_m); | |
1036 | vm_page_unlock_queues(); | |
1037 | PAGE_WAKEUP(dst_m); | |
1038 | vm_object_unlock(dst_object); | |
1039 | } | |
1040 | ||
1041 | } |