* Copyright (c) 1990 University of Utah.
* Copyright (c) 1991 The Regents of the University of California.
* This code is derived from software contributed to Berkeley by
* the Systems Programming Group of the University of Utah Computer
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the University of
* California, Berkeley and its contributors.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
* from: @(#)swap_pager.c 7.4 (Berkeley) 5/7/91
* $Id: swap_pager.c,v 1.18 1994/03/02 02:53:27 davidg Exp $
* Mostly rewritten by John Dyson with help from David Greenman, 12-Jan-1994
extern int vm_pageout_rate_limit
;
extern vm_map_t pager_map
;
extern int vm_pageout_pages_needed
;
} swcleanlist
[NPENDINGIO
] ;
typedef struct swpagerclean
*swp_clean_t
;
extern vm_map_t kernel_map
;
queue_head_t swap_pager_done
; /* list of compileted page cleans */
queue_head_t swap_pager_inuse
; /* list of pending page cleans */
queue_head_t swap_pager_free
; /* list of free pager clean structs */
queue_head_t swap_pager_list
; /* list of "named" anon regions */
queue_head_t swap_pager_un_list
; /* list of "unnamed" anon pagers */
#define SWAP_FREE_NEEDED 0x1 /* need a swap block */
int swap_pager_needflags
;
static queue_head_t
*swp_qs
[]={
&swap_pager_list
, &swap_pager_un_list
, (queue_head_t
*) 0
struct pagerops swappagerops
= {
int npendingio
= NPENDINGIO
;
void swap_pager_finish();
extern int vm_page_count
;
void relpbuf(struct buf
*bp
) ;
dfltpagerops
= &swappagerops
;
queue_init(&swap_pager_list
);
queue_init(&swap_pager_un_list
);
queue_init(&swap_pager_inuse
);
queue_init(&swap_pager_done
);
queue_init(&swap_pager_free
);
* Calculate the swap allocation constants.
dmmin
= CLBYTES
/DEV_BSIZE
;
dmmax
= btodb(SWB_NPAGES
*NBPG
)*2;
* Allocate a pager structure and associated resources.
* Note that if we are called from the pageout daemon (handle == NULL)
* we should not wait for memory as it could resulting in deadlock.
swap_pager_alloc(handle
, size
, prot
, offset
)
register vm_pager_t pager
;
register swp_clean_t spc
;
* kva's are allocated here so that we dont need to keep
* doing kmem_alloc pageables at runtime
for (i
= 0, spc
= swcleanlist
; i
< npendingio
; i
++, spc
++) {
spc
->spc_kva
= kmem_alloc_pageable(pager_map
, NBPG
);
spc
->spc_bp
= malloc(sizeof( *bp
), M_TEMP
,
kmem_free_wakeup(pager_map
, spc
->spc_kva
, NBPG
);
queue_enter(&swap_pager_free
, spc
, swp_clean_t
, spc_list
);
* If this is a "named" anonymous region, look it up and
* return the appropriate pager if it exists.
pager
= vm_pager_lookup(&swap_pager_list
, handle
);
* Use vm_object_lookup to gain a reference
* to the object and also to remove from the
if (vm_object_lookup(pager
) == NULL
)
panic("swap_pager_alloc: bad object");
* Pager doesn't exist, allocate swap management resources
waitok
= handle
? M_WAITOK
: M_NOWAIT
;
pager
= (vm_pager_t
)malloc(sizeof *pager
, M_VMPAGER
, waitok
);
swp
= (sw_pager_t
)malloc(sizeof *swp
, M_VMPGDATA
, waitok
);
free((caddr_t
)pager
, M_VMPAGER
);
swp
->sw_nblocks
= (btodb(size
) + btodb(SWB_NPAGES
* NBPG
) - 1) / btodb(SWB_NPAGES
*NBPG
);
swp
->sw_blocks
= (sw_blk_t
)
malloc(swp
->sw_nblocks
*sizeof(*swp
->sw_blocks
),
if (swp
->sw_blocks
== NULL
) {
free((caddr_t
)swp
, M_VMPGDATA
);
free((caddr_t
)pager
, M_VMPAGER
);
bzero((caddr_t
)swp
->sw_blocks
,
swp
->sw_nblocks
* sizeof(*swp
->sw_blocks
));
for (i
= 0; i
< swp
->sw_nblocks
; i
++) {
for (j
= 0; j
< SWB_NPAGES
; j
++)
swp
->sw_blocks
[i
].swb_block
[j
] = SWB_EMPTY
;
swp
->sw_flags
= SW_NAMED
;
queue_enter(&swap_pager_list
, pager
, vm_pager_t
, pg_list
);
* Consistant with other pagers: return with object
* referenced. Can't do this with handle == NULL
* since it might be the pageout daemon calling.
object
= vm_object_allocate(size
);
vm_object_enter(object
, pager
);
vm_object_setpager(object
, pager
, 0, FALSE
);
queue_init(&pager
->pg_list
);
queue_enter(&swap_pager_un_list
, pager
, vm_pager_t
, pg_list
);
pager
->pg_handle
= handle
;
pager
->pg_ops
= &swappagerops
;
pager
->pg_type
= PG_SWAP
;
pager
->pg_data
= (caddr_t
)swp
;
* returns disk block associated with pager and offset
* additionally, as a side effect returns a flag indicating
* if the block has been written
swap_pager_diskaddr(swp
, offset
, valid
)
ix
= offset
/ (SWB_NPAGES
*NBPG
);
if (swp
->sw_blocks
== NULL
|| ix
>= swp
->sw_nblocks
) {
swb
= &swp
->sw_blocks
[ix
];
ix
= (offset
% (SWB_NPAGES
*NBPG
)) / NBPG
;
*valid
= swb
->swb_valid
& (1<<ix
);
return &swb
->swb_block
[ix
];
* Utility routine to set the valid (written) bit for
* a block associated with a pager and offset
swap_pager_setvalid(swp
, offset
, valid
)
ix
= offset
/ (SWB_NPAGES
*NBPG
);
if (swp
->sw_blocks
== NULL
|| ix
>= swp
->sw_nblocks
)
swb
= &swp
->sw_blocks
[ix
];
ix
= (offset
% (SWB_NPAGES
*NBPG
)) / NBPG
;
swb
->swb_valid
|= (1 << ix
);
swb
->swb_valid
&= ~(1 << ix
);
* this routine frees swap blocks from a specified pager
swap_pager_freespace(pager
, start
, size
)
sw_pager_t swp
= (sw_pager_t
) pager
->pg_data
;
for (i
= start
; i
< round_page(start
+ size
- 1); i
+= NBPG
) {
int *addr
= swap_pager_diskaddr(swp
, i
, 0);
if (addr
&& *addr
!= SWB_EMPTY
) {
rlist_free(&swapmap
, *addr
, *addr
+ btodb(NBPG
) - 1);
* swap_pager_reclaim frees up over-allocated space from all pagers
* this eliminates internal fragmentation due to allocation of space
* for segments that are never swapped to. It has been written so that
* it does not block until the rlist_free operation occurs; it keeps
* Maximum number of blocks (pages) to reclaim per pass
static int reclaims
[MAXRECLAIM
];
* allow only one process to be in the swap_pager_reclaim subroutine
tsleep((caddr_t
) &in_reclaim
, PSWP
, "swrclm", 0);
/* for each pager queue */
for (k
= 0; swp_qs
[k
]; k
++) {
p
= (vm_pager_t
) queue_first(swp_qs
[k
]);
while (reclaimcount
< MAXRECLAIM
&&
!queue_end(swp_qs
[k
], (queue_entry_t
) p
)) {
* see if any blocks associated with a pager has been
* allocated but not used (written)
swp
= (sw_pager_t
) p
->pg_data
;
for (i
= 0; i
< swp
->sw_nblocks
; i
++) {
sw_blk_t swb
= &swp
->sw_blocks
[i
];
for (j
= 0; j
< SWB_NPAGES
; j
++) {
if (swb
->swb_block
[j
] != SWB_EMPTY
&&
(swb
->swb_valid
& (1 << j
)) == 0) {
reclaims
[reclaimcount
++] = swb
->swb_block
[j
];
swb
->swb_block
[j
] = SWB_EMPTY
;
if (reclaimcount
>= MAXRECLAIM
)
p
= (vm_pager_t
) queue_next(&p
->pg_list
);
* free the blocks that have been added to the reclaim list
for (i
= 0; i
< reclaimcount
; i
++) {
rlist_free(&swapmap
, reclaims
[i
], reclaims
[i
] + btodb(NBPG
) - 1);
wakeup((caddr_t
) &in_reclaim
);
wakeup((caddr_t
) &in_reclaim
);
* swap_pager_copy copies blocks from one pager to another and
* destroys the source pager
swap_pager_copy(srcpager
, srcoffset
, dstpager
, dstoffset
, offset
)
sw_pager_t srcswp
, dstswp
;
srcswp
= (sw_pager_t
) srcpager
->pg_data
;
dstswp
= (sw_pager_t
) dstpager
->pg_data
;
* remove the source pager from the swap_pager internal queue
if (srcswp
->sw_flags
& SW_NAMED
) {
queue_remove(&swap_pager_list
, srcpager
, vm_pager_t
, pg_list
);
srcswp
->sw_flags
&= ~SW_NAMED
;
queue_remove(&swap_pager_un_list
, srcpager
, vm_pager_t
, pg_list
);
while (srcswp
->sw_poip
) {
tsleep((caddr_t
)srcswp
, PVM
, "spgout", 0);
* clean all of the pages that are currently active and finished
(void) swap_pager_clean(NULL
, B_WRITE
);
* clear source block before destination object
* (release allocated space)
for (i
= 0; i
< offset
+ srcoffset
; i
+= NBPG
) {
int *addr
= swap_pager_diskaddr(srcswp
, i
, 0);
if (addr
&& *addr
!= SWB_EMPTY
) {
rlist_free(&swapmap
, *addr
, *addr
+ btodb(NBPG
) - 1);
* transfer source to destination
for (i
= 0; i
< dstswp
->sw_osize
; i
+= NBPG
) {
int *srcaddrp
= swap_pager_diskaddr(srcswp
, i
+ offset
+ srcoffset
,
* see if the source has space allocated
if (srcaddrp
&& *srcaddrp
!= SWB_EMPTY
) {
* if the source is valid and the dest has no space, then
* copy the allocation from the srouce to the dest.
dstaddrp
= swap_pager_diskaddr(dstswp
, i
+ dstoffset
, &dstvalid
);
* if the dest already has a valid block, deallocate the
* source block without copying.
if (!dstvalid
&& dstaddrp
&& *dstaddrp
!= SWB_EMPTY
) {
rlist_free(&swapmap
, *dstaddrp
, *dstaddrp
+ btodb(NBPG
) - 1);
if (dstaddrp
&& *dstaddrp
== SWB_EMPTY
) {
swap_pager_setvalid(dstswp
, i
+ dstoffset
, 1);
* if the source is not empty at this point, then deallocate the space.
if (*srcaddrp
!= SWB_EMPTY
) {
rlist_free(&swapmap
, *srcaddrp
, *srcaddrp
+ btodb(NBPG
) - 1);
* deallocate the rest of the source object
for (i
= dstswp
->sw_osize
+ offset
+ srcoffset
; i
< srcswp
->sw_osize
; i
+= NBPG
) {
int *srcaddrp
= swap_pager_diskaddr(srcswp
, i
, 0);
if (srcaddrp
&& *srcaddrp
!= SWB_EMPTY
) {
rlist_free(&swapmap
, *srcaddrp
, *srcaddrp
+ btodb(NBPG
) - 1);
free((caddr_t
)srcswp
->sw_blocks
, M_VMPGDATA
);
free((caddr_t
)srcswp
, M_VMPGDATA
);
free((caddr_t
)srcpager
, M_VMPAGER
);
swap_pager_dealloc(pager
)
* Remove from list right away so lookups will fail if we
* block for pageout completion.
swp
= (sw_pager_t
) pager
->pg_data
;
if (swp
->sw_flags
& SW_NAMED
) {
queue_remove(&swap_pager_list
, pager
, vm_pager_t
, pg_list
);
swp
->sw_flags
&= ~SW_NAMED
;
queue_remove(&swap_pager_un_list
, pager
, vm_pager_t
, pg_list
);
* Wait for all pageouts to finish and remove
* all entries from cleaning list.
tsleep((caddr_t
)swp
, PVM
, "swpout", 0);
(void) swap_pager_clean(NULL
, B_WRITE
);
* Free left over swap blocks
for (i
= 0, bp
= swp
->sw_blocks
; i
< swp
->sw_nblocks
; i
++, bp
++) {
for (j
= 0; j
< SWB_NPAGES
; j
++)
if (bp
->swb_block
[j
] != SWB_EMPTY
) {
rlist_free(&swapmap
, (unsigned)bp
->swb_block
[j
],
(unsigned)bp
->swb_block
[j
] + btodb(NBPG
) - 1);
bp
->swb_block
[j
] = SWB_EMPTY
;
* Free swap management resources
free((caddr_t
)swp
->sw_blocks
, M_VMPGDATA
);
free((caddr_t
)swp
, M_VMPGDATA
);
free((caddr_t
)pager
, M_VMPAGER
);
* swap_pager_getmulti can get multiple pages.
swap_pager_getmulti(pager
, m
, count
, reqpage
, sync
)
return swap_pager_io((sw_pager_t
) pager
->pg_data
, m
, count
, reqpage
, B_READ
);
* swap_pager_getpage gets individual pages
swap_pager_getpage(pager
, m
, sync
)
return swap_pager_io((sw_pager_t
)pager
->pg_data
, marray
, 1, 0, B_READ
);
* swap_pager_putpage writes individual pages
swap_pager_putpage(pager
, m
, sync
)
(void) swap_pager_clean(NULL
, B_WRITE
);
return(swap_pager_io((sw_pager_t
)pager
->pg_data
, marray
, 1, 0, flags
));
swap_pager_block_index(swp
, offset
)
return (offset
/ (SWB_NPAGES
*NBPG
));
swap_pager_block_offset(swp
, offset
)
return (offset
% (SWB_NPAGES
*NBPG
));
* _swap_pager_haspage returns TRUE if the pager has data that has
_swap_pager_haspage(swp
, offset
)
ix
= offset
/ (SWB_NPAGES
*NBPG
);
if (swp
->sw_blocks
== NULL
|| ix
>= swp
->sw_nblocks
) {
swb
= &swp
->sw_blocks
[ix
];
ix
= (offset
% (SWB_NPAGES
*NBPG
)) / NBPG
;
if (swb
->swb_block
[ix
] != SWB_EMPTY
) {
if (swb
->swb_valid
& (1 << ix
))
* swap_pager_haspage is the externally accessible version of
* _swap_pager_haspage above. this routine takes a vm_pager_t
* for an argument instead of sw_pager_t.
swap_pager_haspage(pager
, offset
)
return _swap_pager_haspage((sw_pager_t
) pager
->pg_data
, offset
);
* swap_pager_freepage is a convienience routine that clears the busy
* bit and deallocates a page.
* swap_pager_ridpages is a convienience routine that deallocates all
* but the required page. this is usually used in error returns that
* need to invalidate the "extra" readahead pages.
swap_pager_ridpages(m
, count
, reqpage
)
for (i
= 0; i
< count
; i
++)
swap_pager_freepage(m
[i
]);
* swap_pager_iodone1 is the completion routine for both reads and async writes
if ((bp
->b_flags
& B_READ
) == 0)
* Scaled down version of swap().
* BOGUS: lower level IO routines expect a KVA so we have to map our
* provided physical page into the KVA to keep them happy.
swap_pager_io(swp
, m
, count
, reqpage
, flags
)
vm_offset_t paging_offset
;
int reqaddr
, mydskregion
;
object
= m
[reqpage
]->object
;
paging_offset
= object
->paging_offset
;
* First determine if the page exists in the pager if this is
* a sync read. This quickly handles cases where we are
* following shadow chains looking for the top level object
off
= m
[reqpage
]->offset
+ paging_offset
;
ix
= swap_pager_block_index(swp
, off
);
if (swp
->sw_blocks
== NULL
|| ix
>= swp
->sw_nblocks
) {
/* printf("swap pager: out of range\n"); */
swap_pager_ridpages(m
, count
, reqpage
);
swb
= &swp
->sw_blocks
[ix
];
off
= swap_pager_block_offset(swp
, off
) / NBPG
;
reqaddr
= swb
->swb_block
[off
];
/* make sure that our I/O request is contiguous */
int first
= 0, last
= count
;
int reqdskregion
= reqaddr
/ dmmax
;
if (reqaddr
== SWB_EMPTY
||
(swb
->swb_valid
& (1 << off
)) == 0) {
swap_pager_ridpages(m
, count
, reqpage
);
* search backwards for the first contiguous page to transfer
for (i
= reqpage
- 1; i
>= 0; --i
) {
int *tmpaddr
= swap_pager_diskaddr(swp
,
m
[i
]->offset
+ paging_offset
,&valid
);
if (tmpaddr
== 0 || failed
|| !valid
||
*tmpaddr
!= reqaddr
+ btodb((i
- reqpage
) * NBPG
)) {
swap_pager_freepage(m
[i
]);
mydskregion
= *tmpaddr
/ dmmax
;
if (mydskregion
!= reqdskregion
) {
swap_pager_freepage(m
[i
]);
* search forwards for the last contiguous page to transfer
for (i
= reqpage
+ 1; i
< count
; i
++) {
int *tmpaddr
= swap_pager_diskaddr(swp
, m
[i
]->offset
+ paging_offset
,&valid
);
if (tmpaddr
== 0 || failed
|| !valid
||
*tmpaddr
!= reqaddr
+ btodb((i
- reqpage
) * NBPG
) ) {
swap_pager_freepage(m
[i
]);
mydskregion
= *tmpaddr
/ dmmax
;
if (mydskregion
!= reqdskregion
) {
swap_pager_freepage(m
[i
]);
for (i
= first
; i
< count
; i
++) {
* "m" is a pointer to the array of vm_page_t for paging I/O
* "count" is the number of vm_page_t entries represented by "m"
* "object" is the vm_object_t for I/O
* "reqpage" is the index into "m" for the page actually faulted
* For reads (pageins) and synchronous writes, we clean up
* all completed async pageouts.
if ((flags
& B_ASYNC
) == 0) {
swap_pager_clean(NULL
, flags
);
* For async writes (pageouts), we cleanup completed pageouts so
* that all available resources are freed. Also tells us if this
* page is already being cleaned. If it is, or no resources
* are available, we try again later.
else if (swap_pager_clean(m
[reqpage
], B_WRITE
)) {
swap_pager_ridpages(m
, count
, reqpage
);
return VM_PAGER_TRYAGAIN
;
spc
= NULL
; /* we might not use an spc data structure */
* we allocate a new kva for transfers > 1 page
* but for transfers == 1 page, the swap_pager_free list contains
* entries that have pre-allocated kva's (for efficiency).
if ((flags
& B_READ
) && count
> 1) {
kva
= kmem_alloc_pageable(pager_map
, count
*NBPG
);
* if a kva has not been allocated, we can only do a one page transfer,
* so we free the other pages that might have been allocated by vm_fault.
for (i
= 0; i
< count
; i
++) {
swap_pager_freepage(m
[i
]);
* get a swap pager clean data structure, block until we get it
if (queue_empty(&swap_pager_free
)) {
if ((flags & (B_ASYNC|B_READ)) == B_ASYNC)
return VM_PAGER_TRYAGAIN;
(void) swap_pager_clean(NULL
, B_WRITE
);
wakeup((caddr_t
) &vm_pages_needed
);
while (queue_empty(&swap_pager_free
)) {
swap_pager_needflags
|= SWAP_FREE_NEEDED
;
tsleep((caddr_t
)&swap_pager_free
,
(void) swap_pager_clean(NULL
, B_WRITE
);
wakeup((caddr_t
) &vm_pages_needed
);
queue_remove_first(&swap_pager_free
, spc
, swp_clean_t
, spc_list
);
* Determine swap block and allocate as necessary.
* We try to get SWB_NPAGES first, but then we punt and try
* to get one page. If that fails, we look at the allocation
* data structures to find unused but allocated pages in other
if (reqaddr
== SWB_EMPTY
) {
* if any other pages have been allocated in this block, we
* only try to get one page.
for (i
= 0; i
< SWB_NPAGES
; i
++) {
if (swb
->swb_block
[i
] != SWB_EMPTY
)
ntoget
= (i
== SWB_NPAGES
) ? SWB_NPAGES
: 1;
if (ntoget
== SWB_NPAGES
&&
rlist_alloc(&swapmap
, btodb(ntoget
* NBPG
),&blk
)) {
for (i
= 0; i
< ntoget
; i
++)
swb
->swb_block
[i
] = blk
+ btodb(NBPG
) * i
;
} else if (!rlist_alloc(&swapmap
, btodb(NBPG
), &swb
->swb_block
[off
])) {
* if the allocation has failed, we try to reclaim space and
* here on swap space full.
queue_enter(&swap_pager_free
, spc
, swp_clean_t
, spc_list
);
if (swap_pager_full
== 0)
printf("swap_pager: out of swap space !!!\n");
swap_pager_ridpages(m
, count
, reqpage
);
return(VM_PAGER_TRYAGAIN
);
* map our page(s) into kva for I/O
for (i
= 0; i
< count
; i
++) {
pmap_enter(vm_map_pmap(pager_map
), kva
+ NBPG
* i
,
VM_PAGE_TO_PHYS(m
[i
]), VM_PROT_ALL
, TRUE
);
* get the base I/O offset into the swap file
off
= swap_pager_block_offset(swp
, m
[0]->offset
+ paging_offset
) / NBPG
;
if (flags
& B_READ
&& count
> 1)
printf("obj: 0x%x off: 0x%x poff: 0x%x off: 0x%x, sz: %d blk: %d op: %s\n",
object
, m
[0]->offset
, paging_offset
, off
, count
, swb
->swb_block
[off
], flags
&B_READ
?"r":"w");
* Get a swap buffer header and perform the IO
bp
->b_flags
= B_BUSY
| (flags
& B_READ
);
bp
->b_proc
= &proc0
; /* XXX (but without B_PHYS set this is ok) */
bp
->b_rcred
= bp
->b_wcred
= bp
->b_proc
->p_ucred
;
bp
->b_un
.b_addr
= (caddr_t
) kva
;
bp
->b_blkno
= swb
->swb_block
[off
];
if (swapdev_vp
->v_type
== VBLK
)
bp
->b_dev
= swapdev_vp
->v_rdev
;
bp
->b_bcount
= NBPG
*count
;
if ((bp
->b_flags
& B_READ
) == 0)
swapdev_vp
->v_numoutput
++;
* If this is an async write we set up additional buffer fields
* and place a "cleaning" entry on the inuse queue.
if ((flags
& (B_READ
|B_ASYNC
)) == B_ASYNC
) {
* the completion routine for async writes
bp
->b_iodone
= swap_pager_iodone
;
bp
->b_dirtyend
= bp
->b_bcount
;
queue_enter(&swap_pager_inuse
, spc
, swp_clean_t
, spc_list
);
* we remember that we have used a block for paging.
swb
->swb_valid
|= (1 << off
);
* here for sync write or any read
if ((flags
& B_READ
) == 0) {
* if we are writing, we remember that we have
* actually used a block for paging.
swb
->swb_valid
|= (1 << off
);
* the completion routine for reads and sync writes
bp
->b_iodone
= swap_pager_iodone1
;
if ((flags
& (B_READ
|B_ASYNC
)) == B_ASYNC
) {
if ((bp
->b_flags
& B_DONE
) == B_DONE
) {
swap_pager_clean(NULL
, flags
);
* wait for the sync I/O to complete
while ((bp
->b_flags
& B_DONE
) == 0) {
tsleep((caddr_t
)bp
, PVM
, (flags
& B_READ
)?"swread":"swwrt", 0);
rv
= (bp
->b_flags
& B_ERROR
) ? VM_PAGER_FAIL
: VM_PAGER_OK
;
bp
->b_flags
&= ~(B_BUSY
|B_WANTED
|B_PHYS
|B_DIRTY
|B_CALL
|B_DONE
);
if (bp
->b_flags
& B_READ
) {
* release the physical I/O buffer
* remove the mapping for kernel virtual
pmap_remove(vm_map_pmap(pager_map
), kva
, kva
+ count
* NBPG
);
* if we have written the page, then indicate that the page
if ((flags
& B_READ
) == 0 && rv
== VM_PAGER_OK
) {
m
[reqpage
]->flags
|= PG_CLEAN
;
pmap_clear_modify(VM_PAGE_TO_PHYS(m
[reqpage
]));
* optimization, if a page has been read during the
* pageout process, we activate it.
if ( (m
[reqpage
]->flags
& PG_ACTIVE
) == 0 &&
pmap_is_referenced(VM_PAGE_TO_PHYS(m
[reqpage
])))
vm_page_activate(m
[reqpage
]);
* if we have used an spc, we need to free it.
queue_enter(&swap_pager_free
, spc
, swp_clean_t
, spc_list
);
for (i
= 0; i
< count
; i
++) {
pmap_clear_modify(VM_PAGE_TO_PHYS(m
[i
]));
m
[i
]->flags
&= ~PG_LAUNDRY
;
* whether or not to leave the page activated
* is up in the air, but we should put the page
* on a page queue somewhere. (it already is in
* After some emperical results, it is best
* to deactivate the readahead pages.
vm_page_deactivate(m
[i
]);
* just in case someone was asking for this
* page we now tell them that it is ok to use
* and free the kernel virtual addresses
kmem_free_wakeup(pager_map
, kva
, count
* NBPG
);
register swp_clean_t spc
, tspc
;
if (queue_empty(&swap_pager_done
))
* Look up and removal from done list must be done
* at splbio() to avoid conflicts with swap_pager_iodone.
spc
= (swp_clean_t
) queue_first(&swap_pager_done
);
while (!queue_end(&swap_pager_done
, (queue_entry_t
)spc
)) {
pmap_remove(vm_map_pmap(pager_map
), spc
->spc_kva
, ((vm_offset_t
) spc
->spc_kva
) + NBPG
);
queue_remove(&swap_pager_done
, spc
, swp_clean_t
, spc_list
);
* No operations done, thats all we can do for now.
* The desired page was found to be busy earlier in
* the scan but has since completed.
if (tspc
&& tspc
== spc
) {
queue_enter(&swap_pager_free
, spc
, swp_clean_t
, spc_list
);
return(tspc
? TRUE
: FALSE
);
register swp_clean_t spc
;
vm_page_t m
= spc
->spc_m
;
vm_object_t object
= m
->object
;
extern int vm_pageout_free_min
;
if (--object
->paging_in_progress
== 0)
thread_wakeup((int) object
);
* If no error mark as clean and inform the pmap system.
* If error, mark as dirty so we will try again.
* (XXX could get stuck doing this, should give up after awhile)
if (spc
->spc_flags
& SPC_ERROR
) {
printf("swap_pager_finish: clean of page %x failed\n",
pmap_clear_modify(VM_PAGE_TO_PHYS(m
));
* if a page has been read during pageout, then
if ((m
->flags
& PG_ACTIVE
) == 0 &&
pmap_is_referenced(VM_PAGE_TO_PHYS(m
)))
* we wakeup any processes that are waiting on
* if we need memory desperately, then free it now
if (vm_page_free_count
< vm_page_free_reserved
&&
(m
->flags
& PG_CLEAN
) && m
->wire_count
== 0) {
pmap_page_protect(VM_PAGE_TO_PHYS(m
), VM_PROT_NONE
);
register swp_clean_t spc
;
spc
= (swp_clean_t
) bp
->b_spc
;
queue_remove(&swap_pager_inuse
, spc
, swp_clean_t
, spc_list
);
queue_enter(&swap_pager_done
, spc
, swp_clean_t
, spc_list
);
if (bp
->b_flags
& B_ERROR
) {
spc
->spc_flags
|= SPC_ERROR
;
printf("error %d blkno %d sz %d ",
bp
->b_error
, bp
->b_blkno
, bp
->b_bcount
);
if ((bp
->b_flags
& B_READ
) == 0)
bp
->b_flags
&= ~(B_BUSY
|B_WANTED
|B_PHYS
|B_DIRTY
|B_ASYNC
);
if (--spc
->spc_swp
->sw_poip
== 0) {
wakeup((caddr_t
)spc
->spc_swp
);
if ((swap_pager_needflags
& SWAP_FREE_NEEDED
) ||
queue_empty(&swap_pager_inuse
)) {
swap_pager_needflags
&= ~SWAP_FREE_NEEDED
;
wakeup((caddr_t
)&swap_pager_free
);
wakeup((caddr_t
)&vm_pages_needed
);
if (vm_pageout_pages_needed
) {
wakeup((caddr_t
)&vm_pageout_pages_needed
);
if (queue_empty(&swap_pager_inuse
) ||
(vm_page_free_count
< vm_page_free_min
&&
nswiodone
+ vm_page_free_count
>= vm_page_free_min
) ) {
wakeup((caddr_t
)&vm_pages_needed
);
* allocate a physical buffer
/* get a bp from the swap buffer header pool */
while (bswlist
.av_forw
== NULL
) {
bswlist
.b_flags
|= B_WANTED
;
tsleep((caddr_t
)&bswlist
, PVM
, "wswbuf", 0);
bswlist
.av_forw
= bp
->av_forw
;
* release a physical buffer
bp
->av_forw
= bswlist
.av_forw
;
if (bswlist
.b_flags
& B_WANTED
) {
bswlist
.b_flags
&= ~B_WANTED
;
wakeup((caddr_t
)&bswlist
);
* return true if any swap control structures can be allocated
if( queue_empty( &swap_pager_free
))