hypervisor/src/greatlakes/ontario/src/mmu.s

/*
* ========== Copyright Header Begin ==========================================
*
* Hypervisor Software File: mmu.s
*
* Copyright (c) 2006 Sun Microsystems, Inc. All Rights Reserved.
*
*  - Do no alter or remove copyright notices
*
*  - Redistribution and use of this software in source and binary forms, with
*    or without modification, are permitted provided that the following
*    conditions are met:
*
*  - Redistribution of source code must retain the above copyright notice,
*    this list of conditions and the following disclaimer.
*
*  - Redistribution in binary form must reproduce the above copyright notice,
*    this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*
*    Neither the name of Sun Microsystems, Inc. or the names of contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*     This software is provided "AS IS," without a warranty of any kind.
* ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
* INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
* PARTICULAR PURPOSE OR NON-INFRINGEMENT, ARE HEREBY EXCLUDED. SUN
* MICROSYSTEMS, INC. ("SUN") AND ITS LICENSORS SHALL NOT BE LIABLE FOR
* ANY DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING OR
* DISTRIBUTING THIS SOFTWARE OR ITS DERIVATIVES. IN NO EVENT WILL SUN
* OR ITS LICENSORS BE LIABLE FOR ANY LOST REVENUE, PROFIT OR DATA, OR
* FOR DIRECT, INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL OR PUNITIVE
* DAMAGES, HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY,
* ARISING OUT OF THE USE OF OR INABILITY TO USE THIS SOFTWARE, EVEN IF
* SUN HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
*
* You acknowledge that this software is not designed, licensed or
* intended for use in the design, construction, operation or maintenance of
* any nuclear facility.
*
* ========== Copyright Header End ============================================
*/
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

        .ident  "@(#)mmu.s      1.45    07/05/03 SMI"

/*
 * Niagara mmu code
 */

#include <sys/asm_linkage.h>
#include <hprivregs.h>
#include <asi.h>
#include <traps.h>
#include <mmu.h>
#include <sun4v/traps.h>
#include <sun4v/mmu.h>
#include <mmustat.h>
#include <cpu_errs.h>

#include <guest.h>
#include <offsets.h>
#include <debug.h>
#include <util.h>

        ! %g1   vcpup
        ! %g2   8k-aligned real addr from tag access
        ENTRY_NP(rdmmu_miss)
        ! offset handling
        ! XXX if hypervisor access then panic instead of watchdog_guest
        VCPU2GUEST_STRUCT(%g1, %g7)
        set     8 KB, %g5
        RA2PA_RANGE_CONV(%g7, %g2, %g5, 1f, %g4, %g3)
        !       %g3     PA

        ! tte valid, cp, writable, priv
        mov     1, %g2
        sllx    %g2, 63, %g2
        or      %g2, TTE4U_CP | TTE4U_P | TTE4U_W, %g2
        or      %g3, %g2, %g3
        mov     TLB_IN_REAL, %g2        ! Real bit
        stxa    %g3, [%g2]ASI_DTLB_DATA_IN
        retry

1:
        GUEST_STRUCT(%g1)
        set     8192, %g6
        RANGE_CHECK_IO(%g1, %g2, %g6, .rdmmu_miss_found, .rdmmu_miss_not_found,
            %g3, %g4)
.rdmmu_miss_found:
        mov     %g2, %g3

        ! tte valid, e, writable, priv
        mov     1, %g2
        sllx    %g2, 63, %g2
        or      %g2, TTE4U_E | TTE4U_P | TTE4U_W, %g2
        or      %g3, %g2, %g3
        mov     TLB_IN_REAL, %g2        ! Real bit
        stxa    %g3, [%g2]ASI_DTLB_DATA_IN
        retry


.rdmmu_miss_not_found:
1:
        ! FIXME: This test to be subsumed when we fix the RA mappings
        ! for multiple RA blocks
        ! %g1 guest struct
        ! %g2 real address
        set     GUEST_LDC_MAPIN_BASERA, %g7
        ldx     [ %g1 + %g7 ], %g3
        subcc   %g2, %g3, %g4
        bneg,pn %xcc, 2f
          nop
        set     GUEST_LDC_MAPIN_SIZE, %g5
        ldx     [ %g1 + %g5 ], %g6
        subcc   %g4, %g6, %g0
                ! check regs passed in to mapin_ra:
        bneg,pt %xcc, ldc_dmmu_mapin_ra
          nop

        ENTRY_NP(rdmmu_miss_not_found2)
2:
        LEGION_GOT_HERE
        mov     MMU_FT_INVALIDRA, %g1
        ba,pt   %xcc, revec_dax ! (%g1=ft, %g2=addr, %g3=ctx)
        mov     0, %g3
        SET_SIZE(rdmmu_miss)

        !
        ! %g1 = vcpup
        !
        ENTRY_NP(rimmu_miss)
        mov     MMU_TAG_ACCESS, %g2
        ldxa    [%g2]ASI_IMMU, %g2      /* tag access */
        set     ((1 << 13) - 1), %g3
        andn    %g2, %g3, %g2

        VCPU2GUEST_STRUCT(%g1, %g3)
        RA2PA_RANGE_CONV(%g3, %g2, %g0, 1f, %g4, %g1)
        ! %g1   PA

        ! tte valid, cp, writable, priv
        mov     1, %g2
        sllx    %g2, 63, %g2
        or      %g2, TTE4U_CP | TTE4U_P | TTE4U_W, %g2
        or      %g1, %g2, %g1
        mov     TLB_IN_REAL, %g2        ! Real bit
        stxa    %g1, [%g2]ASI_ITLB_DATA_IN
        retry

1:
        ! %g2 real address
        LEGION_GOT_HERE
        mov     MMU_FT_INVALIDRA, %g1
        ba,pt   %xcc, revec_iax ! (%g1=ft, %g2=addr, %g3=ctx)
        mov     0, %g3
        SET_SIZE(rimmu_miss)


        /*
         * Normal tlb miss handlers
         *
         * Guest miss area:
         *
         * NB:   If it's possible to context switch a guest then
         * the tag access register (tag target too?) needs to
         * be saved/restored.
         */

        /* %g1 contains per CPU area */
        ENTRY_NP(immu_miss)
        rd      %tick, %g2
        stx     %g2, [%g1 + CPU_SCR0]
        ldxa    [%g0]ASI_IMMU, %g3      /* tag target */
        srlx    %g3, TAGTRG_CTX_RSHIFT, %g4     /* ctx from tag target */

        ! %g1 = CPU pointer
        ! %g3 = tag target
        ! %g4 = ctx

.checkitsb0:
        ! for context != 0 and unshared TSB, that ctx == TSB ctx
        brz,pn  %g4, 1f
        mov     %g3, %g2
        ld      [%g1 + CPU_TSBDS_CTXN + TSBD_CTX_INDEX], %g5
        cmp     %g5, -1
        be,pn   %icc, 1f
        nop
        ! if TSB not shared, zero out context for match
        sllx    %g3, TAGTRG_VA_LSHIFT, %g2
        srlx    %g2, TAGTRG_VA_LSHIFT, %g2      ! clear context
1:
        ldxa    [%g0]ASI_IMMU_TSB_PS0, %g5
        ! if TSB desc. specifies xor of TSB index, do it here
        ! e.g. for shared TSBs in S9 xor value is ctx << 4
        ldda    [%g5]ASI_QUAD_LDD, %g6  /* g6 = tag, g7 = data */
        cmp     %g6, %g2
        bne,pn  %xcc, .checkitsb1       ! tag mismatch
        nop
        brlz,pt %g7, .itsbhit           ! TTE valid
        nop

.checkitsb1:
        ! repeat check for second TSB
        brz,pn  %g4, 1f
        mov     %g3, %g2
        ld      [%g1 + CPU_TSBDS_CTXN + TSBD_BYTES + TSBD_CTX_INDEX], %g5
        cmp     %g5, -1
        be,pn   %icc, 1f
        nop
        ! if TSB not shared, zero out context for match
        sllx    %g3, TAGTRG_VA_LSHIFT, %g2
        srlx    %g2, TAGTRG_VA_LSHIFT, %g2      ! clear context
1:
        ldxa    [%g0]ASI_IMMU_TSB_PS1, %g5
        ! if TSB desc. specifies xor of TSB index, do it here
        ldda    [%g5]ASI_QUAD_LDD, %g6  /* g6 = tag, g7 = data */
        cmp     %g6, %g2
        bne,pn  %xcc, .checkipermmaps   ! tag mismatch
        nop
        brgez,pn %g7, .checkipermmaps   ! TTE valid?
        nop

.itsbhit:
        ! extract sz from tte
        TTE_SIZE(%g7, %g4, %g3, .itsb_inv_pgsz)
        btst    TTE_X, %g7      ! must check X bit for IMMU
        bz,pn   %icc, .itsbmiss
        sub     %g4, 1, %g5     ! %g5 page mask

        ! extract ra from tte
        sllx    %g7, 64 - 40, %g3
        srlx    %g3, 64 - 40 + 13, %g3
        sllx    %g3, 13, %g3    ! %g3 real address
        xor     %g7, %g3, %g7   ! %g7 orig tte with ra field zeroed
        andn    %g3, %g5, %g3

        VCPU2GUEST_STRUCT(%g1, %g6)
        RA2PA_RANGE_CONV_UNK_SIZE(%g6, %g3, %g4, .itsb_ra_range, %g5, %g1) ! XXX fault not just a miss
        mov     %g1, %g3
        VCPU_STRUCT(%g1)                ! restore vcpu

        or      %g7, %g3, %g7   ! %g7 new tte with pa

        CLEAR_TTE_LOCK_BIT(%g7, %g5)

        set     TLB_IN_4V_FORMAT, %g5   ! %g5 sun4v-style tte selection
        stxa    %g7, [%g5]ASI_ITLB_DATA_IN
        !
        ! %g1 = CPU pointer
        ! %g7 = TTE
        !
        ldx     [%g1 + CPU_MMUSTAT_AREA], %g6
        brnz,pn %g6, 1f
        nop

        retry

1:
        rd      %tick, %g2
        ldx     [%g1 + CPU_SCR0], %g1
        sub     %g2, %g1, %g5
        !
        ! %g5 = %tick delta
        ! %g6 = MMU statistics area
        ! %g7 = TTE
        !
        inc     MMUSTAT_I, %g6                  /* stats + i */
        ldxa    [%g0]ASI_IMMU, %g3              /* tag target */
        srlx    %g3, TAGTRG_CTX_RSHIFT, %g4     /* ctx from tag target */
        mov     MMUSTAT_CTX0, %g1
        movrnz  %g4, MMUSTAT_CTXNON0, %g1
        add     %g6, %g1, %g6                   /* stats + i + ctx */
        and     %g7, TTE_SZ_MASK, %g7
        sllx    %g7, MMUSTAT_ENTRY_SZ_SHIFT, %g7
        add     %g6, %g7, %g6                   /* stats + i + ctx + pgsz */
        ldx     [%g6 + MMUSTAT_TICK], %g3
        add     %g3, %g5, %g3
        stx     %g3, [%g6 + MMUSTAT_TICK]
        ldx     [%g6 + MMUSTAT_HIT], %g3
        inc     %g3
        stx     %g3, [%g6 + MMUSTAT_HIT]
        retry

        ! %g1 = CPU struct
        ! %g4 = context
.checkipermmaps:
        brnz,pt %g4, .itsbmiss          ! only context zero has perm mappings
          nop
        VCPU2GUEST_STRUCT(%g1, %g2)
        mov     GUEST_PERM_MAPPINGS_INCR*(NPERMMAPPINGS-1), %g3
        add     %g3, GUEST_PERM_MAPPINGS, %g3
        add     %g2, %g3, %g2
        mov     -(GUEST_PERM_MAPPINGS_INCR*(NPERMMAPPINGS-1)), %g3
        rdpr    %tpc, %g4
1:
        ldda    [ %g2 + %g3 ] ASI_QUAD_LDD, %g6         ! Ld TTE (g7) + Tag (g6)

        ! Figure page size match mask
        ! FIXME: Could speed this by storing the mask ... but
        ! atomicity problems with storage. Other option is
        ! store pre-computed page size shift in tag bits 0-13
        brgez,pn %g7, 2f
        and     %g7, TTE_SZ_MASK, %g5
        add     %g5, %g5, %g1
        add     %g5, %g1, %g1           ! Mult size by 3
        add     %g1, 13, %g1            ! Add 13
        mov     1, %g5
        sllx    %g5, %g1, %g5           ! Compute bytes per page
        sub     %g5, 1, %g5             ! Page mask for TTE retrieved

        xor     %g6, %g4, %g6
        andncc  %g6, %g5, %g0           ! Check for tag match

        beq,pt %xcc, 3f
          nop

2:
        brlz,pt %g3, 1b
          add   %g3, GUEST_PERM_MAPPINGS_INCR, %g3

        VCPU_STRUCT(%g1)
        ba,pt   %xcc, .itsbmiss
          mov   %g0, %g4

3:
        ! Found a matching entry - can we load it into the ITLB
        VCPU_STRUCT(%g1)
        add     %g2, %g3, %g2   ! Ptr to map entry

        ! Calculate index into perm bit set
        ldub    [%g1 + CPU_VID], %g3
        and     %g3, MAPPING_XWORD_MASK, %g4
        mov     1, %g5
        sllx    %g5, %g4, %g4   ! Bit in mask
        srlx    %g3, MAPPING_XWORD_SHIFT, %g3
        sllx    %g3, MAPPING_XWORD_BYTE_SHIFT_BITS, %g3
        add     %g2, %g3, %g2

        ldx     [%g2 + MAPPING_ICPUSET], %g3
        btst    %g3, %g4
        bz,pn   %xcc, .itsbmiss
          mov   %g0, %g4

        ! Stuff entry - it's already been swizzled
        set     TLB_IN_4V_FORMAT, %g5   ! %g5 sun4v-style tte selection
        stxa    %g7, [%g5]ASI_ITLB_DATA_IN

        retry

.itsbmiss:
        ldx     [%g1 + CPU_MMU_AREA], %g2
        brz,pn  %g2, watchdog_guest
        .empty

        ! %g1 is CPU pointer
        ! %g2 is MMU Fault Status Area
        ! %g4 is context (possibly shifted - still OK for zero test)
        /* if ctx == 0 and ctx0 set TSBs used, take slow trap */
        /* if ctx != 0 and ctxnon0 set TSBs used, take slow trap */
        mov     CPU_NTSBS_CTXN, %g7
        movrz   %g4, CPU_NTSBS_CTX0, %g7
        ldx     [%g1 + %g7], %g7
        brnz,pn %g7, .islowmiss
        nop

.ifastmiss:
        /*
         * Update MMU_FAULT_AREA_INSTR
         */
        mov     MMU_TAG_ACCESS, %g3
        ldxa    [%g3]ASI_IMMU, %g3      /* tag access */
        set     (NCTXS - 1), %g5
        andn    %g3, %g5, %g4
        and     %g3, %g5, %g5
        stx     %g4, [%g2 + MMU_FAULT_AREA_IADDR]
        stx     %g5, [%g2 + MMU_FAULT_AREA_ICTX]
        /* fast misses do not update MMU_FAULT_AREA_IFT with MMU_FT_FASTMISS */
        ! wrpr  %g0, TT_FAST_IMMU_MISS, %tt     /* already set */
        rdpr    %pstate, %g3
        or      %g3, PSTATE_PRIV, %g3
        wrpr    %g3, %pstate
        rdpr    %tba, %g3
        add     %g3, (TT_FAST_IMMU_MISS << TT_OFFSET_SHIFT), %g3
7:
        rdpr    %tl, %g2
        cmp     %g2, 1  /* trap happened at tl=0 */
        be,pt   %xcc, 1f
        .empty
        set     TRAPTABLE_SIZE, %g5

        cmp     %g2, MAXPTL
        bgu,pn  %xcc, watchdog_guest
        add     %g5, %g3, %g3

1:
        mov     HPSTATE_GUEST, %g5              ! set ENB bit
        jmp     %g3
        wrhpr   %g5, %hpstate

.islowmiss:
        /*
         * Update MMU_FAULT_AREA_INSTR
         */
        mov     MMU_TAG_TARGET, %g3
        ldxa    [%g3]ASI_IMMU, %g3      /* tag target */
        srlx    %g3, TAGTRG_CTX_RSHIFT, %g3
        stx     %g3, [%g2 + MMU_FAULT_AREA_ICTX]
        rdpr    %tpc, %g4
        stx     %g4, [%g2 + MMU_FAULT_AREA_IADDR]
        mov     MMU_FT_MISS, %g4
        stx     %g4, [%g2 + MMU_FAULT_AREA_IFT]
        wrpr    %g0, TT_IMMU_MISS, %tt
        rdpr    %pstate, %g3
        or      %g3, PSTATE_PRIV, %g3
        wrpr    %g3, %pstate
        rdpr    %tba, %g3
        add     %g3, (TT_IMMU_MISS << TT_OFFSET_SHIFT), %g3
        ba,a    7b
        .empty

.itsb_inv_pgsz:
        /* IAX with FT=Invalid Page Size (15), VA, CTX */
        ba,pt   %xcc, .itsb_iax
        mov     MMU_FT_PAGESIZE, %g3

.itsb_ra_range:
        /* IAX with FT=Invalid TSB Entry (16), VA, CTX */
        mov     MMU_FT_INVTSBENTRY, %g3
        /*FALLTHROUGH*/

.itsb_iax:
        !! %g1 = cpup
        ldx     [%g1 + CPU_MMU_AREA], %g2
        brz,pn  %g2, watchdog_guest             ! Nothing we can do about this
        nop
        stx     %g3, [%g2 + MMU_FAULT_AREA_IFT]
        mov     MMU_TAG_TARGET, %g3
        ldxa    [%g3]ASI_IMMU, %g3      /* tag target */
        srlx    %g3, TAGTRG_CTX_RSHIFT, %g3
        stx     %g3, [%g2 + MMU_FAULT_AREA_ICTX]
        rdpr    %tpc, %g3
        stx     %g3, [%g2 + MMU_FAULT_AREA_IADDR]
        REVECTOR(TT_IAX)
        SET_SIZE(immu_miss)


        /* %g1 contains per CPU area */
        ENTRY_NP(dmmu_miss)
        rd      %tick, %g2
        stx     %g2, [%g1 + CPU_SCR0]
        ldxa    [%g0]ASI_DMMU, %g3      /* tag target */
        srlx    %g3, TAGTRG_CTX_RSHIFT, %g4     /* ctx from tag target */

        ! %g1 = CPU pointer
        ! %g3 = tag target
        ! %g4 = ctx

.checkdtsb0:
        ! for context != 0 and unshared TSB, that ctx == TSB ctx
        brz,pn  %g4, 1f
        mov     %g3, %g2
        ld      [%g1 + CPU_TSBDS_CTXN + TSBD_CTX_INDEX], %g5
        cmp     %g5, -1
        be,pn   %icc, 1f
        nop
        ! if TSB not shared, zero out context for match
        sllx    %g3, TAGTRG_VA_LSHIFT, %g2
        srlx    %g2, TAGTRG_VA_LSHIFT, %g2      ! clear context
1:
        ldxa    [%g0]ASI_DMMU_TSB_PS0, %g5
        ! if TSB desc. specifies xor of TSB index, do it here
        ! e.g. for shared TSBs in S9 xor value is ctx << 4
        ldda    [%g5]ASI_QUAD_LDD, %g6  /* g6 = tag, g7 = data */
        cmp     %g6, %g2
        bne,pn  %xcc, .checkdtsb1       ! tag mismatch
        nop
        brlz,pt %g7, .dtsbhit           ! TTE valid
        nop

.checkdtsb1:
        ! repeat check for second TSB
        brz,pn  %g4, 1f
        mov     %g3, %g2
        ld      [%g1 + CPU_TSBDS_CTXN + TSBD_BYTES + TSBD_CTX_INDEX], %g5
        cmp     %g5, -1
        be,pn   %icc, 1f
        nop
        ! if TSB not shared, zero out context for match
        sllx    %g3, TAGTRG_VA_LSHIFT, %g2
        srlx    %g2, TAGTRG_VA_LSHIFT, %g2      ! clear context
1:
        ldxa    [%g0]ASI_DMMU_TSB_PS1, %g5
        ! if TSB desc. specifies xor of TSB index, do it here
        ldda    [%g5]ASI_QUAD_LDD, %g6  /* g6 = tag, g7 = data */
        cmp     %g6, %g2
        bne,pn  %xcc, .checkdpermmaps           ! tag mismatch
        nop
        brgez,pn %g7, .checkdpermmaps           ! TTE valid
        nop

.dtsbhit:
        ! extract sz from tte
        TTE_SIZE(%g7, %g4, %g3, .dtsb_inv_pgsz)
        sub     %g4, 1, %g5     ! %g5 page mask

        ! extract ra from tte
        sllx    %g7, 64 - 40, %g3
        srlx    %g3, 64 - 40 + 13, %g3
        sllx    %g3, 13, %g3    ! %g3 real address
        xor     %g7, %g3, %g7   ! %g7 orig tte with ra field zeroed
        andn    %g3, %g5, %g3
        ldx     [%g1 + CPU_GUEST], %g6


        ! %g1 cpu struct
        ! %g2 --
        ! %g3 raddr
        ! %g4 page size
        ! %g5 --
        ! %g6 guest struct
        ! %g7 TTE ready for pa
        RA2PA_RANGE_CONV_UNK_SIZE(%g6, %g3, %g4, 3f, %g5, %g2)
        mov     %g2, %g3                ! %g3   PA
4:
        ! %g1 cpu struct
        ! %g3 paddr
        ! %g7 TTE ready for pa
        or      %g7, %g3, %g7   ! %g7 new tte with pa

        CLEAR_TTE_LOCK_BIT(%g7, %g5)

        set     TLB_IN_4V_FORMAT, %g5   ! %g5 sun4v-style tte selection
        stxa    %g7, [%g5]ASI_DTLB_DATA_IN
        !
        ! %g1 = CPU pointer
        ! %g7 = TTE
        !
        ldx     [%g1 + CPU_MMUSTAT_AREA], %g6
        brnz,pn %g6, 1f
        nop

        retry

1:
        rd      %tick, %g2
        ldx     [%g1 + CPU_SCR0], %g1
        sub     %g2, %g1, %g5
        !
        ! %g5 = %tick delta
        ! %g6 = MMU statistics area
        ! %g7 = TTE
        !
        inc     MMUSTAT_D, %g6                  /* stats + d */
        ldxa    [%g0]ASI_DMMU, %g3              /* tag target */
        srlx    %g3, TAGTRG_CTX_RSHIFT, %g4     /* ctx from tag target */
        mov     MMUSTAT_CTX0, %g1
        movrnz  %g4, MMUSTAT_CTXNON0, %g1
        add     %g6, %g1, %g6                   /* stats + d + ctx */
        and     %g7, TTE_SZ_MASK, %g7
        sllx    %g7, MMUSTAT_ENTRY_SZ_SHIFT, %g7
        add     %g6, %g7, %g6                   /* stats + d + ctx + pgsz */
        ldx     [%g6 + MMUSTAT_TICK], %g3
        add     %g3, %g5, %g3
        stx     %g3, [%g6 + MMUSTAT_TICK]
        ldx     [%g6 + MMUSTAT_HIT], %g3
        inc     %g3
        stx     %g3, [%g6 + MMUSTAT_HIT]
        retry


3:
        ! %g1 cpu struct
        ! %g2 --
        ! %g3 raddr
        ! %g4 page size
        ! %g5 --
        ! %g6 guest struct
        ! %g7 TTE ready for pa
        ! check for IO address
        ! branch back to 4b with pa in %g3
        ! must preserve %g1 and %g7
        RANGE_CHECK_IO(%g6, %g3, %g4, .dmmu_miss_io_found,
            .dmmu_miss_io_not_found, %g2, %g5)
.dmmu_miss_io_found:
        ba,a    4b
          nop

        ! %g1 cpu struct
        ! %g2 --
        ! %g3 raddr
        ! %g4 page size
        ! %g5 --
        ! %g6 guest struct
        ! %g7 TTE ready for pa
.dmmu_miss_io_not_found:
        ! Last chance - check the LDC mapin area
        ldx     [ %g6 + GUEST_LDC_MAPIN_BASERA ], %g5
        subcc   %g3, %g5, %g5
        bneg,pn %xcc, .dtsb_ra_range
          nop
        ldx     [ %g6 + GUEST_LDC_MAPIN_SIZE ], %g2
        subcc   %g5, %g2, %g0
        bneg,pt %xcc, ldc_dtsb_hit
          nop

                /* fall thru */

        ENTRY_NP(dtsb_miss)

        ! %g1 = CPU struct
        ! %g4 = context
.checkdpermmaps:
        brnz,pt %g4, .dtsbmiss          ! only context zero has perm mappings
          nop
        VCPU2GUEST_STRUCT(%g1, %g2)
        mov     GUEST_PERM_MAPPINGS_INCR*(NPERMMAPPINGS-1), %g3
        add     %g3, GUEST_PERM_MAPPINGS, %g3
        add     %g2, %g3, %g2
        mov     -(GUEST_PERM_MAPPINGS_INCR*(NPERMMAPPINGS-1)), %g3

        mov     MMU_TAG_ACCESS, %g4
        ldxa    [%g4]ASI_DMMU, %g4      /* tag access */
        set     (NCTXS - 1), %g5
        andn    %g4, %g5, %g4

1:
        ldda    [ %g2 + %g3 ] ASI_QUAD_LDD, %g6         ! Ld TTE (g7) + Tag (g6)

        ! Figure page size match mask
        ! FIXME: Could speed this by storing the mask ... but
        ! atomicity problems with storage. Other option is
        ! store pre-computed page size shift in tag bits 0-13
        brgez,pn %g7, 2f
        and     %g7, TTE_SZ_MASK, %g5
        add     %g5, %g5, %g1
        add     %g5, %g1, %g1           ! Mult size by 3
        add     %g1, 13, %g1            ! Add 13
        mov     1, %g5
        sllx    %g5, %g1, %g5           ! Compute bytes per page
        sub     %g5, 1, %g5             ! Page mask for TTE retrieved

        xor     %g6, %g4, %g6
        andncc  %g6, %g5, %g0           ! Check for tag match

        beq,pt %xcc, 3f
          nop

2:
        brlz,pt %g3, 1b
          add   %g3, GUEST_PERM_MAPPINGS_INCR, %g3

        VCPU_STRUCT(%g1)
        ba,pt   %xcc, .dtsbmiss
          mov   %g0, %g4

3:
        ! Found a matching entry - can we load it into the DTLB
        VCPU_STRUCT(%g1)
        add     %g2, %g3, %g2   ! Ptr to map entry

        ! Calculate index into perm bit set
        ldub    [%g1 + CPU_VID], %g3
        and     %g3, MAPPING_XWORD_MASK, %g4
        mov     1, %g5
        sllx    %g5, %g4, %g4   ! Bit in mask
        srlx    %g3, MAPPING_XWORD_SHIFT, %g3
        sllx    %g3, MAPPING_XWORD_BYTE_SHIFT_BITS, %g3
        add     %g2, %g3, %g2

        ldx     [%g2 + MAPPING_DCPUSET], %g3
        btst    %g3, %g4
        bz,pn   %xcc, .dtsbmiss
          mov   %g0, %g4

        ! Stuff entry - it's already been swizzled
        set     TLB_IN_4V_FORMAT, %g5   ! %g5 sun4v-style tte selection
        stxa    %g7, [%g5]ASI_DTLB_DATA_IN

        retry

.dtsbmiss:
        ldx     [%g1 + CPU_MMU_AREA], %g2
        brz,pn  %g2, watchdog_guest
        .empty

        ! %g1 is CPU pointer
        ! %g2 is MMU Fault Status Area
        ! %g4 is context (possibly shifted - still OK for zero test)
        /* if ctx == 0 and ctx0 set TSBs used, take slow trap */
        /* if ctx != 0 and ctxnon0 set TSBs used, take slow trap */
        mov     CPU_NTSBS_CTXN, %g7
        movrz   %g4, CPU_NTSBS_CTX0, %g7
        ldx     [%g1 + %g7], %g7
        brnz,pn %g7, .dslowmiss
        nop

.dfastmiss:
        /*
         * Update MMU_FAULT_AREA_DATA
         */
        mov     MMU_TAG_ACCESS, %g3
        ldxa    [%g3]ASI_DMMU, %g3      /* tag access */
        set     (NCTXS - 1), %g5
        andn    %g3, %g5, %g4
        and     %g3, %g5, %g5
        stx     %g4, [%g2 + MMU_FAULT_AREA_DADDR]
        stx     %g5, [%g2 + MMU_FAULT_AREA_DCTX]
        /* fast misses do not update MMU_FAULT_AREA_DFT with MMU_FT_FASTMISS */
        ! wrpr  %g0, TT_FAST_DMMU_MISS, %tt     /* already set */
        rdpr    %pstate, %g3
        or      %g3, PSTATE_PRIV, %g3
        wrpr    %g3, %pstate
        rdpr    %tba, %g3
        add     %g3, (TT_FAST_DMMU_MISS << TT_OFFSET_SHIFT), %g3
7:
        rdpr    %tl, %g2
        cmp     %g2, 1 /* trap happened at tl=0 */
        be,pt   %xcc, 1f
        .empty
        set     TRAPTABLE_SIZE, %g5

        cmp     %g2, MAXPTL
        bgu,pn  %xcc, watchdog_guest
        add     %g5, %g3, %g3

1:
        mov     HPSTATE_GUEST, %g5      ! set ENB bit
        jmp     %g3
        wrhpr   %g5, %hpstate

.dslowmiss:
        /*
         * Update MMU_FAULT_AREA_DATA
         */
        mov     MMU_TAG_ACCESS, %g3
        ldxa    [%g3]ASI_DMMU, %g3      /* tag access */
        set     (NCTXS - 1), %g5
        andn    %g3, %g5, %g4
        and     %g3, %g5, %g5
        stx     %g4, [%g2 + MMU_FAULT_AREA_DADDR]
        stx     %g5, [%g2 + MMU_FAULT_AREA_DCTX]
        mov     MMU_FT_MISS, %g4
        stx     %g4, [%g2 + MMU_FAULT_AREA_DFT]
        wrpr    %g0, TT_DMMU_MISS, %tt
        rdpr    %pstate, %g3
        or      %g3, PSTATE_PRIV, %g3
        wrpr    %g3, %pstate
        rdpr    %tba, %g3
        add     %g3, (TT_DMMU_MISS << TT_OFFSET_SHIFT), %g3
        ba,a    7b
        .empty

.dtsb_inv_pgsz:
        /* DAX with FT=Invalid Page Size (15), VA, CTX */
        ba,pt   %xcc, .dtsb_dax
        mov     MMU_FT_PAGESIZE, %g3

.dtsb_ra_range:
        /* DAX with FT=Invalid TSB Entry (16), VA, CTX */
        mov     MMU_FT_INVTSBENTRY, %g3
        /*FALLTHROUGH*/

.dtsb_dax:
        !! %g1 = cpup
        ldx     [%g1 + CPU_MMU_AREA], %g2
        brz,pn  %g2, watchdog_guest             ! Nothing we can do about this
        nop
        stx     %g3, [%g2 + MMU_FAULT_AREA_DFT]
        mov     MMU_TAG_ACCESS, %g3
        ldxa    [%g3]ASI_DMMU, %g3      /* tag access */
        set     (NCTXS - 1), %g5
        andn    %g3, %g5, %g4
        and     %g3, %g5, %g5
        stx     %g4, [%g2 + MMU_FAULT_AREA_DADDR]
        stx     %g5, [%g2 + MMU_FAULT_AREA_DCTX]
        REVECTOR(TT_DAX)
        SET_SIZE(dmmu_miss)

        /* %g2 contains guest's miss info pointer (hv phys addr) */
        ENTRY_NP(dmmu_prot)
        /*
         * TLB parity errors can cause normal MMU traps (N1 PRM
         * section 12.3.3 and 12.3.4).  Check here for an outstanding
         * parity error and have ue_err handle it instead.
         */
        ldxa    [%g0]ASI_SPARC_ERR_STATUS, %g1  ! SPARC err reg
        set     (SPARC_ESR_DMDU | SPARC_ESR_DMSU), %g3  ! is it a dmdu/dmsu err
        btst    %g3, %g1
        bnz     %xcc, ue_err                    ! err handler takes care of it
        /*
         * Update MMU_FAULT_AREA_DATA
         */
        mov     MMU_TAG_ACCESS, %g3
        ldxa    [%g3]ASI_DMMU, %g3      /* tag access */
        set     (NCTXS - 1), %g5
        andn    %g3, %g5, %g4
        and     %g3, %g5, %g5
        stx     %g4, [%g2 + MMU_FAULT_AREA_DADDR]
        stx     %g5, [%g2 + MMU_FAULT_AREA_DCTX]
        /* fast misses do not update MMU_FAULT_AREA_DFT with MMU_FT_FASTPROT */
        wrpr    %g0, TT_FAST_DMMU_PROT, %tt     /* already set? XXXQ */
        rdpr    %pstate, %g3
        or      %g3, PSTATE_PRIV, %g3
        wrpr    %g3, %pstate
        rdpr    %tba, %g3
        add     %g3, (TT_FAST_DMMU_PROT << TT_OFFSET_SHIFT), %g3

        rdpr    %tl, %g2
        cmp     %g2, 1 /* trap happened at tl=0 */
        be,pt   %xcc, 1f
        .empty
        set     TRAPTABLE_SIZE, %g5

        cmp     %g2, MAXPTL
        bgu,pn  %xcc, watchdog_guest
        add     %g5, %g3, %g3

1:
        mov     HPSTATE_GUEST, %g5      ! set ENB bit
        jmp     %g3
        wrhpr   %g5, %hpstate
        SET_SIZE(dmmu_prot)


/*
 * set all TSB base registers to dummy
 * call sequence:
 * in:
 *      %g7 return address
 *
 * volatile:
 *      %g1
 */
        ENTRY_NP(set_dummytsb_ctx0)
        ROOT_STRUCT(%g1)
        ldx     [%g1 + CONFIG_DUMMYTSB], %g1

        stxa    %g1, [%g0]ASI_DTSBBASE_CTX0_PS0
        stxa    %g1, [%g0]ASI_ITSBBASE_CTX0_PS0
        stxa    %g1, [%g0]ASI_DTSBBASE_CTX0_PS1
        stxa    %g1, [%g0]ASI_ITSBBASE_CTX0_PS1

        stxa    %g0, [%g0]ASI_DTSB_CONFIG_CTX0
        jmp     %g7 + 4
        stxa    %g0, [%g0]ASI_ITSB_CONFIG_CTX0
        SET_SIZE(set_dummytsb_ctx0)

        ENTRY_NP(set_dummytsb_ctxN)
        ROOT_STRUCT(%g1)
        ldx     [%g1 + CONFIG_DUMMYTSB], %g1

        stxa    %g1, [%g0]ASI_DTSBBASE_CTXN_PS0
        stxa    %g1, [%g0]ASI_ITSBBASE_CTXN_PS0
        stxa    %g1, [%g0]ASI_DTSBBASE_CTXN_PS1
        stxa    %g1, [%g0]ASI_ITSBBASE_CTXN_PS1

        stxa    %g0, [%g0]ASI_DTSB_CONFIG_CTXN
        jmp     %g7 + 4
        stxa    %g0, [%g0]ASI_ITSB_CONFIG_CTXN
        SET_SIZE(set_dummytsb_ctxN)


        ENTRY_NP(dmmu_err)
        /*
         * TLB parity errors can cause normal MMU traps (N1 PRM
         * section 12.3.3 and 12.3.4).  Check here for an outstanding
         * parity error and have ue_err handle it instead.
         */
        ldxa    [%g0]ASI_SPARC_ERR_STATUS, %g1  ! SPARC err reg
        set     (SPARC_ESR_DMDU | SPARC_ESR_DMSU), %g2  ! is it a dmdu/dmsu err
        btst    %g2, %g1
        bnz     %xcc, ue_err                    ! err handler takes care of it
        .empty

        VCPU_STRUCT(%g3)
        ldx     [%g3 + CPU_MMU_AREA], %g3
        brz,pn  %g3, watchdog_guest             ! Nothing we can do about this
        .empty
        ! %g3 - MMU_FAULT_AREA

        /*
         * Update MMU_FAULT_AREA_DATA
         */
        mov     MMU_SFAR, %g4
        ldxa    [%g4]ASI_DMMU, %g4
        stx     %g4, [%g3 + MMU_FAULT_AREA_DADDR]
        mov     MMU_SFSR, %g5
        ldxa    [%g5]ASI_DMMU, %g4 ! Capture SFSR
        stxa    %g0, [%g5]ASI_DMMU ! Clear SFSR

        mov     MMU_TAG_ACCESS, %g5
        ldxa    [%g5]ASI_DMMU, %g5
        set     (NCTXS - 1), %g6
        and     %g5, %g6, %g5
        stx     %g5, [%g3 + MMU_FAULT_AREA_DCTX]

        rdpr    %tt, %g1
        cmp     %g1, TT_DAX
        bne,pn  %xcc, 3f
        mov     MMU_FT_MULTIERR, %g6 ! unknown FT or multiple bits

        ! %g4 - sfsr
        srlx    %g4, MMU_SFSR_FT_SHIFT, %g5
        andcc   %g5, MMU_SFSR_FT_MASK, %g5
        bz,pn   %xcc, 2f
        nop
        ! %g5 - fault type
        ! %g6 - sun4v ft
        andncc  %g5, MMU_SFSR_FT_PRIV, %g0
        movz    %xcc, MMU_FT_PRIV, %g6 ! priv is only bit set
        andncc  %g5, MMU_SFSR_FT_SO, %g0
        movz    %xcc, MMU_FT_SO, %g6    ! so is only bit set
        andncc  %g5, MMU_SFSR_FT_ATOMICIO, %g0
        movz    %xcc, MMU_FT_NCATOMIC, %g6 ! atomicio is only bit set
        andncc  %g5, MMU_SFSR_FT_ASI, %g0
        movz    %xcc, MMU_FT_BADASI, %g6 ! badasi is only bit set
        andncc  %g5, MMU_SFSR_FT_NFO, %g0
        movz    %xcc, MMU_FT_NFO, %g6   ! nfo is only bit set
        andncc  %g5, (MMU_SFSR_FT_VARANGE | MMU_SFSR_FT_VARANGE2), %g0
        movz    %xcc, MMU_FT_VARANGE, %g6 ! varange are only bits set
2:      stx     %g6, [%g3 + MMU_FAULT_AREA_DFT]
3:      REVECTOR(%g1)
        SET_SIZE(dmmu_err)


        ENTRY_NP(immu_err)
        /*
         * TLB parity errors can cause normal MMU traps (N1 PRM
         * section 12.3.1.  Check here for an outstanding
         * parity error and have ue_err handle it instead.
         */
        ldxa    [%g0]ASI_SPARC_ERR_STATUS, %g1  ! SPARC err reg
        set     SPARC_ESR_IMDU, %g2             ! is it a imdu err
        btst    %g2, %g1
        bnz     %xcc, ue_err                    ! err handler takes care of it
        rdhpr   %htstate, %g1
        btst    HTSTATE_HPRIV, %g1
        bnz,pn  %xcc, badtrap
        .empty

        VCPU_STRUCT(%g3)
        ldx     [%g3 + CPU_MMU_AREA], %g3
        brz,pn  %g3, watchdog_guest     ! Nothing we can do about this
        nop

        ! %g3 - MMU_FAULT_AREA
        /* decode sfsr, update MMU_FAULT_AREA_INSTR */
        rdpr    %tpc, %g4
        stx     %g4, [%g3 + MMU_FAULT_AREA_IADDR]

        mov     MMU_PCONTEXT, %g5
        ldxa    [%g5]ASI_MMU, %g5
        movrnz  %g2, 0, %g5 ! primary ctx for TL=0, nucleus ctx for TL>0
        stx     %g5, [%g3 + MMU_FAULT_AREA_ICTX]

        ! %g6 - sun4v ft
        mov     MMU_FT_MULTIERR, %g6 ! unknown FT or multiple bits

        mov     MMU_SFSR, %g5
        ldxa    [%g5]ASI_IMMU, %g4 ! Capture SFSR
        stxa    %g0, [%g5]ASI_IMMU ! Clear SFSR
        ! %g4 - sfsr
        srlx    %g4, MMU_SFSR_FT_SHIFT, %g5
        andcc   %g5, MMU_SFSR_FT_MASK, %g5
        bz,pn   %xcc, 1f
        nop
        ! %g5 - fault type
        andncc  %g5, MMU_SFSR_FT_PRIV, %g0
        movz    %xcc, MMU_FT_PRIV, %g6 ! priv is only bit set
        andncc  %g5, (MMU_SFSR_FT_VARANGE | MMU_SFSR_FT_VARANGE2), %g0
        movz    %xcc, MMU_FT_VARANGE, %g6 ! varange are only bits set
1:      stx     %g6, [%g3 + MMU_FAULT_AREA_IFT]
        REVECTOR(TT_IAX)
        SET_SIZE(immu_err)