Initial commit of OpenSPARC T2 architecture model.
[OpenSPARC-T2-SAM] / sam-t2 / sam / cpus / vonk / n2 / lib / cpu / src / N2_TrieTlb.cc
// ========== Copyright Header Begin ==========================================
//
// OpenSPARC T2 Processor File: N2_TrieTlb.cc
// Copyright (c) 2006 Sun Microsystems, Inc. All Rights Reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES.
//
// The above named program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public
// License version 2 as published by the Free Software Foundation.
//
// The above named program is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public
// License along with this work; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
//
// ========== Copyright Header End ============================================
#include "N2_Tlb.h"
#include "N2_TrieTlb.h"
#include <new>
N2_TrieTlb::N2_TrieTlb( N2_Tlb* _tlb )/*{{{*/
:
tlb(_tlb),
fail_node1(0),
fail_node2(0),
fail_node3(0),
fail_page5(0),
fail_page3(0),
fail_page1(0),
free_node1(0),
free_node2(0),
free_node3(0),
free_page5(0),
free_page3(0),
free_page1(0)
{
fail_page1 = new Page1(0);
fail_page3 = new Page3(fail_page1);
fail_page5 = new Page5(fail_page3);
fail_node3 = new Node3(fail_page5);
fail_node2 = new Node2(fail_node3);
fail_node1 = new Node1(fail_node2);
fail_node1->fail_node();
fail_node2->fail_node();
fail_node3->fail_node();
fail_page5->fail_node();
fail_page3->fail_node();
fail_page1->fail_node();
for (uint_t p=0; p < PID_SIZE; p++)
{
real[p] = fail_node1;
for (uint_t c=0; c < CTX_SIZE; c++)
virt[p][c] = fail_node1;
}
}
/*}}}*/
N2_TrieTlb::N2_TrieTlb( N2_TrieTlb& trie, N2_Tlb* _tlb )/*{{{*/
:
tlb(_tlb),
fail_node1(trie.fail_node1),
fail_node2(trie.fail_node2),
fail_node3(trie.fail_node3),
fail_page5(trie.fail_page5),
fail_page3(trie.fail_page3),
fail_page1(trie.fail_page1),
free_node1(trie.free_node1),
free_node2(trie.free_node2),
free_node3(trie.free_node3),
free_page5(trie.free_page5),
free_page3(trie.free_page3),
free_page1(trie.free_page1)
{
trie.free_node1 = 0;
trie.free_node2 = 0;
trie.free_node3 = 0;
trie.free_page5 = 0;
trie.free_page3 = 0;
trie.free_page1 = 0;
for (uint_t p=0; p < PID_SIZE; p++)
{
real[p] = trie.copy(trie.real[p]);
for (uint_t c=0; c < CTX_SIZE; c++)
virt[p][c] = trie.copy(trie.virt[p][c]);
}
}
/*}}}*/
N2_TrieTlb::~N2_TrieTlb()/*{{{*/
{
for (uint_t p=0; p < PID_SIZE; p++)
demap_all(0,p);
while (free_node1)
{
Node1* help = free_node1;
free_node1 = free_node1->next;
delete help;
}
while (free_node2)
{
Node2* help = free_node2;
free_node2 = free_node2->next;
delete help;
}
while (free_node3)
{
Node3* help = free_node3;
free_node3 = free_node3->next;
delete help;
}
while (free_page5)
{
Page5* help = free_page5;
free_page5 = free_page5->next;
delete help;
}
while (free_page3)
{
Page3* help = free_page3;
free_page3 = free_page3->next;
delete help;
}
while (free_page1)
{
Page1* help = free_page1;
free_page1 = free_page1->next;
delete help;
}
}
/*}}}*/
N2_TrieTlb::Node1* N2_TrieTlb::copy( Node1* r_n1 )/*{{{*/
{
if (r_n1 == fail_node1)
return r_n1;
Node1* l_n1 = new_node1();
for (uint_t i0=0; i0 < Node1::SIZE; i0++)
{
Node2* r_n2 = r_n1->idx(i0);
if (r_n2 == fail_node2)
continue;
Node2* l_n2 = new_node2();
l_n1->insert(i0,l_n2);
for (uint_t i1=0; i1 < Node2::SIZE; i1++)
{
Node3* r_n3 = r_n2->idx(i1);
if (r_n3 == fail_node3)
continue;
Node3* l_n3 = new_node3();
l_n2->insert(i1,l_n3);
for (uint_t i2=0; i2 < Node3::SIZE; i2++)
{
Page5* r_p5 = r_n3->idx(i2);
if (r_p5 == fail_page5)
continue;
Page5* l_p5 = new_page5();
l_p5->tte = r_p5->tte;
l_n3->insert(i2,l_p5);
for (uint_t i3=0; i3 < Page5::SIZE; i3++)
{
Page3* r_p3 = r_p5->idx(i3);
if (r_p3 == fail_page3)
continue;
Page3* l_p3 = new_page3();
l_p3->tte = r_p3->tte;
l_p5->insert(i3,l_p3);
for (uint_t i4=0; i4 < Page3::SIZE; i4++)
{
Page1* r_p1 = r_p3->idx(i4);
if (r_p1 == fail_page1)
continue;
Page1* l_p1 = new_page1();
l_p1->tte = r_p1->tte;
l_p3->insert(i4,l_p1);
for (uint_t i5=0; i5 < Page1::SIZE; i5++)
{
SS_Tte* tte = r_p1->idx(i5);
if (tte)
l_p1->insert(i5,tte);
}
}
}
}
}
}
return l_n1;
}
/*}}}*/
void N2_TrieTlb::insert( SS_Strand* strand, SS_Tte* tte, SS_Tte* rem_tte )/*{{{*/
{
Node1* n1;
if (tte->is_virt())
{
n1 = virt[tte->pid()][tte->context()];
if (n1 == fail_node1)
{
n1 = new_node1();
virt[tte->pid()][tte->context()] = n1;
}
}
else if (tte->is_real())
{
n1 = real[tte->pid()];
if (n1 == fail_node1)
{
n1 = new_node1();
real[tte->pid()] = n1;
}
}
else
{
remove(rem_tte);
return;
}
// For the va we use the non aligned tag value, e.g we don't
// use tte->virt_page. We do this so we auto demap the
// correct smaller page(s).
SS_Vaddr va = tte->tag();
uint_t ps = tte->page_size();
Node2* n2 = n1->at(va);
Node3* n3 = n2->at(va);
Page5* p5 = n3->at(va);
Page3* p3 = p5->at(va);
Page1* p1 = p3->at(va);
Page5* p5_new = 0;
Page3* p3_new = 0;
Page1* p1_new = 0;
SS_Tte* p5_tte = p5->tte;
SS_Tte* p3_tte = p3->tte;
SS_Tte* p1_tte = p1->tte;
SS_Tte* p0_tte = p1->at(va);
// Inserts the TTE and auto demap the overlapping TTEs. E.g. demap
// p0_tte, p1_tte, p3_tte, and p5_tte is they are valid TTEs (!= 0).
// This insert and demap has to be (like in hardware) an atomic operation.
// We do this by creating a new path with the TTEs that are to be
// demapped removed and the new TTE inserted.
if (p0_tte)
{
// We have a full path: p5, p3, p1 and the have to demap a p0 TTE.
// Optimise for a simple case: replace p0_tte, as Solaris often
// changes readonly TTEs into readwrite TTEs: a result from fork().
// The unoptimised case creates a new path and updates the TTEs and
// inserts the path in the trie as the last operation.
if ((ps == 0) && (p1_tte == 0) && (p3_tte == 0) && (p5_tte == 0))
{
p1->at(va) = tte;
if (p0_tte != rem_tte)
{
tlb->clr_used(p0_tte->index);
tlb->invalidate_tte(strand,p0_tte);
remove(rem_tte);
}
return;
}
else
{
p5_new = new_page5(p5);
p3_new = new_page3(p3);
p1_new = new_page1(p1);
p5_new->at(va) = p3_new;
p3_new->at(va) = p1_new;
p5_new->tte = (ps == 5) ? tte : 0;
p3_new->tte = (ps == 3) ? tte : 0;
p1_new->tte = (ps == 1) ? tte : 0;
if (ps == 0)
p1_new->at(va) = tte;
else
p1_new->remove(va,0);
if (p1_new->is_empty())
{
p3_new->remove(va,fail_page1);
if (p3_new->is_empty())
{
p5_new->remove(va,fail_page3);
del_page3(p3_new);
p3_new = 0;
}
del_page1(p1_new);
p1_new = 0;
}
n3->at(va) = p5_new;
}
}
else if (p1_tte)
{
// Again we have a full path: p5, p3, p1 and the smallest TTE we have
// to demap is p1 TTE. Optimise for a trivial case. Otherwise create
// a new path with the TTEs updated and insert in the trie at the end.
if ((ps == 1) && (p3_tte == 0) && (p5_tte == 0))
{
p1->tte = tte;
if (p1_tte != rem_tte)
{
tlb->clr_used(p1_tte->index);
tlb->invalidate_tte(strand,p1_tte);
remove(rem_tte);
}
return;
}
else
{
p5_new = new_page5(p5);
p3_new = new_page3(p3);
p1_new = new_page1(p1);
p5_new->at(va) = p3_new;
p3_new->at(va) = p1_new;
p5_new->tte = (ps == 5) ? tte : 0;
p3_new->tte = (ps == 3) ? tte : 0;
p1_new->tte = (ps == 1) ? tte : 0;
if (ps == 0)
p1_new->insert(va,tte);
if (p1_new->is_empty())
{
p3_new->remove(va,fail_page1);
if (p3_new->is_empty())
{
p5_new->remove(va,fail_page3);
del_page3(p3_new);
p3_new = 0;
}
del_page1(p1_new);
p1_new = 0;
}
n3->at(va) = p5_new;
}
}
else if (p3_tte)
{
// We have at least a p5 and p3 node. Select the trivial case for
// optimisation. Else switch between possibly shrinking the path
// (ps >= 3) or extending the path.
if ((ps == 3) && (p5_tte == 0))
{
p3->tte = tte;
if (p3_tte != rem_tte)
{
tlb->clr_used(p3_tte->index);
tlb->invalidate_tte(strand,p3_tte);
remove(rem_tte);
}
return;
}
else if (ps >= 3)
{
p5_new = new_page5(p5);
p3_new = new_page3(p3);
p5_new->at(va) = p3_new;
p5_new->tte = (ps == 5) ? tte : 0;
p3_new->tte = (ps == 3) ? tte : 0;
if (p3_new->is_empty())
{
p5_new->remove(va,fail_page3);
del_page3(p3_new);
p3_new = 0;
}
n3->at(va) = p5_new;
}
else
{
Page1* p1_tmp;
p5_new = new_page5(p5);
p3_new = new_page3(p3);
p5_new->at(va) = p3_new;
if (p1 == fail_page1)
{
p1 = new_page1();
p3_new->insert(va,p1);
p1_tmp = p1;
}
else
{
p1_new = new_page1(p1);
p3_new->at(va) = p1_new;
p1_tmp = p1_new;
}
p5_new->tte = 0;
p3_new->tte = 0;
p1_tmp->tte = (ps == 1) ? tte : 0;
if (ps == 0)
p1_tmp->insert(va,tte);
n3->at(va) = p5_new;
}
}
else if (p5_tte)
{
// We have a p5 node. Optimise the simple case. Otherwise
// insert and create missing path.
if (ps == 5)
{
p5->tte = tte;
if (p5_tte != rem_tte)
{
tlb->clr_used(p5_tte->index);
tlb->invalidate_tte(strand,p5_tte);
remove(rem_tte);
}
return;
}
else
{
Page1* p1_tmp;
Page3* p3_tmp;
p5_new = new_page5(p5);
p5_new->tte = 0;
if (p3 == fail_page3)
{
p3 = new_page3();
p5_new->insert(va,p3);
p3_tmp = p3;
}
else
{
p3_new = new_page3(p3);
p5_new->at(va) = p3_new;
p3_tmp = p3_new;
}
if (ps == 3)
{
p3_tmp->tte = tte;
}
else
{
p3_tmp->tte = 0;
if (p1 == fail_page1)
{
p1 = new_page1();
p3_tmp->insert(va,p1);
p1_tmp = p1;
}
else
{
p1_new = new_page1(p1);
p3_new->at(va) = p1_new;
p1_tmp = p1_new;
}
if (ps == 1)
{
p1_tmp->tte = tte;
}
else
{
p1_tmp->tte = 0;
p1_tmp->insert(va,tte);
}
}
n3->at(va) = p5_new;
}
}
else if (p1 != fail_page1)
{
// We have no overlapping TTEs and the path p5, p3, p1 is there.
// This is a simple case, likely to happen often. We can just set
// the new TTE, remove the replaced TTE and return.
if (ps == 0)
p1->insert(va,tte);
else if (ps == 1)
p1->tte = tte;
else if (ps == 3)
p3->tte = tte;
else
p5->tte = tte;
remove(rem_tte);
return;
}
else
{
// The path is partial there. Create as much as we need, based on
// the page size (ps) of the TTE to be inserted.
if (p5 == fail_page5)
{
if (n3 == fail_node3)
{
if (n2 == fail_node2)
{
n2 = new_node2();
n1->insert(va,n2);
}
n3 = new_node3();
n2->insert(va,n3);
}
p5 = new_page5();
n3->insert(va,p5);
}
if (ps == 5)
{
p5->tte = tte;
}
else
{
if (p3 == fail_page3)
{
p3 = new_page3();
p5->insert(va,p3);
}
if (ps == 3)
{
p3->tte = tte;
}
else
{
if (p1 == fail_page1)
{
p1 = new_page1();
p3->insert(va,p1);
}
if (ps == 1)
p1->tte = tte;
else
p1->insert(va,tte);
}
}
}
// Now invalidate all the TTEs that got autodemapped.
// However, don't invalidate the TTE that is selected
// by the replacement algorithm to make place for the
// new TTE.
if (p0_tte == rem_tte)
rem_tte = 0;
else if (p0_tte)
{
tlb->clr_used(p0_tte->index);
tlb->invalidate_tte(strand,p0_tte);
}
if (p1_tte == rem_tte)
rem_tte = 0;
else if (p1_tte)
{
tlb->clr_used(p1_tte->index);
tlb->invalidate_tte(strand,p1_tte);
}
if (p3_tte == rem_tte)
rem_tte = 0;
else if (p3_tte)
{
tlb->clr_used(p3_tte->index);
tlb->invalidate_tte(strand,p3_tte);
}
if (p5_tte == rem_tte)
rem_tte = 0;
else if (p5_tte)
{
tlb->clr_used(p5_tte->index);
tlb->invalidate_tte(strand,p5_tte);
}
// Free up the pages that got copied to make auto-demap
// an atomic operation. Note we don;t clear the tte pointers
// and we don't demolish the p5 to p3 to p1 path. We
// do this so that lookup can detect multi hits properly.
if (p5_new)
del_page5(p5);
if (p3_new)
del_page3(p3);
if (p1_new)
del_page1(p1);
remove(rem_tte);
}
/*}}}*/
void N2_TrieTlb::remove( SS_Tte* tte )/*{{{*/
{
Node1* n1;
if (tte == 0)
return;
else if (tte->is_virt())
n1 = virt[tte->pid()][tte->context()];
else if (tte->is_real())
n1 = real[tte->pid()];
else
return;
SS_Vaddr va = tte->virt_page;
uint_t ps = tte->page_size();
Node2* n2 = n1->at(va);
Node3* n3 = n2->at(va);
Page5* p5 = n3->at(va);
Page3* p3 = p5->at(va);
Page1* p1 = p3->at(va);
assert(n1 != fail_node1);
assert(n2 != fail_node2);
assert(n3 != fail_node3);
assert(p5 != fail_page5);
if (ps == 5)
{
assert((p5->tte == tte) || (p5->tte == 0));
p5->tte = 0;
}
else
{
assert(p3 != fail_page3);
if (ps == 3)
{
assert((p3->tte == tte) || (p3->tte == 0));
p3->tte = 0;
}
else
{
assert(p1 != fail_page1);
if (ps == 1)
{
assert((p1->tte == tte) || (p1->tte == 0));
p1->tte = 0;
}
else
{
assert(ps == 0);
assert((p1->at(va) == tte) || (p1->at(va) == 0));
p1->remove(va,0);
}
if (p1->is_empty())
{
p3->remove(va,fail_page1);
del_page1(p1);
}
}
if (p3->is_empty())
{
p5->remove(va,fail_page3);
del_page3(p3);
}
}
if (p5->is_empty())
{
n3->remove(va,fail_page5);
del_page5(p5);
if (n3->is_empty())
{
n2->remove(va,fail_node3);
del_node3(n3);
if (n2->is_empty())
{
n1->remove(va,fail_node2);
del_node2(n2);
if (n1->is_empty())
{
if (tte->is_real())
real[tte->pid()] = fail_node1;
else
virt[tte->pid()][tte->context()] = fail_node1;
del_node1(n1);
}
}
}
}
}
/*}}}*/
// demap_trie() remove TTEs from a Trie in an atomic fashion.
//
// Lookup searches with va from largest TTE to smaller TTE.
// When demapping we remove TTEs from smaller to largest.
// This way the TTE found by lookup is always the largest or
// one that persist after the demap, e.g. it makes the demap
// appear atomic.
bool N2_TrieTlb::demap_trie( SS_Strand* strand, Node1* n1, SS_Vaddr va )/*{{{*/
{
Node2* n2 = n1->at(va);
Node3* n3 = n2->at(va);
Page5* p5 = n3->at(va);
Page3* p3 = p5->at(va);
Page1* p1 = p3->at(va);
SS_Tte* tte;
if ((tte = p1->at(va)))
{
p1->remove(va,0);
tlb->clr_used(tte->index);
tlb->invalidate_tte(strand,tte);
}
if ((tte = p1->tte))
{
p1->tte = 0;
tlb->clr_used(tte->index);
tlb->invalidate_tte(strand,tte);
}
if (p1->is_empty())
{
p3->remove(va,fail_page1);
del_page1(p1);
}
if ((tte = p3->tte))
{
p3->tte = 0;
tlb->clr_used(tte->index);
tlb->invalidate_tte(strand,tte);
}
if (p3->is_empty())
{
p5->remove(va,fail_page3);
del_page3(p3);
}
if ((tte = p5->tte))
{
p5->tte = 0;
tlb->clr_used(tte->index);
tlb->invalidate_tte(strand,tte);
}
if (p5->is_empty())
{
n3->remove(va,fail_page5);
del_page5(p5);
}
if (n3->is_empty())
{
n2->remove(va,fail_node3);
del_node3(n3);
}
if (n2->is_empty())
{
n1->remove(va,fail_node2);
del_node2(n2);
}
if (n1->is_empty())
{
del_node1(n1);
return true;
}
return false;
}
/*}}}*/
void N2_TrieTlb::demap_trie( SS_Strand* strand, Node1* n1 )/*{{{*/
{
assert(n1 != fail_node1);
if (!n1->is_empty_trie())
{
for (uint_t i=0; i < Node1::SIZE; i++)
{
Node2* n2 = n1->idx(i);
if (n2 == fail_node2)
continue;
n1->remove(i,fail_node2);
demap_trie(strand,n2);
}
}
del_node1(n1);
}
/*}}}*/
void N2_TrieTlb::demap_trie( SS_Strand* strand, Node2* n2 )/*{{{*/
{
assert(n2 != fail_node2);
if (!n2->is_empty_trie())
{
for (uint_t i=0; i < Node2::SIZE; i++)
{
Node3* n3 = n2->idx(i);
if (n3 == fail_node3)
continue;
n2->remove(i,fail_node3);
demap_trie(strand,n3);
}
}
del_node2(n2);
}
/*}}}*/
void N2_TrieTlb::demap_trie( SS_Strand* strand, Node3* n3 )/*{{{*/
{
assert(n3 != fail_node3);
if (!n3->is_empty_trie())
{
for (uint_t i=0; i < Node3::SIZE; i++)
{
Page5* p5 = n3->idx(i);
if (p5 == fail_page5)
continue;
n3->remove(i,fail_page5);
demap_trie(strand,p5);
}
}
del_node3(n3);
}
/*}}}*/
void N2_TrieTlb::demap_trie( SS_Strand* strand, Page5* p5 )/*{{{*/
{
assert(p5 != fail_page5);
SS_Tte* tte;
if (!p5->is_empty_trie())
{
for (uint_t i=0; i < Page5::SIZE; i++)
{
Page3* p3 = p5->idx(i);
if (p3 == fail_page3)
continue;
p5->remove(i,fail_page3);
demap_trie(strand,p3);
}
}
if ((tte = p5->tte))
{
p5->tte = 0;
if (strand)
{
tlb->clr_used(tte->index);
tlb->invalidate_tte(strand,tte);
}
}
del_page5(p5);
}
/*}}}*/
void N2_TrieTlb::demap_trie( SS_Strand* strand, Page3* p3 )/*{{{*/
{
assert(p3 != fail_page3);
SS_Tte* tte;
if (!p3->is_empty_trie())
{
for (uint_t i=0; i < Page3::SIZE; i++)
{
Page1* p1 = p3->idx(i);
if (p1 == fail_page1)
continue;
p3->remove(i,fail_page1);
demap_trie(strand,p1);
}
}
if ((tte = p3->tte))
{
p3->tte = 0;
if (strand)
{
tlb->clr_used(tte->index);
tlb->invalidate_tte(strand,tte);
}
}
del_page3(p3);
}
/*}}}*/
void N2_TrieTlb::demap_trie( SS_Strand* strand, Page1* p1 )/*{{{*/
{
assert(p1 != fail_page1);
SS_Tte* tte;
if (!p1->is_empty_trie())
{
for (uint_t i=0; i < Page1::SIZE; i++)
{
if ((tte = p1->idx(i)))
{
p1->remove(i,0);
if (strand)
{
tlb->clr_used(tte->index);
tlb->invalidate_tte(strand,tte);
}
}
}
}
if ((tte = p1->tte))
{
p1->tte = 0;
if (strand)
{
tlb->clr_used(tte->index);
tlb->invalidate_tte(strand,tte);
}
}
del_page1(p1);
}
/*}}}*/
void N2_TrieTlb::demap_virt( SS_Strand* strand, uint_t pid, uint_t ctx, SS_Vaddr va )/*{{{*/
{
Node1* n1 = virt[pid][ctx];
if (demap_trie(strand,n1,va))
virt[pid][ctx] = fail_node1;
}
/*}}}*/
void N2_TrieTlb::demap_virt( SS_Strand* strand, uint_t pid, uint_t ctx )/*{{{*/
{
Node1* n1 = virt[pid][ctx];
virt[pid][ctx] = fail_node1;
if (n1 != fail_node1)
demap_trie(strand,n1);
}
/*}}}*/
void N2_TrieTlb::demap_virt( SS_Strand* strand, uint_t pid )/*{{{*/
{
for (uint_t ctx=0; ctx < CTX_SIZE; ctx++)
demap_virt(strand,pid,ctx);
}
/*}}}*/
void N2_TrieTlb::demap_real( SS_Strand* strand, uint_t pid, SS_Vaddr ra )/*{{{*/
{
Node1* n1 = real[pid];
if (demap_trie(strand,n1,ra))
real[pid] = fail_node1;
}
/*}}}*/
void N2_TrieTlb::demap_real( SS_Strand* strand, uint_t pid )/*{{{*/
{
Node1* n1 = real[pid];
real[pid] = fail_node1;
if (n1 != fail_node1)
demap_trie(strand,n1);
}
/*}}}*/
void N2_TrieTlb::demap_all( SS_Strand* strand, uint_t pid )/*{{{*/
{
demap_virt(strand,pid);
demap_real(strand,pid);
}
/*}}}*/