/*
* <---------------- ravail --------------------->
* <-- diff ------> <--- avail ----------------->
* <---- len ----------->
* | Previous lines | line being parsed nl extra |
* ^
* b
*
*/
static ssize_t
next_line(struct archive_read *a,
const char **b, ssize_t *avail, ssize_t *ravail, ssize_t *nl)
{
ssize_t len;
int quit;
quit = 0;
if (*avail == 0) {
*nl = 0;
len = 0;
} else
len = get_line_size(*b, *avail, nl);
/*
* Read bytes more while it does not reach the end of line.
*/
while (*nl == 0 && len == *avail && !quit) {
ssize_t diff = *ravail - *avail;
size_t nbytes_req = (*ravail+1023) & ~1023U;
ssize_t tested;
/*
* Place an arbitrary limit on the line length.
* mtree is almost free-form input and without line length limits,
* it can consume a lot of memory.
*/
if (len >= MAX_LINE_LEN)
return (-1);
/* Increase reading bytes if it is not enough to at least
* new two lines. */
if (nbytes_req < (size_t)*ravail + 160)
nbytes_req <<= 1;
*b = __archive_read_ahead(a, nbytes_req, avail);
if (*b == NULL) {
if (*ravail >= *avail)
return (0);
/* Reading bytes reaches the end of file. */
*b = __archive_read_ahead(a, *avail, avail);
quit = 1;
}
*ravail = *avail;
*b += diff;
*avail -= diff;
tested = len;/* Skip some bytes we already determinated. */
len = get_line_size(*b + len, *avail - len, nl);
if (len >= 0)
len += tested;
}
return (len);
}
void
Cache :: line_evict(Line *line)
{
NVLOG1("%s\tline_evict addr 0x%lx data 0x%lx\n", this->_name.c_str(), line->addr, (Addr)line->pdata);
// evict in all child caches
for (size_t child_i = 0; child_i<_children.size() && line->sharers>0; child_i++) {
if (!bit(line->sharers, child_i)) continue; // a child is not a sharer, so continue
Cache *child = dynamic_cast<Cache *>(_children[child_i]); assert(child!=NULL);
for (Addr line_addr_iter=line->addr; line_addr_iter<line->addr+get_line_size(); line_addr_iter += child->get_line_size()) {
Line *child_line = child->addr2line_internal(line_addr_iter);
if (child_line == NULL) continue;
child->line_evict(child_line);
}
}
#ifdef HAS_HTM
if (pprocessor) {
pprocessor->cb_line_evicted(line);
}
#endif
if (!_parent_cache) {
// notify the processor of the line removal
#ifdef HAS_HTM
assert(pprocessor);
pprocessor->cb_line_evicted(line);
#endif
}
this->line_data_writeback(line); // check if there is any data to writeback
free(line->pdata);
line->pdata = NULL;
// remove in this cache as well
this->line_rm(line);
}
void
Cache :: line_writer_to_sharer(Line *line, size_t &latency, bool children_only)
{
// make this cache (and child caches)
// only sharers of the line (downgrade from a writer)
NVLOG1("%s\tline_writer_to_sharer 0x%lx +%ld cycles\n", this->_name.c_str(), line->addr, _hit_latency);
latency += _hit_latency;
this->stats.writebacks_inc();
for (size_t child_i = 0; child_i<_children.size(); child_i++) {
if (!bit(line->sharers, child_i)) continue; // a child is not a sharer, so continue
Cache *child = dynamic_cast<Cache *>(_children[child_i]); assert(child!=NULL);
for (Addr line_addr_iter=line->addr; line_addr_iter<line->addr+get_line_size(); line_addr_iter += child->get_line_size())
{
if (line_addr_iter!=line->addr) {
// also measure the latency for other line segments
NVLOG1("%s\tline_writer_to_sharer 0x%lx +%ld cycles\n", this->_name.c_str(), line_addr_iter, _hit_latency);
latency += _hit_latency; // response from child to parent
this->stats.writebacks_inc();
}
Line *child_line = child->addr2line_internal(line_addr_iter);
if (!child_line) continue;
child->line_writer_to_sharer(child_line, latency);
}
}
if (!children_only) {
this->line_data_writeback(line);
line->state &= ~(LINE_MOD | LINE_EXC);
line->state |= LINE_SHR;
}
NVLOG1("%s\t0x%lx\t new state %s sharers %lx\n", _name.c_str(), line->addr, state2str(line->state).c_str(), line->sharers );
}
bool
Cache :: line_make_owner_in_child_caches(Line *line, unsigned child_index)
{
NVLOG1("%s\tline_make_owner_in_child_caches 0x%lx sharers %lx caller %d\n", this->_name.c_str(), line->addr, line->sharers, child_index);
uint8_t __attribute__((unused)) line_state_orig = line->state;
uint64_t __attribute__((unused)) line_sharers_orig = line->sharers;
// evict the line in all but the requesting child cache
for (size_t child_i = 0; child_i<_children.size() && line->sharers>0; child_i++) {
Cache *child = dynamic_cast<Cache *>(_children[child_i]); assert(child!=NULL);
// NVLOG1("%s\tchecking for 0x%lx %lx 1\n", child->_name.c_str(), line->addr, line->sharers);
if (!bit(line->sharers, child_i)) continue; // a child is not a sharer, so continue
// NVLOG1("%s\tchecking for 0x%lx %lx 2\n", child->_name.c_str(), line->addr, line->sharers);
if (child_i == child_index) continue; // skip the child cache that called us
// NVLOG1("%s\tchecking for 0x%lx %lx 3\n", child->_name.c_str(), line->addr, line->sharers);
for (Addr line_addr_iter=line->addr; line_addr_iter<line->addr+get_line_size(); line_addr_iter += child->get_line_size())
{
NVLOG1("%s\tis line sharer, checking segment 0x%lx\n", child->_name.c_str(), line_addr_iter);
Line *child_line = child->addr2line_internal(line_addr_iter);
if (!child_line) continue;
NVLOG1("%s\thas segment 0x%lx, evicting\n", child->_name.c_str(), line_addr_iter);
child->line_evict(child_line);
}
}
line->sharers = 0;
setbit(line->sharers, child_index);
NVLOG1("%s\tline 0x%lx\t state %s->%s sharers 0x%lx->0x%lx\n", _name.c_str(), line->addr, state2str(line_state_orig).c_str(), state2str(line->state).c_str(), line_sharers_orig, line->sharers);
return true;
}
static ssize_t
next_line(struct archive_read *a,
const char **b, ssize_t *avail, ssize_t *ravail, ssize_t *nl)
{
ssize_t len;
int quit;
quit = 0;
if (*avail == 0) {
*nl = 0;
len = 0;
} else
len = get_line_size(*b, *avail, nl);
/*
* Read bytes more while it does not reach the end of line.
*/
while (*nl == 0 && len == *avail && !quit) {
ssize_t diff = *ravail - *avail;
size_t nbytes_req = (*ravail+1023) & ~1023U;
ssize_t tested;
/* Increase reading bytes if it is not enough to at least
* new two lines. */
if (nbytes_req < (size_t)*ravail + 160)
nbytes_req <<= 1;
*b = __archive_read_ahead(a, nbytes_req, avail);
if (*b == NULL) {
if (*ravail >= *avail)
return (0);
/* Reading bytes reaches the end of file. */
*b = __archive_read_ahead(a, *avail, avail);
quit = 1;
}
*ravail = *avail;
*b += diff;
*avail -= diff;
tested = len;/* Skip some bytes we already determinated. */
len = get_line_size(*b, *avail, nl);
if (len >= 0)
len += tested;
}
return (len);
}
Line *
Cache :: addr2line_internal(const Addr addr, const bool search_reverse)
{
Addr line_addr = floor(addr, get_line_size());
size_t direct_entry = this->addr2directentry(line_addr);
if (search_reverse) {
Line *line = _entries[direct_entry].get_no_reorder_reverse(line_addr);
return line;
} else {
Line *line = _entries[direct_entry].get_no_reorder(line_addr);
return line;
}
}
void read_file_into_matrix(char * file, char * format, int elem_size,
void * data, int * max_rows, int * max_cols)
{
int r, c, rows, cols;
FILE * f;
rows = getNumberLines(file);
cols = get_line_size(file);
if ((rows >= (*max_rows)) || (cols >= (*max_cols)))
{
printf("Needs size rows = %i, cols = %i\n", rows + 1, cols + 1);
exit(-1);
}
f = fopen(file, "rt");
if (f == NULL)
{
printf("Cannot open file %s for reading\n", file);
exit(-1);
}
for (r = 0; r < rows; r++)
for (c = 0; c < cols; c++)
{
fscanf(f, format, ((char *) data) + (cols * r + c) * elem_size);
if (feof(f))
{
printf("feof reached at row = %i, column = %i\n", r, c);
exit(0);
}
}
fclose(f);
(*max_rows) = rows;
(*max_cols) = cols;
}
void
Cache :: line_rm_recursive(Addr addr)
{
Line *line = addr2line_internal(addr);
if (line==NULL) return;
NVLOG1("%s\tline_rm_recursive addr 0x%lx\n", this->_name.c_str(), addr);
for (size_t child_i = 0; child_i<_children.size() && line->sharers>0; child_i++) {
if (!bit(line->sharers, child_i)) continue; // a child is not a sharer, so continue
Cache *child = dynamic_cast<Cache *>(_children[child_i]); assert(child!=NULL);
for (Addr line_addr_iter=line->addr; line_addr_iter<line->addr+get_line_size(); line_addr_iter += child->get_line_size()) {
child->line_rm_recursive(line_addr_iter);
}
}
#ifdef HAS_HTM
if (pprocessor) {
pprocessor->cb_line_evicted(line);
}
#endif
free(line->pdata);
line->pdata = NULL;
// remove in this cache as well
this->line_rm(line);
}
void
Cache :: line_data_writeback(Line *line)
{
if (line->pdata == NULL) return;
assert(! ((line->state & (LINE_MOD | LINE_EXC)) && (line->state & LINE_TXW)) );
if (!(line->state & (LINE_MOD | LINE_TXW))) return;
if (_parent_cache && line->parent_line)
{
NVLOG1("%s\tline_data_writeback to %s 0x%lx data 0x%lx -> 0x%lx\n", this->_name.c_str(), _parent_cache->_name.c_str(), line->addr, (Addr)line->pdata, (Addr)line->parent_line->pdata);
assert(line->parent_line->pdata != NULL);
memcpy(line->parent_line->pdata+(line->addr - line->parent_line->addr), line->pdata, get_line_size());
if (line->state & LINE_MOD) { line->parent_line->state |= LINE_MOD; } // propagate modified state
NVLOG1("%s\t0x%lx\t new state %s sharers %lx\n", _parent_cache->_name.c_str(), line->parent_line->addr, state2str(line->parent_line->state).c_str(), line->parent_line->sharers );
}
else
{
NVLOG1("%s\tline_data_writeback to main_mem 0x%lx data <- 0x%lx\n", this->_name.c_str(), line->addr, (Addr)line->pdata);
// write the line data to the memory
// Fault fault = rw_array_silent(line->addr, get_line_size(), line->pdata, true);
// assert(fault == NoFault);
}
}
void
Cache :: line_get(Addr addr, uint8_t line_state_req, size_t &latency, uint8_t *&pdata)
{
//////////////////////////////////////////////////////////////////
//
// proudly serving cache requests
// since september 2008
//
//////////////////////////////////////////////////////////////////
//
// line_state_req(uest) values:
// LINE_SHR - request a read
// LINE_EXC - request an exclusive access (invalidate other readers)
// LINE_MOD - perform a write (invalidate other readers and mark line as dirty)
bool set_overflow = false;
Line *overflow_line = NULL;
// this adds a line (if it's not already in the cache) but with LINE_INV state
Line *line = addr2line(addr, set_overflow, overflow_line);
// if some line has been replaced, get the value and evict/invalidate the same line and its parts
// in all child caches
if (set_overflow) {
NVLOG1("%s\tline_get overflow 0x%lx\n", _name.c_str(), overflow_line->addr);
this->line_evict(overflow_line);
}
uint8_t __attribute__((unused)) line_state_orig = line->state;
uint64_t __attribute__((unused)) line_sharers_orig = line->sharers;
bool hit = true;
size_t old_ticks = latency;
if (line->state & LINE_MOD)
{
switch (line_state_req) {
case LINE_MOD:
case LINE_EXC:
case LINE_SHR:
break;
default:
assert(!"invalid line_state request!");
}
}
else if (line->state & LINE_EXC)
{ // line is EXCLUSIVE when it is not modified and there is no other line sharer
// TODO update the functionality to reflect this;
// first line sharer should automatically get exclusive access;
// this exclusive access should be reduced to shared when another core requests line copy (for read)
switch (line_state_req) {
case LINE_MOD:
if (_is_writeback_cache || !_parent_cache) {
// mark line as dirty (for writeback)
line->state |= line_state_req;
} else {
// write latency should include writing to the parent
_parent->line_get_intercache(addr, line_state_req, latency, _index_in_parent, line->parent_line);
// TODO should also write data to the parent cache
hit = false;
break;
}
case LINE_EXC:
case LINE_SHR:
break;
default:
assert(!"invalid line_state request!");
}
}
else if (line->state & LINE_SHR)
{
switch (line_state_req) {
case LINE_MOD:
case LINE_EXC:
if (_parent_cache) {
if (_is_writeback_cache) {
_parent->line_get_intercache(addr, LINE_EXC, latency, _index_in_parent, line->parent_line);
} else {
_parent->line_get_intercache(addr, line_state_req, latency, _index_in_parent, line->parent_line);
}
hit = false;
}
line->state |= line_state_req;
break;
case LINE_SHR:
break;
default:
assert(!"invalid line_state request!");
}
}
else if (line->state == LINE_INV)
{
hit = false;
switch (line_state_req) {
case LINE_TXW:
case LINE_TXR:
_parent->line_get_intercache(addr, LINE_SHR, latency, _index_in_parent, line->parent_line);
line->state |= line->parent_line->state & ( LINE_TXR | LINE_TXW);
line->state |= (LINE_SHR | line_state_req);
break;
case LINE_MOD:
case LINE_EXC:
if (_is_writeback_cache) {
_parent->line_get_intercache(addr, LINE_EXC, latency, _index_in_parent, line->parent_line);
} else {
_parent->line_get_intercache(addr, line_state_req, latency, _index_in_parent, line->parent_line);
}
line->state |= line_state_req;
break;
case LINE_SHR:
_parent->line_get_intercache(addr, LINE_SHR, latency, _index_in_parent, line->parent_line);
line->state |= line_state_req;
break;
default:
assert(!"invalid line_state request!");
}
}
else
{
NVLOG_ERROR("invalid current line_state %x\n", line->state);
assert(false && "invalid current line_state!");
}
if (line->pdata == NULL) {
line->pdata = (uint8_t*)malloc(get_line_size());
if (_parent_cache && line->parent_line) { // if data found in parent cache
NVLOG1("%s\tline_get data copy from %s 0x%lx data 0x%lx -> 0x%lx\n", this->_name.c_str(), _parent_cache->_name.c_str(), line->addr, (Addr)line->parent_line->pdata, (Addr)line->pdata);
memcpy(line->pdata, line->parent_line->pdata+(line->addr - line->parent_line->addr), get_line_size());
} else {
// get the data from the memory
NVLOG1("%s\tline_get data copy from memory 0x%lx data -> 0x%lx\n", this->_name.c_str(), line->addr, (Addr)line->pdata);
//Fault fault = rw_array_silent(line->addr, get_line_size(), line->pdata, false/*READ*/);
//assert(fault == NoFault);
}
}
pdata = line->pdata;
latency += _hit_latency;
// update statistics
if (hit) {
this->stats.hits_inc();
} else {
this->stats.misses_inc();
if (line_state_req & LINE_MOD || line_state_req & LINE_EXC)
{ this->stats.misses_st_inc(); }
else
{ this->stats.misses_ld_inc(); }
}
this->stats.ticks_inc(latency - old_ticks);
NVLOG1("%s\tline_get 0x%lx\t state %s->%s sharers 0x%lx->0x%lx\n", _name.c_str(), line->addr, state2str(line_state_orig).c_str(), state2str(line->state).c_str(), line_sharers_orig, line->sharers);
assert(! ((line->state & (LINE_MOD | LINE_EXC)) && (line->state & LINE_TXW)) );
}
void
Cache :: line_get_intercache(Addr addr, uint8_t line_state_req, size_t &latency, unsigned child_index, Line *&parent_line)
{
//////////////////////////////////////////////////////////////////////
// serving line relocations inside the cache hierarchy
//////////////////////////////////////////////////////////////////////
bool set_overflow = false;
Line *overflow_line = NULL;
// this adds a line (if it's not already in the cache) but with LINE_INV state
Line *line = addr2line(addr, set_overflow, overflow_line);
if (set_overflow) {
// if some line has been replaced, get the value and invalidate the same line and its parts
// in all child caches
NVLOG1("%s\tline_get overflow 0x%lx\n", _name.c_str(), overflow_line->addr);
this->line_evict(overflow_line);
}
uint8_t __attribute__((unused)) line_state_orig = line->state;
uint64_t __attribute__((unused)) line_sharers_orig = line->sharers;
bool hit = true;
size_t old_ticks = latency;
if (line->state & (LINE_MOD | LINE_EXC))
{
// this directory already owns a line (exclusively)
// just in case we'll invalidate the line in
// all other child-caches
switch (line_state_req) {
case LINE_MOD:
if (line->sharers != (1ULL<<child_index)) {
this->line_make_owner_in_child_caches(line, child_index);
setbit(line->sharers, child_index);
}
if (_is_writeback_cache || !_parent_cache) {
line->state |= line_state_req;
} else {
_parent->line_get_intercache(addr, line_state_req, latency, _index_in_parent, line->parent_line);
hit = false;
break;
}
break;
case LINE_EXC:
// this processor wants to have an exclusive copy
// (and it didn't have it until now, as we got an intercache request)
if (line->sharers != (1ULL<<child_index)) {
this->line_make_owner_in_child_caches(line, child_index);
setbit(line->sharers, child_index);
}
break;
case LINE_SHR:
if (line->sharers != (1ULL<<child_index)) {
this->line_writer_to_sharer(line, latency);
// and do not write the data up in the memory hierarchy
setbit(line->sharers, child_index);
}
break;
default:
assert(!"invalid line_state request!");
}
//assert(line_state_req == LINE_SHR || line_state_req == LINE_EXC || line_state_req == LINE_MOD);
line->state |= line_state_req; // we might also reduce from writer to LINE_SHR in this cache
}
else if (line->state & LINE_SHR)
{
switch (line_state_req) {
case LINE_MOD:
case LINE_EXC:
this->line_make_owner_in_child_caches(line, child_index);
setbit(line->sharers, child_index);
if (_parent_cache) {
if (_is_writeback_cache) {
_parent->line_get_intercache(addr, LINE_EXC, latency, _index_in_parent, line->parent_line);
} else {
_parent->line_get_intercache(addr, line_state_req, latency, _index_in_parent, line->parent_line);
}
hit = false;
}
line->state |= line_state_req;
break;
case LINE_SHR:
setbit(line->sharers, child_index);
break;
default:
assert(!"invalid line_state request!");
}
}
else if (line->state == LINE_INV)
{
hit = false;
switch (line_state_req) {
case LINE_MOD:
case LINE_EXC:
if (_is_writeback_cache) {
_parent->line_get_intercache(addr, LINE_EXC, latency, _index_in_parent, line->parent_line);
} else {
_parent->line_get_intercache(addr, line_state_req, latency, _index_in_parent, line->parent_line);
}
if (line->parent_line) { line->state = line->parent_line->state; }
line->state |= line_state_req;
setbit(line->sharers, child_index);
break;
case LINE_SHR:
_parent->line_get_intercache(addr, LINE_SHR, latency, _index_in_parent, line->parent_line);
if (line->parent_line) { line->state = line->parent_line->state; }
line->state |= line_state_req;
setbit(line->sharers, child_index);
break;
default:
assert(!"invalid line_state request!");
}
}
else
{
NVLOG_ERROR("invalid current line_state %x\n", line->state);
assert(false && "invalid current line_state!");
}
if (line->pdata == NULL) {
line->pdata = (uint8_t*)malloc(get_line_size());
if (_parent_cache && line->parent_line) { // if data found in any parent cache
NVLOG1("%s\tline_get data copy from %s 0x%lx data 0x%lx -> 0x%lx\n", this->_name.c_str(), _parent_cache->_name.c_str(), line->addr, (Addr)line->parent_line->pdata, (Addr)line->pdata);
memcpy(line->pdata, line->parent_line->pdata+(line->addr - line->parent_line->addr), get_line_size());
} else {
// get the data from the memory
NVLOG1("%s\tline_get data copy from memory 0x%lx data 0x%lx\n", this->_name.c_str(), line->addr, (Addr)line->pdata);
// Fault fault = rw_array_silent(line->addr, get_line_size(), line->pdata, false/*READ*/);
// assert(fault == NoFault);
}
}
parent_line = line;
latency += _hit_latency;
// update statistics
if (hit) {
this->stats.hits_inc();
} else {
this->stats.misses_inc();
if (line_state_req & LINE_MOD || line_state_req & LINE_EXC)
{ this->stats.misses_st_inc(); }
else
{ this->stats.misses_ld_inc(); }
}
this->stats.ticks_inc(latency - old_ticks);
NVLOG1("%s\tline_get 0x%lx\t state %s->%s sharers 0x%lx->0x%lx\n", _name.c_str(), line->addr, state2str(line_state_orig).c_str(), state2str(line->state).c_str(), line_sharers_orig, line->sharers);
assert(! ((line->state & (LINE_MOD | LINE_EXC)) && (line->state & LINE_TXW)) );
}
