// Attempt to mmap a file for read-only access.
void MmapFileMap(MemoryMappedFile* self, const char *fn)
{
TimingScope timing_scope(&g_Stats.m_MmapCalls, &g_Stats.m_MmapTimeCycles);
MmapFileUnmap(self);
int fd = open(fn, O_RDONLY);
if (-1 == fd)
goto error;
struct stat stbuf;
if (0 != fstat(fd, &stbuf))
goto error;
self->m_Address = mmap(NULL, stbuf.st_size, PROT_READ, MAP_FILE|MAP_PRIVATE, fd, 0);
self->m_Size = stbuf.st_size;
self->m_SysData[0] = fd;
if (self->m_Address)
return;
error:
if (-1 != fd)
close(fd);
Clear(self);
}
bool DigestCacheSave(DigestCache* self, MemAllocHeap* serialization_heap, const char* tmp_filename)
{
TimingScope timing_scope(nullptr, &g_Stats.m_DigestCacheSaveTimeCycles);
BinaryWriter writer;
BinaryWriterInit(&writer, serialization_heap);
BinarySegment *main_seg = BinaryWriterAddSegment(&writer);
BinarySegment *array_seg = BinaryWriterAddSegment(&writer);
BinarySegment *string_seg = BinaryWriterAddSegment(&writer);
BinaryLocator array_ptr = BinarySegmentPosition(array_seg);
auto save_record = [=](size_t index, const HashRecord* hr)
{
const DigestCacheRecord* r = (const DigestCacheRecord*) hr;
BinarySegmentWriteUint64(array_seg, r->m_Timestamp);
BinarySegmentWriteUint64(array_seg, r->m_AccessTime);
BinarySegmentWriteUint32(array_seg, r->m_Hash);
BinarySegmentWrite(array_seg, &r->m_ContentDigest, sizeof(r->m_ContentDigest));
BinarySegmentWritePointer(array_seg, BinarySegmentPosition(string_seg));
BinarySegmentWriteStringData(string_seg, r->m_String);
BinarySegmentWriteUint32(array_seg, 0); // m_Padding
#if ENABLED(USE_FAST_HASH)
BinarySegmentWriteUint32(array_seg, 0); // m_Padding
#endif
};
HashTableWalk(&self->m_Table, save_record);
BinarySegmentWriteUint32(main_seg, DigestCacheState::MagicNumber);
BinarySegmentWriteInt32(main_seg, (int) self->m_Table.m_RecordCount);
BinarySegmentWritePointer(main_seg, array_ptr);
// Unmap old state to avoid sharing conflicts on Windows.
MmapFileUnmap(&self->m_StateFile);
self->m_State = nullptr;
bool success = BinaryWriterFlush(&writer, tmp_filename);
if (success)
{
success = RenameFile(tmp_filename, self->m_StateFilename);
}
else
{
remove(tmp_filename);
}
BinaryWriterDestroy(&writer);
return success;
}
void DigestCacheInit(DigestCache* self, size_t heap_size, const char* filename)
{
ReadWriteLockInit(&self->m_Lock);
self->m_State = nullptr;
self->m_StateFilename = filename;
HeapInit(&self->m_Heap, heap_size, HeapFlags::kDefault);
LinearAllocInit(&self->m_Allocator, &self->m_Heap, heap_size / 2, "digest allocator");
MmapFileInit(&self->m_StateFile);
HashTableInit(&self->m_Table, &self->m_Heap, HashTable::kFlagPathStrings);
self->m_AccessTime = time(nullptr);
MmapFileMap(&self->m_StateFile, filename);
if (MmapFileValid(&self->m_StateFile))
{
const DigestCacheState* state = (const DigestCacheState*) self->m_StateFile.m_Address;
if (DigestCacheState::MagicNumber == state->m_MagicNumber)
{
const uint64_t time_now = time(nullptr);
// Throw out records that haven't been accessed in a week.
const uint64_t cutoff_time = time_now - 7 * 24 * 60 * 60;
self->m_State = state;
//HashTablePrepareBulkInsert(&self->m_Table, state->m_Records.GetCount());
for (const FrozenDigestRecord& record : state->m_Records)
{
if (record.m_AccessTime < cutoff_time)
continue;
DigestCacheRecord* r = LinearAllocate<DigestCacheRecord>(&self->m_Allocator);
r->m_Hash = record.m_FilenameHash;
r->m_ContentDigest = record.m_ContentDigest;
r->m_Next = nullptr;
r->m_String = record.m_Filename.Get();
r->m_Timestamp = record.m_Timestamp;
r->m_AccessTime = record.m_AccessTime;
HashTableInsert(&self->m_Table, r);
}
Log(kDebug, "digest cache initialized -- %d entries", state->m_Records.GetCount());
}
else
{
MmapFileUnmap(&self->m_StateFile);
}
}
}
void MmapFileDestroy(MemoryMappedFile* self)
{
MmapFileUnmap(self);
}
bool ScanCacheSave(ScanCache* self, const char* fn, MemoryMappedFile* prev_mapping, MemAllocHeap* heap)
{
TimingScope timing_scope(nullptr, &g_Stats.m_ScanCacheSaveTime);
MemAllocLinear* scratch = self->m_Allocator;
MemAllocLinearScope scratch_scope(scratch);
ScanCacheWriter writer;
ScanCacheWriterInit(&writer, heap);
// Save new view of the scan cache
//
// Algorithm:
//
// - Get all records from the dynamic table (stuff we put in this session)
const uint32_t record_count = self->m_RecordCount;
ScanCache::Record **dyn_records = LinearAllocateArray<ScanCache::Record*>(scratch, record_count);
{
uint32_t records_out = 0;
for (uint32_t ti = 0, tsize = self->m_TableSize; ti < tsize; ++ti)
{
ScanCache::Record* chain = self->m_Table[ti];
while (chain)
{
dyn_records[records_out++] = chain;
chain = chain->m_Next;
}
}
CHECK(records_out == record_count);
}
// - Sort these records in key order (by SHA-1 hash)
std::sort(dyn_records, dyn_records + record_count, SortRecordsByHash);
const ScanData *scan_data = self->m_FrozenData;
uint32_t frozen_count = scan_data ? scan_data->m_EntryCount : 0;
const HashDigest *frozen_digests = scan_data ? scan_data->m_Keys.Get() : nullptr;
const ScanCacheEntry *frozen_entries = scan_data ? scan_data->m_Data.Get() : nullptr;
const uint64_t *frozen_times = scan_data ? scan_data->m_AccessTimes.Get() : nullptr;
const uint8_t *frozen_access = self->m_FrozenAccess;
const uint64_t now = time(nullptr);
// Keep old entries for a week.
const uint64_t timestamp_cutoff = now - 60 * 60 * 24 * 7;
auto key_dynamic = [=](size_t index) -> const HashDigest* { return &dyn_records[index]->m_Key; };
auto key_frozen = [=](size_t index) { return frozen_digests + index; };
auto save_dynamic = [&writer, dyn_records, now](size_t index)
{
SaveRecord(
&writer,
&dyn_records[index]->m_Key,
dyn_records[index]->m_Includes,
dyn_records[index]->m_IncludeCount,
dyn_records[index]->m_FileTimestamp,
now);
};
auto save_frozen = [&](size_t index)
{
uint64_t timestamp = frozen_times[index];
if (frozen_access[index])
timestamp = now;
if (timestamp > timestamp_cutoff)
{
SaveRecord(
&writer,
frozen_digests + index,
frozen_entries[index].m_IncludedFiles.GetArray(),
frozen_entries[index].m_IncludedFiles.GetCount(),
frozen_entries[index].m_FileTimestamp,
timestamp);
}
};
TraverseSortedArrays(record_count, save_dynamic, key_dynamic, frozen_count, save_frozen, key_frozen);
// Unmap the previous file from RAM so we can overwrite it on Windows.
MmapFileUnmap(prev_mapping);
self->m_FrozenData = nullptr;
bool result = ScanCacheWriterFlush(&writer, fn);
ScanCacheWriterDestroy(&writer);
return result;
}