예제 #1
0
// Attempt to mmap a file for read-only access.
void MmapFileMap(MemoryMappedFile* self, const char *fn)
{
  TimingScope timing_scope(&g_Stats.m_MmapCalls, &g_Stats.m_MmapTimeCycles);

  MmapFileUnmap(self);

  int fd = open(fn, O_RDONLY);

  if (-1 == fd)
    goto error;

  struct stat stbuf;
  if (0 != fstat(fd, &stbuf))
    goto error;

  self->m_Address    = mmap(NULL, stbuf.st_size, PROT_READ, MAP_FILE|MAP_PRIVATE, fd, 0);
  self->m_Size       = stbuf.st_size;
  self->m_SysData[0] = fd;

  if (self->m_Address)
    return;

error:
  if (-1 != fd)
    close(fd);

  Clear(self);
}
예제 #2
0
bool DigestCacheSave(DigestCache* self, MemAllocHeap* serialization_heap, const char* tmp_filename)
{
  TimingScope timing_scope(nullptr, &g_Stats.m_DigestCacheSaveTimeCycles);

  BinaryWriter writer;
  BinaryWriterInit(&writer, serialization_heap);

  BinarySegment *main_seg   = BinaryWriterAddSegment(&writer);
  BinarySegment *array_seg  = BinaryWriterAddSegment(&writer);
  BinarySegment *string_seg = BinaryWriterAddSegment(&writer);
  BinaryLocator  array_ptr  = BinarySegmentPosition(array_seg);

  auto save_record = [=](size_t index, const HashRecord* hr)
  {
    const DigestCacheRecord* r = (const DigestCacheRecord*) hr;

    BinarySegmentWriteUint64(array_seg, r->m_Timestamp);
    BinarySegmentWriteUint64(array_seg, r->m_AccessTime);
    BinarySegmentWriteUint32(array_seg, r->m_Hash);
    BinarySegmentWrite(array_seg, &r->m_ContentDigest, sizeof(r->m_ContentDigest));
    BinarySegmentWritePointer(array_seg, BinarySegmentPosition(string_seg));
    BinarySegmentWriteStringData(string_seg, r->m_String);
    BinarySegmentWriteUint32(array_seg, 0); // m_Padding
#if ENABLED(USE_FAST_HASH)
    BinarySegmentWriteUint32(array_seg, 0); // m_Padding
#endif
  };

  HashTableWalk(&self->m_Table, save_record);

  BinarySegmentWriteUint32(main_seg, DigestCacheState::MagicNumber);
  BinarySegmentWriteInt32(main_seg, (int) self->m_Table.m_RecordCount);
  BinarySegmentWritePointer(main_seg, array_ptr);

  // Unmap old state to avoid sharing conflicts on Windows.
  MmapFileUnmap(&self->m_StateFile);
  self->m_State = nullptr;

  bool success = BinaryWriterFlush(&writer, tmp_filename);

  if (success)
  {
    success = RenameFile(tmp_filename, self->m_StateFilename);
  }
  else
  {
    remove(tmp_filename);
  }

  BinaryWriterDestroy(&writer);

  return success;
}
예제 #3
0
void DigestCacheInit(DigestCache* self, size_t heap_size, const char* filename)
{
  ReadWriteLockInit(&self->m_Lock);

  self->m_State = nullptr;
  self->m_StateFilename = filename;

  HeapInit(&self->m_Heap, heap_size, HeapFlags::kDefault);
  LinearAllocInit(&self->m_Allocator, &self->m_Heap, heap_size / 2, "digest allocator");
  MmapFileInit(&self->m_StateFile);
  HashTableInit(&self->m_Table, &self->m_Heap, HashTable::kFlagPathStrings);

  self->m_AccessTime = time(nullptr);

  MmapFileMap(&self->m_StateFile, filename);
  if (MmapFileValid(&self->m_StateFile))
  {
    const DigestCacheState* state = (const DigestCacheState*) self->m_StateFile.m_Address;
    if (DigestCacheState::MagicNumber == state->m_MagicNumber)
    {
      const uint64_t time_now = time(nullptr);

      // Throw out records that haven't been accessed in a week.
      const uint64_t cutoff_time = time_now - 7 * 24 * 60 * 60;

      self->m_State = state;

      //HashTablePrepareBulkInsert(&self->m_Table, state->m_Records.GetCount());

      for (const FrozenDigestRecord& record : state->m_Records)
      {
        if (record.m_AccessTime < cutoff_time)
          continue;

        DigestCacheRecord* r = LinearAllocate<DigestCacheRecord>(&self->m_Allocator);
        r->m_Hash          = record.m_FilenameHash;
        r->m_ContentDigest = record.m_ContentDigest;
        r->m_Next          = nullptr;
        r->m_String        = record.m_Filename.Get();
        r->m_Timestamp     = record.m_Timestamp;
        r->m_AccessTime    = record.m_AccessTime;
        HashTableInsert(&self->m_Table, r);
      }
      Log(kDebug, "digest cache initialized -- %d entries", state->m_Records.GetCount());
    }
    else
    {
      MmapFileUnmap(&self->m_StateFile);
    }
  }
}
예제 #4
0
void MmapFileDestroy(MemoryMappedFile* self)
{
  MmapFileUnmap(self);
}
예제 #5
0
bool ScanCacheSave(ScanCache* self, const char* fn, MemoryMappedFile* prev_mapping, MemAllocHeap* heap)
{
  TimingScope timing_scope(nullptr, &g_Stats.m_ScanCacheSaveTime);

  MemAllocLinear* scratch = self->m_Allocator;

  MemAllocLinearScope scratch_scope(scratch);

  ScanCacheWriter writer;
  ScanCacheWriterInit(&writer, heap);

  // Save new view of the scan cache
  //
  // Algorithm:
  // 
  // - Get all records from the dynamic table (stuff we put in this session)
  const uint32_t      record_count = self->m_RecordCount;
  ScanCache::Record **dyn_records  = LinearAllocateArray<ScanCache::Record*>(scratch, record_count);

  {
    uint32_t records_out = 0;
    for (uint32_t ti = 0, tsize = self->m_TableSize; ti < tsize; ++ti)
    {
      ScanCache::Record* chain = self->m_Table[ti];
      while (chain)
      {
        dyn_records[records_out++] = chain;
        chain                      = chain->m_Next;
      }
    }

    CHECK(records_out == record_count);
  }

  // - Sort these records in key order (by SHA-1 hash)
  std::sort(dyn_records, dyn_records + record_count, SortRecordsByHash);

  const ScanData       *scan_data      = self->m_FrozenData;
  uint32_t              frozen_count   = scan_data ? scan_data->m_EntryCount : 0;
  const HashDigest     *frozen_digests = scan_data ? scan_data->m_Keys.Get() : nullptr;
  const ScanCacheEntry *frozen_entries = scan_data ? scan_data->m_Data.Get() : nullptr;
  const uint64_t       *frozen_times   = scan_data ? scan_data->m_AccessTimes.Get() : nullptr;
  const uint8_t        *frozen_access  = self->m_FrozenAccess;

  const uint64_t now = time(nullptr);

  // Keep old entries for a week.
  const uint64_t timestamp_cutoff = now - 60 * 60 * 24 * 7;

  auto key_dynamic = [=](size_t index) -> const HashDigest* { return &dyn_records[index]->m_Key; };
  auto key_frozen = [=](size_t index) { return frozen_digests + index; };

  auto save_dynamic = [&writer, dyn_records, now](size_t index)
  {
    SaveRecord(
        &writer,
        &dyn_records[index]->m_Key,
        dyn_records[index]->m_Includes,
        dyn_records[index]->m_IncludeCount,
        dyn_records[index]->m_FileTimestamp,
        now);
  };

  auto save_frozen = [&](size_t index)
  {
    uint64_t timestamp = frozen_times[index];
    if (frozen_access[index])
      timestamp = now;

    if (timestamp > timestamp_cutoff)
    {
      SaveRecord(
          &writer,
          frozen_digests + index, 
          frozen_entries[index].m_IncludedFiles.GetArray(),
          frozen_entries[index].m_IncludedFiles.GetCount(),
          frozen_entries[index].m_FileTimestamp,
          timestamp);
    }
  };

  TraverseSortedArrays(record_count, save_dynamic, key_dynamic, frozen_count, save_frozen, key_frozen);

  // Unmap the previous file from RAM so we can overwrite it on Windows.
  MmapFileUnmap(prev_mapping);

  self->m_FrozenData = nullptr;

  bool result = ScanCacheWriterFlush(&writer, fn);

  ScanCacheWriterDestroy(&writer);

  return result;
}