TEST(SimhashTest, Repeat)
{
std::vector<Simhash::hash_t> hashes(100, 0xDEADBEEF);
Simhash::hash_t expected = 0xDEADBEEF;
EXPECT_EQ(expected, Simhash::compute(hashes));
}
void rom_load_manager::process_disk_entries(const char *regiontag, const rom_entry *parent_region, const rom_entry *romp, const char *locationtag)
{
/* loop until we hit the end of this region */
for ( ; !ROMENTRY_ISREGIONEND(romp); romp++)
{
/* handle files */
if (ROMENTRY_ISFILE(romp))
{
auto chd = std::make_unique<open_chd>(regiontag);
util::hash_collection hashes(ROM_GETHASHDATA(romp));
chd_error err;
/* make the filename of the source */
std::string filename = std::string(ROM_GETNAME(romp)).append(".chd");
/* first open the source drive */
LOG(("Opening disk image: %s\n", filename.c_str()));
err = chd_error(open_disk_image(machine().options(), &machine().system(), romp, chd->orig_chd(), locationtag));
if (err != CHDERR_NONE)
{
handle_missing_file(romp, std::string(), err);
chd = nullptr;
continue;
}
/* get the header and extract the SHA1 */
util::hash_collection acthashes;
acthashes.add_sha1(chd->orig_chd().sha1());
/* verify the hash */
if (hashes != acthashes)
{
m_errorstring.append(string_format("%s WRONG CHECKSUMS:\n", filename));
dump_wrong_and_correct_checksums(hashes, acthashes);
m_warnings++;
}
else if (hashes.flag(util::hash_collection::FLAG_BAD_DUMP))
{
m_errorstring.append(string_format("%s CHD NEEDS REDUMP\n", filename));
m_knownbad++;
}
/* if not read-only, make the diff file */
if (!DISK_ISREADONLY(romp))
{
/* try to open or create the diff */
err = open_disk_diff(machine().options(), romp, chd->orig_chd(), chd->diff_chd());
if (err != CHDERR_NONE)
{
m_errorstring.append(string_format("%s DIFF CHD ERROR: %s\n", filename, chd_file::error_string(err)));
m_errors++;
chd = nullptr;
continue;
}
}
/* we're okay, add to the list of disks */
LOG(("Assigning to handle %d\n", DISK_GETINDEX(romp)));
m_chd_list.push_back(std::move(chd));
}
}
}
media_auditor::summary media_auditor::audit_media(const char *validation)
{
// start fresh
m_record_list.reset();
// store validation for later
m_validation = validation;
// temporary hack until romload is update: get the driver path and support it for
// all searches
const char *driverpath = m_enumerator.config().devicelist().find("root")->searchpath();
// iterate over ROM sources and regions
int found = 0;
int required = 0;
int sharedFound = 0;
int sharedRequired = 0;
for (const rom_source *source = rom_first_source(m_enumerator.config()); source != NULL; source = rom_next_source(*source))
{
// determine the search path for this source and iterate through the regions
m_searchpath = source->searchpath();
// also determine if this is the driver's specific ROMs or not
bool source_is_gamedrv = (dynamic_cast<const driver_device *>(source) != NULL);
// now iterate over regions and ROMs within
for (const rom_entry *region = rom_first_region(*source); region != NULL; region = rom_next_region(region))
{
// temporary hack: add the driver path & region name
astring combinedpath(source->searchpath(), ";", driverpath);
if(ROMREGION_ISLOADBYNAME(region))
{
combinedpath=combinedpath.cat(";");
combinedpath=combinedpath.cat(ROMREGION_GETTAG(region));
}
m_searchpath = combinedpath;
for (const rom_entry *rom = rom_first_file(region); rom; rom = rom_next_file(rom))
{
hash_collection hashes(ROM_GETHASHDATA(rom));
bool shared = also_used_by_parent(hashes) >= 0;
// if a dump exists, then at least one entry is required
if (!hashes.flag(hash_collection::FLAG_NO_DUMP))
{
required++;
if (shared)
{
sharedRequired++;
}
}
// audit a file
audit_record *record = NULL;
if (ROMREGION_ISROMDATA(region))
record = audit_one_rom(rom);
// audit a disk
else if (ROMREGION_ISDISKDATA(region))
record = audit_one_disk(rom);
// skip if no record
if (record == NULL)
continue;
// if we got a record back,
if (record->status() != audit_record::STATUS_NOT_FOUND && source_is_gamedrv)
{
found++;
if (shared)
{
sharedFound++;
}
}
}
}
}
// if we found nothing unique to this set & the set needs roms that aren't in the parent or the parent isn't found either, then we don't have the set at all
if (found == sharedFound && required > 0 && (required != sharedRequired || sharedFound == 0))
m_record_list.reset();
// return a summary
return summarize();
}
void info_xml_creator::output_rom(device_t &device)
{
// iterate over 3 different ROM "types": BIOS, ROMs, DISKs
for (int rom_type = 0; rom_type < 3; rom_type++)
for (const rom_entry *region = rom_first_region(device); region != NULL; region = rom_next_region(region))
{
bool is_disk = ROMREGION_ISDISKDATA(region);
// disk regions only work for disks
if ((is_disk && rom_type != 2) || (!is_disk && rom_type == 2))
continue;
// iterate through ROM entries
for (const rom_entry *rom = rom_first_file(region); rom != NULL; rom = rom_next_file(rom))
{
bool is_bios = ROM_GETBIOSFLAGS(rom);
const char *name = ROM_GETNAME(rom);
int offset = ROM_GETOFFSET(rom);
const char *merge_name = NULL;
char bios_name[100];
// BIOS ROMs only apply to bioses
if ((is_bios && rom_type != 0) || (!is_bios && rom_type == 0))
continue;
// if we have a valid ROM and we are a clone, see if we can find the parent ROM
hash_collection hashes(ROM_GETHASHDATA(rom));
if (!hashes.flag(hash_collection::FLAG_NO_DUMP))
merge_name = get_merge_name(hashes);
if (&device != &m_drivlist.config().root_device())
merge_name = NULL;
// scan for a BIOS name
bios_name[0] = 0;
if (!is_disk && is_bios)
{
// scan backwards through the ROM entries
for (const rom_entry *brom = rom - 1; brom != m_drivlist.driver().rom; brom--)
if (ROMENTRY_ISSYSTEM_BIOS(brom))
{
strcpy(bios_name, ROM_GETNAME(brom));
break;
}
}
astring output;
// opening tag
if (!is_disk)
output.cat("\t\t<rom");
else
output.cat("\t\t<disk");
// add name, merge, bios, and size tags */
if (name != NULL && name[0] != 0)
output.catprintf(" name=\"%s\"", xml_normalize_string(name));
if (merge_name != NULL)
output.catprintf(" merge=\"%s\"", xml_normalize_string(merge_name));
if (bios_name[0] != 0)
output.catprintf(" bios=\"%s\"", xml_normalize_string(bios_name));
if (!is_disk)
output.catprintf(" size=\"%d\"", rom_file_size(rom));
// dump checksum information only if there is a known dump
if (!hashes.flag(hash_collection::FLAG_NO_DUMP))
{
// iterate over hash function types and print m_output their values
astring tempstr;
output.catprintf(" %s", hashes.attribute_string(tempstr));
}
else
output.cat(" status=\"nodump\"");
// append a region name
output.catprintf(" region=\"%s\"", ROMREGION_GETTAG(region));
// for non-disk entries, print offset
if (!is_disk)
output.catprintf(" offset=\"%x\"", offset);
// for disk entries, add the disk index
else
{
output.catprintf(" index=\"%x\"", DISK_GETINDEX(rom));
output.catprintf(" writable=\"%s\"", DISK_ISREADONLY(rom) ? "no" : "yes");
}
// add optional flag
if (ROM_ISOPTIONAL(rom))
output.cat(" optional=\"yes\"");
output.cat("/>\n");
fprintf(m_output, "%s", output.c_str());
}
}
}
bool CompressFileToBlob(const std::string& infile, const std::string& outfile, u32 sub_type,
int block_size, CompressCB callback, void* arg)
{
bool scrubbing = false;
if (IsGCZBlob(infile))
{
PanicAlertT("\"%s\" is already compressed! Cannot compress it further.", infile.c_str());
return false;
}
File::IOFile inf(infile, "rb");
if (!inf)
{
PanicAlertT("Failed to open the input file \"%s\".", infile.c_str());
return false;
}
File::IOFile f(outfile, "wb");
if (!f)
{
PanicAlertT("Failed to open the output file \"%s\".\n"
"Check that you have permissions to write the target folder and that the media can be written.",
outfile.c_str());
return false;
}
if (sub_type == 1)
{
if (!DiscScrubber::SetupScrub(infile, block_size))
{
PanicAlertT("\"%s\" failed to be scrubbed. Probably the image is corrupt.", infile.c_str());
return false;
}
scrubbing = true;
}
z_stream z = {};
if (deflateInit(&z, 9) != Z_OK)
{
DiscScrubber::Cleanup();
return false;
}
callback("Files opened, ready to compress.", 0, arg);
CompressedBlobHeader header;
header.magic_cookie = kBlobCookie;
header.sub_type = sub_type;
header.block_size = block_size;
header.data_size = File::GetSize(infile);
// round upwards!
header.num_blocks = (u32)((header.data_size + (block_size - 1)) / block_size);
std::vector<u64> offsets(header.num_blocks);
std::vector<u32> hashes(header.num_blocks);
std::vector<u8> out_buf(block_size);
std::vector<u8> in_buf(block_size);
// seek past the header (we will write it at the end)
f.Seek(sizeof(CompressedBlobHeader), SEEK_CUR);
// seek past the offset and hash tables (we will write them at the end)
f.Seek((sizeof(u64) + sizeof(u32)) * header.num_blocks, SEEK_CUR);
// Now we are ready to write compressed data!
u64 position = 0;
int num_compressed = 0;
int num_stored = 0;
int progress_monitor = std::max<int>(1, header.num_blocks / 1000);
bool success = true;
for (u32 i = 0; i < header.num_blocks; i++)
{
if (i % progress_monitor == 0)
{
const u64 inpos = inf.Tell();
int ratio = 0;
if (inpos != 0)
ratio = (int)(100 * position / inpos);
std::string temp = StringFromFormat("%i of %i blocks. Compression ratio %i%%", i, header.num_blocks, ratio);
bool was_cancelled = !callback(temp, (float)i / (float)header.num_blocks, arg);
if (was_cancelled)
{
success = false;
break;
}
}
offsets[i] = position;
size_t read_bytes;
if (scrubbing)
read_bytes = DiscScrubber::GetNextBlock(inf, in_buf.data());
else
inf.ReadArray(in_buf.data(), header.block_size, &read_bytes);
if (read_bytes < header.block_size)
std::fill(in_buf.begin() + read_bytes, in_buf.begin() + header.block_size, 0);
int retval = deflateReset(&z);
z.next_in = in_buf.data();
z.avail_in = header.block_size;
z.next_out = out_buf.data();
z.avail_out = block_size;
if (retval != Z_OK)
{
ERROR_LOG(DISCIO, "Deflate failed");
success = false;
break;
}
int status = deflate(&z, Z_FINISH);
int comp_size = block_size - z.avail_out;
u8* write_buf;
int write_size;
if ((status != Z_STREAM_END) || (z.avail_out < 10))
{
//PanicAlert("%i %i Store %i", i*block_size, position, comp_size);
// let's store uncompressed
write_buf = in_buf.data();
offsets[i] |= 0x8000000000000000ULL;
write_size = block_size;
num_stored++;
}
else
{
// let's store compressed
//PanicAlert("Comp %i to %i", block_size, comp_size);
write_buf = out_buf.data();
write_size = comp_size;
num_compressed++;
}
if (!f.WriteBytes(write_buf, write_size))
{
PanicAlertT(
"Failed to write the output file \"%s\".\n"
"Check that you have enough space available on the target drive.",
outfile.c_str());
success = false;
break;
}
position += write_size;
hashes[i] = HashAdler32(write_buf, write_size);
}
header.compressed_data_size = position;
if (!success)
{
// Remove the incomplete output file.
f.Close();
File::Delete(outfile);
}
else
{
// Okay, go back and fill in headers
f.Seek(0, SEEK_SET);
f.WriteArray(&header, 1);
f.WriteArray(offsets.data(), header.num_blocks);
f.WriteArray(hashes.data(), header.num_blocks);
}
// Cleanup
deflateEnd(&z);
DiscScrubber::Cleanup();
if (success)
{
callback("Done compressing disc image.", 1.0f, arg);
}
return success;
}
media_auditor::summary media_auditor::audit_media(const char *validation)
{
// start fresh
m_record_list.reset();
// store validation for later
m_validation = validation;
// temporary hack until romload is update: get the driver path and support it for
// all searches
const char *driverpath = m_enumerator.config().root_device().searchpath();
int found = 0;
int required = 0;
int shared_found = 0;
int shared_required = 0;
// iterate over devices and regions
device_iterator deviter(m_enumerator.config().root_device());
for (device_t *device = deviter.first(); device != NULL; device = deviter.next())
{
// determine the search path for this source and iterate through the regions
m_searchpath = device->searchpath();
// now iterate over regions and ROMs within
for (const rom_entry *region = rom_first_region(*device); region != NULL; region = rom_next_region(region))
{
// temporary hack: add the driver path & region name
astring combinedpath(device->searchpath(), ";", driverpath);
if (device->shortname())
combinedpath.cat(";").cat(device->shortname());
m_searchpath = combinedpath;
for (const rom_entry *rom = rom_first_file(region); rom; rom = rom_next_file(rom))
{
const char *name = ROM_GETNAME(rom);
hash_collection hashes(ROM_GETHASHDATA(rom));
device_t *shared_device = find_shared_device(*device, name, hashes, ROM_GETLENGTH(rom));
// count the number of files with hashes
if (!hashes.flag(hash_collection::FLAG_NO_DUMP) && !ROM_ISOPTIONAL(rom))
{
required++;
if (shared_device != NULL)
shared_required++;
}
// audit a file
audit_record *record = NULL;
if (ROMREGION_ISROMDATA(region))
record = audit_one_rom(rom);
// audit a disk
else if (ROMREGION_ISDISKDATA(region))
record = audit_one_disk(rom);
if (record != NULL)
{
// count the number of files that are found.
if (record->status() == audit_record::STATUS_GOOD || (record->status() == audit_record::STATUS_FOUND_INVALID && find_shared_device(*device, name, record->actual_hashes(), record->actual_length()) == NULL))
{
found++;
if (shared_device != NULL)
shared_found++;
}
record->set_shared_device(shared_device);
}
}
}
}
// if we only find files that are in the parent & either the set has no unique files or the parent is not found, then assume we don't have the set at all
if (found == shared_found && required > 0 && (required != shared_required || shared_found == 0))
{
m_record_list.reset();
return NOTFOUND;
}
// return a summary
return summarize(m_enumerator.driver().name);
}
//-------------------------------------------------
// audit_software
//-------------------------------------------------
media_auditor::summary media_auditor::audit_software(const char *list_name, software_info *swinfo, const char *validation)
{
// start fresh
m_record_list.reset();
// store validation for later
m_validation = validation;
astring combinedpath(swinfo->shortname, ";", list_name, PATH_SEPARATOR, swinfo->shortname);
m_searchpath = combinedpath;
int found = 0;
int required = 0;
// now iterate over software parts
for ( software_part *part = software_find_part( swinfo, NULL, NULL ); part != NULL; part = software_part_next( part ) )
{
// now iterate over regions
for ( const rom_entry *region = part->romdata; region; region = rom_next_region( region ) )
{
// now iterate over rom definitions
for (const rom_entry *rom = rom_first_file(region); rom; rom = rom_next_file(rom))
{
hash_collection hashes(ROM_GETHASHDATA(rom));
// count the number of files with hashes
if (!hashes.flag(hash_collection::FLAG_NO_DUMP) && !ROM_ISOPTIONAL(rom))
{
required++;
}
// audit a file
audit_record *record = NULL;
if (ROMREGION_ISROMDATA(region))
{
record = audit_one_rom(rom);
}
// audit a disk
else if (ROMREGION_ISDISKDATA(region))
{
record = audit_one_disk(rom);
}
// count the number of files that are found.
if (record != NULL && (record->status() == audit_record::STATUS_GOOD || record->status() == audit_record::STATUS_FOUND_INVALID))
{
found++;
}
}
}
}
if (found == 0 && required > 0)
{
m_record_list.reset();
return NOTFOUND;
}
// return a summary
return summarize(list_name);
}
void run ()
{
testcase ("add/traverse");
beast::Journal const j; // debug journal
tests::TestFamily f(j);
// h3 and h4 differ only in the leaf, same terminal node (level 19)
uint256 h1, h2, h3, h4, h5;
h1.SetHex ("092891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
h2.SetHex ("436ccbac3347baa1f1e53baeef1f43334da88f1f6d70d963b833afd6dfa289fe");
h3.SetHex ("b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
h4.SetHex ("b92891fe4ef6cee585fdc6fda2e09eb4d386363158ec3321b8123e5a772c6ca8");
h5.SetHex ("a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
SHAMap sMap (SHAMapType::FREE, f, beast::Journal());
SHAMapItem i1 (h1, IntToVUC (1)), i2 (h2, IntToVUC (2)), i3 (h3, IntToVUC (3)), i4 (h4, IntToVUC (4)), i5 (h5, IntToVUC (5));
unexpected (!sMap.addItem (i2, true, false), "no add");
unexpected (!sMap.addItem (i1, true, false), "no add");
std::shared_ptr<SHAMapItem> i;
i = sMap.peekFirstItem ();
unexpected (!i || (*i != i1), "bad traverse");
i = sMap.peekNextItem (i->getTag ());
unexpected (!i || (*i != i2), "bad traverse");
i = sMap.peekNextItem (i->getTag ());
unexpected (i, "bad traverse");
sMap.addItem (i4, true, false);
sMap.delItem (i2.getTag ());
sMap.addItem (i3, true, false);
i = sMap.peekFirstItem ();
unexpected (!i || (*i != i1), "bad traverse");
i = sMap.peekNextItem (i->getTag ());
unexpected (!i || (*i != i3), "bad traverse");
i = sMap.peekNextItem (i->getTag ());
unexpected (!i || (*i != i4), "bad traverse");
i = sMap.peekNextItem (i->getTag ());
unexpected (i, "bad traverse");
testcase ("snapshot");
uint256 mapHash = sMap.getHash ();
std::shared_ptr<SHAMap> map2 = sMap.snapShot (false);
unexpected (sMap.getHash () != mapHash, "bad snapshot");
unexpected (map2->getHash () != mapHash, "bad snapshot");
unexpected (!sMap.delItem (sMap.peekFirstItem ()->getTag ()), "bad mod");
unexpected (sMap.getHash () == mapHash, "bad snapshot");
unexpected (map2->getHash () != mapHash, "bad snapshot");
testcase ("build/tear");
{
std::vector<uint256> keys(8);
keys[0].SetHex ("b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[1].SetHex ("b92881fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[2].SetHex ("b92691fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[3].SetHex ("b92791fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[4].SetHex ("b91891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[5].SetHex ("b99891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[6].SetHex ("f22891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[7].SetHex ("292891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
std::vector<uint256> hashes(8);
hashes[0].SetHex ("B7387CFEA0465759ADC718E8C42B52D2309D179B326E239EB5075C64B6281F7F");
hashes[1].SetHex ("FBC195A9592A54AB44010274163CB6BA95F497EC5BA0A8831845467FB2ECE266");
hashes[2].SetHex ("4E7D2684B65DFD48937FFB775E20175C43AF0C94066F7D5679F51AE756795B75");
hashes[3].SetHex ("7A2F312EB203695FFD164E038E281839EEF06A1B99BFC263F3CECC6C74F93E07");
hashes[4].SetHex ("395A6691A372387A703FB0F2C6D2C405DAF307D0817F8F0E207596462B0E3A3E");
hashes[5].SetHex ("D044C0A696DE3169CC70AE216A1564D69DE96582865796142CE7D98A84D9DDE4");
hashes[6].SetHex ("76DCC77C4027309B5A91AD164083264D70B77B5E43E08AEDA5EBF94361143615");
hashes[7].SetHex ("DF4220E93ADC6F5569063A01B4DC79F8DB9553B6A3222ADE23DEA02BBE7230E5");
SHAMap map (SHAMapType::FREE, f, beast::Journal());
expect (map.getHash() == uint256(), "bad initial empty map hash");
for (int i = 0; i < keys.size(); ++i)
{
SHAMapItem item (keys[i], IntToVUC (i));
map.addItem (item, true, false);
expect (map.getHash() == hashes[i], "bad buildup map hash");
}
for (int i = keys.size() - 1; i >= 0; --i)
{
expect (map.getHash() == hashes[i], "bad teardown hash");
map.delItem (keys[i]);
}
expect (map.getHash() == uint256(), "bad final empty map hash");
}
}
