void decompressResource()
{
for(bool bDone = false;!bDone;) //Loop until we're done
{
ThreadConvertHelper dh;
wstring sFilename;
std::lock_guard<std::mutex> lock(g_Mutex);
if(!g_lThreadedResources.size()) //Done
{
bDone = true;
}
else
{
//Grab the top item off the list
dh = g_lThreadedResources.front();
sFilename = getName(dh.id); //Mutex on this too, since getName() isn't really threadsafe
makeFolder(dh.id); //Also create folder (not threadsafe, either)
g_lThreadedResources.pop_front(); //Done with this element
}
//Let user know which resource we're converting now
if(!bDone)
{
g_iCurResource++;
if(!(sFilename == RESIDMAP_NAME && g_iCurResource == 1))
{
if(g_bProgressOverwrite)
{
cout << "\rDecompressing file " << g_iCurResource << " out of " << g_iNumResources;
cout.flush();
}
else
cout << "Decompressing file " << g_iCurResource << " out of " << g_iNumResources << ": " << ws2s(sFilename) << endl;
}
}
// Release ownership of the mutex object
if(sFilename == RESIDMAP_NAME && g_iCurResource == 1) //Don't release residmap.dat mutex until we've read in all the filenames
{
g_iNumResources--;
}
if(bDone)
{
continue; //Stop here if done
}
if(dh.bCompressed) //Compressed
{
uint8_t* tempData = decompress(&dh.data);
if(tempData == NULL)
{
cout << "Error decompressing file " << ws2s(sFilename) << endl;
return;
}
free(dh.data.data); //Free this compressed memory
dh.data.data = tempData; //Now we have the decompressed data
}
//See if this was a PNG image. Convert PNG images from the data in RAM
if(sFilename.find(L".png") != wstring::npos ||
sFilename.find(L".PNG") != wstring::npos ||
sFilename.find(L"coloritemicon") != wstring::npos ||
sFilename.find(L"colorbgicon") != wstring::npos ||
sFilename.find(L"greybgicon") != wstring::npos) //Also would include .png.normal files as well
{
convertToPNG(sFilename.c_str(), dh.data.data, dh.data.decompressedSize); //Do the conversion to PNG
}
else //For other file types, go ahead and write to the file before converting
{
//Write this out to the file
FILE* fOut = fopen(ws2s(sFilename).c_str(), "wb");
if(fOut == NULL)
{
cout << "Unable to open output file " << ws2s(sFilename) << endl;
return;
}
fwrite(dh.data.data, 1, dh.data.decompressedSize, fOut);
fclose(fOut);
}
free(dh.data.data); //Free memory from this file
/*
//Convert wordPackDict.dat to XML
if(sFilename.find(L"wordPackDict.dat") != wstring::npos)
{
wordPackToXML(sFilename.c_str());
unlink(ws2s(sFilename).c_str());
}
//Convert sndmanifest.dat to XML
else if(sFilename.find(L"sndmanifest.dat") != wstring::npos)
{
sndManifestToXML(sFilename.c_str());
unlink(ws2s(sFilename).c_str());
}
//Convert itemmanifest.dat to XML
else if(sFilename.find(L"itemmanifest.dat") != wstring::npos)
{
itemManifestToXML(sFilename.c_str());
unlink(ws2s(sFilename).c_str());
}
//Convert letterdb.dat to XML
else if(sFilename.find(L"letterdb.dat") != wstring::npos)
{
letterToXML(sFilename.c_str());
unlink(ws2s(sFilename).c_str());
}
//Convert catalogdb.dat to XML
else if(sFilename.find(L"catalogdb.dat") != wstring::npos)
{
catalogToXML(sFilename.c_str());
unlink(ws2s(sFilename).c_str());
}
//Convert combodb.dat to XML
else if(sFilename.find(L"combodb.dat") != wstring::npos)
{
comboDBToXML(sFilename.c_str());
unlink(ws2s(sFilename).c_str());
}
//Convert residmap.dat to XML
else if(sFilename.find(L"residmap.dat") != wstring::npos)
{
residMapToXML(sFilename.c_str());
unlink(ws2s(sFilename).c_str());
}
//Convert .flac binary files to OGG
else if(sFilename.find(L".flac") != wstring::npos ||
sFilename.find(L".FLAC") != wstring::npos)
{
wstring s = sFilename;
s += L".ogg";
binaryToOgg(sFilename.c_str(), s.c_str());
unlink(ws2s(sFilename).c_str()); //Delete temporary .flac file
}
//Convert vdata/fontmanifest.dat to XML
else if(sFilename.find(L"fontmanifest.dat") != wstring::npos)
{
fontManifestToXML(sFilename);
unlink(ws2s(sFilename).c_str());
}
//Convert font files to XML
else if(sFilename.find(L".font.xml") != wstring::npos)
{
fontToXML(sFilename);
}
//Convert vdata/loctexmanifest.bin to XML
else if(sFilename.find(L"loctexmanifest.bin") != wstring::npos)
{
LoctexManifestToXML(sFilename);
unlink(ws2s(sFilename).c_str());
}
//Convert vdata/myPicturesImage.dat to XML
else if(sFilename.find(L"myPicturesImage.dat") != wstring::npos)
{
myPicturesToXML(sFilename);
unlink(ws2s(sFilename).c_str());
}
//Convert vdata/smokeImage.dat to XML
else if(sFilename.find(L"smokeImage.dat") != wstring::npos)
{
smokeImageToXML(sFilename);
unlink(ws2s(sFilename).c_str());
}
//Convert vdata/fluidPalettes.dat to XML
else if(sFilename.find(L"fluidPalettes.dat") != wstring::npos)
{
fluidPalettesToXML(sFilename);
unlink(ws2s(sFilename).c_str());
}
*/
if(sFilename == RESIDMAP_NAME && g_iCurResource == 1)
{
g_iCurResource--;
}
}
return;
}
bool mvt_tile::decode(std::string &message) {
layers.clear();
std::string src;
if (is_compressed(message)) {
std::string uncompressed;
decompress(message, uncompressed);
src = uncompressed;
} else {
src = message;
}
protozero::pbf_reader reader(src);
while (reader.next()) {
switch (reader.tag()) {
case 3: /* layer */
{
protozero::pbf_reader layer_reader(reader.get_message());
mvt_layer layer;
while (layer_reader.next()) {
switch (layer_reader.tag()) {
case 1: /* name */
layer.name = layer_reader.get_string();
break;
case 3: /* key */
layer.keys.push_back(layer_reader.get_string());
break;
case 4: /* value */
{
protozero::pbf_reader value_reader(layer_reader.get_message());
mvt_value value;
while (value_reader.next()) {
switch (value_reader.tag()) {
case 1: /* string */
value.type = mvt_string;
value.string_value = value_reader.get_string();
break;
case 2: /* float */
value.type = mvt_float;
value.numeric_value.float_value = value_reader.get_float();
break;
case 3: /* double */
value.type = mvt_double;
value.numeric_value.double_value = value_reader.get_double();
break;
case 4: /* int */
value.type = mvt_int;
value.numeric_value.int_value = value_reader.get_int64();
break;
case 5: /* uint */
value.type = mvt_uint;
value.numeric_value.uint_value = value_reader.get_uint64();
break;
case 6: /* sint */
value.type = mvt_sint;
value.numeric_value.sint_value = value_reader.get_sint64();
break;
case 7: /* bool */
value.type = mvt_bool;
value.numeric_value.bool_value = value_reader.get_bool();
break;
default:
value_reader.skip();
break;
}
}
layer.values.push_back(value);
break;
}
case 5: /* extent */
layer.extent = layer_reader.get_uint32();
break;
case 15: /* version */
layer.version = layer_reader.get_uint32();
break;
case 2: /* feature */
{
protozero::pbf_reader feature_reader(layer_reader.get_message());
mvt_feature feature;
std::vector<uint32_t> geoms;
while (feature_reader.next()) {
switch (feature_reader.tag()) {
case 1: /* id */
feature.id = feature_reader.get_uint64();
feature.has_id = true;
break;
case 2: /* tag */
{
auto pi = feature_reader.get_packed_uint32();
for (auto it = pi.first; it != pi.second; ++it) {
feature.tags.push_back(*it);
}
break;
}
case 3: /* feature type */
feature.type = feature_reader.get_enum();
break;
case 4: /* geometry */
{
auto pi = feature_reader.get_packed_uint32();
for (auto it = pi.first; it != pi.second; ++it) {
geoms.push_back(*it);
}
break;
}
default:
feature_reader.skip();
break;
}
}
long long px = 0, py = 0;
for (size_t g = 0; g < geoms.size(); g++) {
uint32_t geom = geoms[g];
uint32_t op = geom & 7;
uint32_t count = geom >> 3;
if (op == mvt_moveto || op == mvt_lineto) {
for (size_t k = 0; k < count && g + 2 < geoms.size(); k++) {
px += protozero::decode_zigzag32(geoms[g + 1]);
py += protozero::decode_zigzag32(geoms[g + 2]);
g += 2;
feature.geometry.push_back(mvt_geometry(op, px, py));
}
} else {
feature.geometry.push_back(mvt_geometry(op, 0, 0));
}
}
layer.features.push_back(feature);
break;
}
default:
layer_reader.skip();
break;
}
}
for (size_t i = 0; i < layer.keys.size(); i++) {
layer.key_map.insert(std::pair<std::string, size_t>(layer.keys[i], i));
}
for (size_t i = 0; i < layer.values.size(); i++) {
layer.value_map.insert(std::pair<mvt_value, size_t>(layer.values[i], i));
}
layers.push_back(layer);
break;
}
default:
reader.skip();
break;
}
}
return true;
}
output_t decompress_as (const compressed_data& in)
{
output_t out;
decompress(in, out);
return out;
}
void MapBlock::deSerialize_pre22(std::istream &is, u8 version, bool disk)
{
u32 nodecount = MAP_BLOCKSIZE*MAP_BLOCKSIZE*MAP_BLOCKSIZE;
// Initialize default flags
is_underground = false;
m_day_night_differs = false;
m_lighting_expired = false;
m_generated = true;
// Make a temporary buffer
u32 ser_length = MapNode::serializedLength(version);
SharedBuffer<u8> databuf_nodelist(nodecount * ser_length);
// These have no compression
if(version <= 3 || version == 5 || version == 6)
{
char tmp;
is.read(&tmp, 1);
if(is.gcount() != 1)
throw SerializationError
("MapBlock::deSerialize: no enough input data");
is_underground = tmp;
is.read((char*)*databuf_nodelist, nodecount * ser_length);
if(is.gcount() != nodecount * ser_length)
throw SerializationError
("MapBlock::deSerialize: no enough input data");
}
else if(version <= 10)
{
u8 t8;
is.read((char*)&t8, 1);
is_underground = t8;
{
// Uncompress and set material data
std::ostringstream os(std::ios_base::binary);
decompress(is, os, version);
std::string s = os.str();
if(s.size() != nodecount)
throw SerializationError
("MapBlock::deSerialize: invalid format");
for(u32 i=0; i<s.size(); i++)
{
databuf_nodelist[i*ser_length] = s[i];
}
}
{
// Uncompress and set param data
std::ostringstream os(std::ios_base::binary);
decompress(is, os, version);
std::string s = os.str();
if(s.size() != nodecount)
throw SerializationError
("MapBlock::deSerialize: invalid format");
for(u32 i=0; i<s.size(); i++)
{
databuf_nodelist[i*ser_length + 1] = s[i];
}
}
if(version >= 10)
{
// Uncompress and set param2 data
std::ostringstream os(std::ios_base::binary);
decompress(is, os, version);
std::string s = os.str();
if(s.size() != nodecount)
throw SerializationError
("MapBlock::deSerialize: invalid format");
for(u32 i=0; i<s.size(); i++)
{
databuf_nodelist[i*ser_length + 2] = s[i];
}
}
}
// All other versions (newest)
else
{
u8 flags;
is.read((char*)&flags, 1);
is_underground = (flags & 0x01) ? true : false;
m_day_night_differs = (flags & 0x02) ? true : false;
m_lighting_expired = (flags & 0x04) ? true : false;
if(version >= 18)
m_generated = (flags & 0x08) ? false : true;
// Uncompress data
std::ostringstream os(std::ios_base::binary);
decompress(is, os, version);
std::string s = os.str();
if(s.size() != nodecount*3)
throw SerializationError
("MapBlock::deSerialize: decompress resulted in size"
" other than nodecount*3");
// deserialize nodes from buffer
for(u32 i=0; i<nodecount; i++)
{
databuf_nodelist[i*ser_length] = s[i];
databuf_nodelist[i*ser_length + 1] = s[i+nodecount];
databuf_nodelist[i*ser_length + 2] = s[i+nodecount*2];
}
/*
NodeMetadata
*/
if(version >= 14)
{
// Ignore errors
try{
if(version <= 15)
{
std::string data = deSerializeString(is);
std::istringstream iss(data, std::ios_base::binary);
m_node_metadata->deSerialize(iss, m_gamedef);
}
else
{
//std::string data = deSerializeLongString(is);
std::ostringstream oss(std::ios_base::binary);
decompressZlib(is, oss);
std::istringstream iss(oss.str(), std::ios_base::binary);
m_node_metadata->deSerialize(iss, m_gamedef);
}
}
catch(SerializationError &e)
{
errorstream<<"WARNING: MapBlock::deSerialize(): Ignoring an error"
<<" while deserializing node metadata"<<std::endl;
}
}
}
// Deserialize node data
for(u32 i=0; i<nodecount; i++)
{
data[i].deSerialize(&databuf_nodelist[i*ser_length], version);
}
if(disk)
{
/*
Versions up from 9 have block objects. (DEPRECATED)
*/
if(version >= 9){
u16 count = readU16(is);
// Not supported and length not known if count is not 0
if(count != 0){
errorstream<<"WARNING: MapBlock::deSerialize_pre22(): "
<<"Ignoring stuff coming at and after MBOs"<<std::endl;
return;
}
}
/*
Versions up from 15 have static objects.
*/
if(version >= 15)
m_static_objects.deSerialize(is);
// Timestamp
if(version >= 17){
setTimestamp(readU32(is));
m_disk_timestamp = m_timestamp;
} else {
setTimestamp(BLOCK_TIMESTAMP_UNDEFINED);
}
// Dynamically re-set ids based on node names
NameIdMapping nimap;
// If supported, read node definition id mapping
if(version >= 21){
nimap.deSerialize(is);
// Else set the legacy mapping
} else {
content_mapnode_get_name_id_mapping(&nimap);
}
correctBlockNodeIds(&nimap, data, m_gamedef);
}
// Legacy data changes
// This code has to convert from pre-22 to post-22 format.
INodeDefManager *nodedef = m_gamedef->ndef();
for(u32 i=0; i<nodecount; i++)
{
const ContentFeatures &f = nodedef->get(data[i].getContent());
// Mineral
if(nodedef->getId("default:stone") == data[i].getContent()
&& data[i].getParam1() == 1)
{
data[i].setContent(nodedef->getId("default:stone_with_coal"));
data[i].setParam1(0);
}
else if(nodedef->getId("default:stone") == data[i].getContent()
&& data[i].getParam1() == 2)
{
data[i].setContent(nodedef->getId("default:stone_with_iron"));
data[i].setParam1(0);
}
// facedir_simple
if(f.legacy_facedir_simple)
{
data[i].setParam2(data[i].getParam1());
data[i].setParam1(0);
}
// wall_mounted
if(f.legacy_wallmounted)
{
u8 wallmounted_new_to_old[8] = {0x04, 0x08, 0x01, 0x02, 0x10, 0x20, 0, 0};
u8 dir_old_format = data[i].getParam2();
u8 dir_new_format = 0;
for(u8 j=0; j<8; j++)
{
if((dir_old_format & wallmounted_new_to_old[j]) != 0)
{
dir_new_format = j;
break;
}
}
data[i].setParam2(dir_new_format);
}
}
}
int main(int argc, char **argv)
{
if (argc < 2)
{
return 1;
}
{
// Queue
fileQueue = new PathQueue();
for (int i = 1; i < argc; ++i)
{
boost::filesystem::path argPath(argv[i]);
if (boost::filesystem::exists(argPath)) // Does the file exist?
{
if (boost::filesystem::is_regular_file(argPath))
{
// TODO: NOT YET IMPLEMENTED
// Execute file process
#ifdef FILEIO
std::cout << std::endl << "Reading file" << argPath << std::endl;
#endif
fileQueue->push(argPath);
}
else if (boost::filesystem::is_directory(argPath))
{
// TODO: NOT YET IMPLEMENTED
// Execute on all files in the directory
recurseProcessor(argPath);
}
else
{
return -1;
}
}
}
}
auto lz4Proc = new LZ4::Processor();
std::ifstream fIn;
std::ofstream fOut;
size_t fBeg, fEnd;
std::string oName;
std::string::size_type pAt;
bool bIsCompressed;
char *data;
while (fileQueue->size() > 0)
{
// FIRST THINGS
oName = fileQueue->accPop().string();
bIsCompressed = boost::algorithm::ends_with(oName,".lz4");
fIn.open(oName,std::ios::binary);
if (!fIn.is_open())
{
continue;
}
fIn.seekg(0, std::ios::end);
fEnd = fIn.tellg();
fIn.seekg(0, std::ios::beg);
fBeg = fIn.tellg();
data = new char[(fEnd - fBeg)];
fIn.read(data, fEnd - fBeg);
fIn.close();
std::cout << "File is " << fEnd - fBeg << std::endl;
if (bIsCompressed)
{
lz4Proc->decompress(data, int(fEnd - fBeg));
pAt = oName.find_last_of('.');
oName = oName.substr(0, pAt);
}
else
{
lz4Proc->compress(data, int(fEnd - fBeg));
oName = oName + ".lz4";
}
delete data;
if (lz4Proc->len() == 0) continue;
std::cout << lz4Proc->len() << std::endl;
fOut.open(oName, std::ios::binary);
if (!fOut.is_open()) continue;
fOut.write(lz4Proc->ptr(), lz4Proc->len());
fOut.close();
}
delete fileQueue;
getchar();
return 0;
}
int main(void)
{
aplist *head = NULL;
aplist *cur;
bool attached = true;
// change this to false if you are only testing servos
bool wifi_scan = true;
// change this for emulator
bool emulate = false;
float pain = 0.9f;
time_t update = 0;
uint8 num;
uint8 servo_pos = 0;
uint8 current_servo = 1;
unsigned char SERVO_PINS[3] = { SERVO_PIN1 ,SERVO_PIN2 ,SERVO_PIN3 } ;
uint16 val[3] = { 0 };
uint8 i;
touchPosition touch;
videoSetMode(MODE_4_2D);
vramSetBankA(VRAM_A_MAIN_BG);
// set up our bitmap background
bgInit(3, BgType_Bmp16, BgSize_B16_256x256, 0,0);
decompress(cclogoBitmap, BG_GFX, LZ77Vram);
// initialise lower screen for textoutput
consoleDemoInit();
if (emulate == false) {
iprintf("Initializing WiFi.. ");
Wifi_InitDefault(false);
while (Wifi_CheckInit() != 1) {
}
Wifi_ScanMode();
iprintf("done\n");
iprintf("Initializing DS brut.. ");
uart_init();
uart_set_spi_rate(1000);
iprintf("done\n\n\n");
iprintf("Using servo pins: %u %u %u\n\n", SERVO_PIN1, SERVO_PIN2, SERVO_PIN3);
iprintf("Default pain multiplier: %.2f\n\n", pain);
swiDelay(60000000);
while (1) {
scanKeys();
touchRead(&touch);
if (keysDown() & KEY_X) {
if (attached) {
servo_detach(SERVO_PIN1);
servo_detach(SERVO_PIN2);
servo_detach(SERVO_PIN3);
attached = false;
} else {
attached = true;
}
}
if (keysDown() & KEY_A) {
if (attached) {
uint8 i = 0;
for (i=0;i<3;i++) {
servo_set(SERVO_PINS[i],0);
}
}
//servo_set(SERVO_PIN1, 180-((rand() % 100)+(rand() % 50)+(rand() % 25)));
//servo_set(SERVO_PIN2, 180-((rand() % 100)+(rand() % 50)+(rand() % 25)));
//servo_set(SERVO_PIN3, 180-((rand() % 100)+(rand() % 50)+(rand() % 25)));
}
if (keysDown() & KEY_B) {
if (wifi_scan == true) {
wifi_scan = false;
}
else {
wifi_scan = true;
}
}
if (keysDown() & KEY_DOWN) {
pain -= 0.1f;
if (pain < 0.0f) {
pain = 0.0f;
}
update = time(NULL);
}
if (keysDown() & KEY_UP) {
pain += 0.1f;
if (2.0f < pain) {
pain = 2.0f;
}
update = time(NULL);
}
if (keysDown() & KEY_L) {
current_servo += 1;
if (current_servo > 3) {
current_servo = 1;
}
}
consoleClear();
if (wifi_scan == true) {
num = 0;
cur = head;
iprintf("\n");
while (cur && num < 15) {
// display
if (!(cur->flags & 0x2)) {
cur = cur->next;
continue;
}
iprintf("%2u ", num);
if (cur->ssid[0] == '\0') {
iprintf("%02x%02x%02x%02x%02x%02x", cur->mac[0], cur->mac[1], cur->mac[2], cur->mac[3], cur->mac[4], cur->mac[5]);
} else {
iprintf("%s", cur->ssid);
}
iprintf(" @ %u", cur->rssi);
if ((cur->flags & 0x06) == 0x06) {
iprintf(" WPA");
} else if (cur->flags & 0x02) {
iprintf(" WEP");
}
// calculate servo commands
if (attached && num < 3) {
val[num] = (uint16)(cur->rssi*pain);
if (180 < val[num]) {
val[num] = 180;
}
iprintf(" %u", val[num]);
}
iprintf("\n");
if (num == 2) {
iprintf ("\n");
}
num++;
cur = cur->next;
}
iprintf("\n");
if (time(NULL) < update+3) {
printf("\npain multiplier: %.2f\n", pain);
}
// set the servo to zero if we don't have enough wifi nodes
for (i=num; i<3; i++) {
val[i] = 0;
}
if (attached) {
servo_set(SERVO_PIN1, (uint8)val[0]);
servo_set(SERVO_PIN2, 180-(uint8)val[1]);
servo_set(SERVO_PIN3, (uint8)val[2]);
}
updateApList(&head, 0x00, NULL, NULL);
sortApList(&head, false);
}
if (KEY_TOUCH && attached && !wifi_scan) {
servo_pos = (touch.rawx / 3850.) * 180;
iprintf ("servo pos x: %d;", servo_pos);
iprintf ("\n");
servo_set(SERVO_PINS[current_servo-1],servo_pos);
iprintf ("current servo: %d", current_servo);
iprintf ("\n\n");
}
swiWaitForVBlank();
}
}
while (1) {
swiWaitForVBlank();
}
return 0;
}
void ZlibFile::createDecompressedMemoryFile()
{
MemoryFile * memoryFile = new MemoryFile(decompress(), m_uncompressedLength);
close();
m_decompressedMemoryFile = memoryFile;
}
int do_decompress(u8 *input, int len, u8 *output, void (*error)(char *x))
{
return decompress(input, len, NULL, NULL, output, NULL, error);
}
int main(void)
{
int n,
cport_nr = 4, /* 6 (COM5 on windows) */
bdrate = 38400; /* 9600 baud */
uint8_t buf[BUF_SIZE + 1];
uint8_t data[MAX_DATA_SIZE];
char mode[] = {'8','N','1',0};
if (RS232_OpenComport(cport_nr, bdrate, mode)) {
printf("Can not open comport\n");
return(0);
}
int received_cnt = 0;
while (1) {
n = RS232_PollComport(cport_nr, buf, BUF_SIZE);
if (n > 0) {
/* always put a "null" at the end of a string! */
buf[n] = 0;
printf("Received %i bytes: %s\n", n, (char *) buf);
if (arr_search("TKENDTKENDTKENDTKEND", BUF_SIZE, buf, n) > 0) {
printf("%s\n", "Starting reception...");
memset(data, 0, MAX_DATA_SIZE); /* Initialize the array */
received_cnt = 0;
}
else {
int pos = -1;
/* If receiving the end of current compressed buffer, send & reinitialize */
if ((pos = arr_search("TKENDTKENDTKENDTKE", DELIMITER_LEN, buf, n)) >= 0) {
int eff_len = pos - DELIMITER_LEN + 1;
uint8_t temp[eff_len];
memcpy(temp, buf, eff_len);
memcpy(data + received_cnt, temp, eff_len);
received_cnt += eff_len;
/* Discard the last five bytes of indicators*/
char size_buf[2];
memcpy(size_buf, buf + pos + 1, 2);
int size = size_buf[1] + (size_buf[0] << 4);
printf("Received data total size: %d\n", size);
uint8_t comp[size];
memcpy(comp, data, size);
decompress(comp, size);
/* Clean up */
memset(data, 0, MAX_DATA_SIZE);
received_cnt = 0;
/* Also need to store rest of the data to avoid loss */
// uint8_t lost[n - eff_len - 5];
// memcpy(lost, buf + pos + 1, n - pos - 1);
// memcpy(data, lost, n - pos - 1);
// received_cnt += n - pos - 1;
} else { /* If regular data packets, store it*/
memcpy(data + received_cnt, buf, n);
received_cnt = received_cnt + n;
}
}
}
#ifdef _WIN32
Sleep(100);
#else
usleep(100000); /* sleep for 100 milliSeconds */
#endif
}
return (0);
}
int PFSLoader::Open(FILE *fp) {
struct_header s3d_header;
struct_directory_header s3d_dir_header;
struct_directory s3d_dir;
struct_data_block s3d_data;
struct_fn_header *s3d_fn_header;
struct_fn_entry *s3d_fn_entry;
uint32 *offsets;
char *temp, *temp2;
int i, j, pos, inf, tmp, running = 0;
fread(&s3d_header, sizeof(struct_header), 1, fp);
if(s3d_header.magicCookie[0] != 'P' || s3d_header.magicCookie[1] != 'F' || s3d_header.magicCookie[2] != 'S' || s3d_header.magicCookie[3] != ' ')
return 0;
this->fp = fp;
fseek(fp, s3d_header.offset, SEEK_SET);
fread(&s3d_dir_header, sizeof(struct_directory_header), 1, fp);
this->count = s3d_dir_header.count - 1;
this->filenames = new char *[s3d_dir_header.count];
this->files = new uint32[s3d_dir_header.count - 1];
offsets = new uint32[s3d_dir_header.count - 1];
for(i = 0; i < (int)s3d_dir_header.count; ++i) {
fread(&s3d_dir, sizeof(struct_directory), 1, fp);
if(s3d_dir.crc == ntohl(0xC90A5861)) {
pos = ftell(fp);
fseek(fp, s3d_dir.offset, SEEK_SET);
temp = new char[s3d_dir.size];
memset(temp, 0, s3d_dir.size);
inf = 0;
while(inf < (int)s3d_dir.size) {
fread(&s3d_data, sizeof(struct_data_block), 1, fp);
temp2 = new char[s3d_data.deflen];
fread(temp2, s3d_data.deflen, 1, fp);
decompress(temp2, temp + inf, s3d_data.deflen, s3d_data.inflen);
delete[] temp2;
inf += s3d_data.inflen;
}
fseek(fp, pos, SEEK_SET);
s3d_fn_header = (struct_fn_header *) temp;
pos = sizeof(struct_fn_header);
for(j = 0; j < (int)s3d_fn_header->fncount; ++j) {
s3d_fn_entry = (struct_fn_entry *) &temp[pos];
this->filenames[j] = new char[s3d_fn_entry->fnlen + 1];
this->filenames[j][s3d_fn_entry->fnlen] = 0;
memcpy(this->filenames[j], &temp[pos + sizeof(struct_fn_entry)], s3d_fn_entry->fnlen);
pos += sizeof(struct_fn_entry) + s3d_fn_entry->fnlen;
}
}
else {
this->files[running] = ftell(fp) - 12;
offsets[running] = s3d_dir.offset;
++running;
}
}
for(i = s3d_dir_header.count - 2; i > 0; i--) {
for(j = 0; j < i; j++) {
if(offsets[j] > offsets[j+1]) {
tmp = offsets[j];
offsets[j] = offsets[j + 1];
offsets[j + 1] = tmp;
tmp = this->files[j];
this->files[j] = this->files[j + 1];
this->files[j + 1] = tmp;
}
}
}
return 1;
}
std::vector<uint8_t> decompress(const std::vector<uint8_t>& data)
{
return decompress(&data[0], data.size());
}
void BitVector::rleANDnon(BitVector& bv) {
// decompress
// run nonANDnon
decompress();
nonANDnon(bv);
}
/* Entry point for the program. Opens the file, and executes the mode specified by command line args */
int main(int argc, char* argv[])
{
if(argc != 3 || (strcmp(argv[1], "-c") != 0 && strcmp(argv[1], "-d") != 0 && strcmp(argv[1], "-t") != 0))
{
printf("usage: huff [-c | -d | -t] file\n");
return -1;
}
/* Execute the correct mode */
if(strcmp(argv[1], "-c") == 0)
{
//append the extension to the name
const char * extension = ".temp";
char *tempFile = malloc(strlen(argv[2])+strlen(extension) + 1);
strncpy(tempFile, argv[2], strlen(argv[2]));
strcat(tempFile, extension);
//RLE encode the file
rle_encode(argv[2], tempFile);
//create a vairiable to hold hold the file length
unsigned long long fileLength = 0;
//get a buffer of the contents of the file as a unsigned char*
unsigned char *file_pointer = openFile(tempFile, &fileLength);
//huff compress the output.
compress(file_pointer, fileLength, argv[2]);
free(file_pointer);
//free(tempFile);
remove(tempFile);
}
else if(strcmp(argv[1], "-d") == 0)
{
unsigned long long fileLength = 0;
unsigned char *file_pointer = openFile(argv[2], &fileLength);
decompress(file_pointer, fileLength, argv[2]);
//remove the .hurl extension from the fileName;
char *tempFileName = calloc(strlen(argv[2]), sizeof(char));
strncpy(tempFileName, argv[2], strlen(argv[2]) - strlen(".hurl"));
strncat(tempFileName, ".temp", sizeof(".temp"));
char *outputFileName = calloc(sizeof(char), strlen(argv[2]));
strncpy(outputFileName, argv[2], strlen(argv[2]));
outputFileName[strlen(outputFileName) -5] = '\0';
rle_decode(tempFileName, outputFileName);
free(file_pointer);
remove(tempFileName);
}
else if(strcmp(argv[1], "-t") == 0)
{
unsigned long long fileLength = 0;
unsigned char *file_pointer = openFile(argv[2], &fileLength);
print_table(file_pointer, fileLength, argv[2]);
free(file_pointer);
}
// free(file_pointer);
return 0;
}
std::pair<char*, size_t> LZ4RunReader::next()
{
// Is there a complete key in the buffer?
if( decomp_.fill() >= sizeof(size_t) &&
decomp_.fill() >= (sizeof(size_t) + next_size()) )
{
auto ret = std::make_pair(decomp_.base() + decomp_.lo() + sizeof(size_t),
next_size());
decomp_.advance_lo(ret.second + sizeof(size_t));
return ret;
}
// No key. Did we hit eof?
else if( eof_ )
{
if( decomp_.fill() )
{
WARNING("Run file had " << decomp_.fill() << " extraneous bytes at end");
}
if( comp_.fill() )
{
WARNING("Run file consumed with " << comp_.fill() << " compressed"
" bytes remaining");
}
return std::pair<char*, size_t>(nullptr, 0);
}
// We need some more data
else
{
// Fill buffer until we have hit the trigger point, and we have a
// a complete key
while( !eof_ &&
( decomp_.fill() < trigger_ ||
decomp_.fill() < (sizeof(size_t) + next_size()) ) )
{
// First try to decompress that which we have
if( comp_.fill() )
{
decompress();
// Skip read if we got enough data
if( decomp_.fill() >= trigger_ &&
decomp_.fill() >= (sizeof(size_t) + next_size()) )
{
continue;
}
}
if( poll(fds_, 1, -1) < 0 )
{
WARNING("poll() failed, input may have terminated prematurely.");
eof_ = true;
}
// Read into the compressed buffer
int bytes_read = read(fds_[0].fd, comp_.base() + comp_.hi(),
comp_.size() - comp_.fill());
if( bytes_read <= 0 )
{
eof_ = true;
if( bytes_read < 0 )
{
WARNING("read() failed, input may have terminated prematurely.");
}
}
else
{
comp_.advance_hi(bytes_read);
decompress();
}
}
// Warn if eof without a complete key
if( eof_ && ( decomp_.fill() < sizeof(size_t) ||
decomp_.fill() < (sizeof(size_t) + next_size()) ) )
{
WARNING("Run file had " << decomp_.fill() << " extraneous bytes at end");
return std::pair<char*, size_t>(nullptr, 0);
}
// We now have at least one key in the buffer. Return the first.
auto ret = std::make_pair(decomp_.base() + decomp_.lo() + sizeof(size_t),
next_size());
decomp_.advance_lo(ret.second + sizeof(size_t));
return ret;
}
}
void TPC::fixupCubeMap() {
/* Do various fixups to the cube maps. This includes rotating and swapping a
* few sides around. This is done by the original games as well.
*/
if (!isCubeMap())
return;
for (size_t j = 0; j < getMipMapCount(); j++) {
assert(getLayerCount() > 0);
const size_t index0 = 0 * getMipMapCount() + j;
assert(index0 < _mipMaps.size());
const int32 width = _mipMaps[index0]->width;
const int32 height = _mipMaps[index0]->height;
const uint32 size = _mipMaps[index0]->size;
for (size_t i = 1; i < getLayerCount(); i++) {
const size_t index = i * getMipMapCount() + j;
assert(index < _mipMaps.size());
if ((width != _mipMaps[index]->width ) ||
(height != _mipMaps[index]->height) ||
(size != _mipMaps[index]->size ))
throw Common::Exception("Cube map layer dimensions mismatch");
}
}
// Since we need to rotate the individual cube sides, we need to decompress them all
decompress();
// Swap the first two sides of the cube maps
for (size_t j = 0; j < getMipMapCount(); j++) {
const size_t index0 = 0 * getMipMapCount() + j;
const size_t index1 = 1 * getMipMapCount() + j;
assert((index0 < _mipMaps.size()) && (index1 < _mipMaps.size()));
MipMap &mipMap0 = *_mipMaps[index0];
MipMap &mipMap1 = *_mipMaps[index1];
SWAP(mipMap0.data, mipMap1.data);
}
const int bpp = (_formatRaw == kPixelFormatRGB8) ? 3 : ((_formatRaw == kPixelFormatRGBA8) ? 4 : 0);
if (bpp == 0)
return;
// Rotate the cube sides so that they're all oriented correctly
for (size_t i = 0; i < getLayerCount(); i++) {
for (size_t j = 0; j < getMipMapCount(); j++) {
const size_t index = i * getMipMapCount() + j;
assert(index < _mipMaps.size());
MipMap &mipMap = *_mipMaps[index];
static const int rotation[6] = { 1, 3, 0, 2, 2, 0 };
rotate90(mipMap.data, mipMap.width, mipMap.height, bpp, rotation[i]);
}
}
}
static int copy_one_extent(struct btrfs_root *root, int fd,
struct extent_buffer *leaf,
struct btrfs_file_extent_item *fi, u64 pos)
{
struct btrfs_multi_bio *multi = NULL;
struct btrfs_device *device;
char *inbuf, *outbuf = NULL;
ssize_t done, total = 0;
u64 bytenr;
u64 ram_size;
u64 disk_size;
u64 length;
u64 size_left;
u64 dev_bytenr;
u64 count = 0;
int compress;
int ret;
int dev_fd;
compress = btrfs_file_extent_compression(leaf, fi);
bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
disk_size = btrfs_file_extent_disk_num_bytes(leaf, fi);
ram_size = btrfs_file_extent_ram_bytes(leaf, fi);
size_left = disk_size;
/* we found a hole */
if (disk_size == 0)
return 0;
inbuf = malloc(disk_size);
if (!inbuf) {
fprintf(stderr, "No memory\n");
return -1;
}
if (compress != BTRFS_COMPRESS_NONE) {
outbuf = malloc(ram_size);
if (!outbuf) {
fprintf(stderr, "No memory\n");
free(inbuf);
return -1;
}
}
again:
length = size_left;
ret = btrfs_map_block(&root->fs_info->mapping_tree, READ,
bytenr, &length, &multi, 0, NULL);
if (ret) {
free(inbuf);
free(outbuf);
fprintf(stderr, "Error mapping block %d\n", ret);
return ret;
}
device = multi->stripes[0].dev;
dev_fd = device->fd;
device->total_ios++;
dev_bytenr = multi->stripes[0].physical;
kfree(multi);
if (size_left < length)
length = size_left;
size_left -= length;
done = pread(dev_fd, inbuf+count, length, dev_bytenr);
if (done < length) {
free(inbuf);
free(outbuf);
fprintf(stderr, "Short read %d\n", errno);
return -1;
}
count += length;
bytenr += length;
if (size_left)
goto again;
if (compress == BTRFS_COMPRESS_NONE) {
while (total < ram_size) {
done = pwrite(fd, inbuf+total, ram_size-total,
pos+total);
if (done < 0) {
free(inbuf);
fprintf(stderr, "Error writing: %d %s\n", errno, strerror(errno));
return -1;
}
total += done;
}
free(inbuf);
return 0;
}
ret = decompress(inbuf, outbuf, disk_size, ram_size);
free(inbuf);
if (ret) {
free(outbuf);
return ret;
}
while (total < ram_size) {
done = pwrite(fd, outbuf+total, ram_size-total, pos+total);
if (done < 0) {
free(outbuf);
fprintf(stderr, "Error writing: %d %s\n", errno, strerror(errno));
return -1;
}
total += done;
}
free(outbuf);
return 0;
}
Id
repo_add_deb(Repo *repo, const char *deb, int flags)
{
Pool *pool = repo->pool;
Repodata *data;
unsigned char buf[4096], *bp;
int l, l2, vlen, clen, ctarlen;
unsigned char *ctgz;
unsigned char pkgid[16];
unsigned char *ctar;
int gotpkgid;
FILE *fp;
Solvable *s;
struct stat stb;
data = repo_add_repodata(repo, flags);
if ((fp = fopen(flags & REPO_USE_ROOTDIR ? pool_prepend_rootdir_tmp(pool, deb) : deb, "r")) == 0)
{
pool_error(pool, -1, "%s: %s", deb, strerror(errno));
return 0;
}
if (fstat(fileno(fp), &stb))
{
pool_error(pool, -1, "fstat: %s", strerror(errno));
fclose(fp);
return 0;
}
l = fread(buf, 1, sizeof(buf), fp);
if (l < 8 + 60 || strncmp((char *)buf, "!<arch>\ndebian-binary ", 8 + 16) != 0)
{
pool_error(pool, -1, "%s: not a deb package", deb);
fclose(fp);
return 0;
}
vlen = atoi((char *)buf + 8 + 48);
if (vlen < 0 || vlen > l)
{
pool_error(pool, -1, "%s: not a deb package", deb);
fclose(fp);
return 0;
}
vlen += vlen & 1;
if (l < 8 + 60 + vlen + 60)
{
pool_error(pool, -1, "%s: unhandled deb package", deb);
fclose(fp);
return 0;
}
if (strncmp((char *)buf + 8 + 60 + vlen, "control.tar.gz ", 16) != 0)
{
pool_error(pool, -1, "%s: control.tar.gz is not second entry", deb);
fclose(fp);
return 0;
}
clen = atoi((char *)buf + 8 + 60 + vlen + 48);
if (clen <= 0 || clen >= 0x100000)
{
pool_error(pool, -1, "%s: control.tar.gz has illegal size", deb);
fclose(fp);
return 0;
}
ctgz = solv_calloc(1, clen + 4);
bp = buf + 8 + 60 + vlen + 60;
l -= 8 + 60 + vlen + 60;
if (l > clen)
l = clen;
if (l)
memcpy(ctgz, bp, l);
if (l < clen)
{
if (fread(ctgz + l, clen - l, 1, fp) != 1)
{
pool_error(pool, -1, "%s: unexpected EOF", deb);
solv_free(ctgz);
fclose(fp);
return 0;
}
}
fclose(fp);
gotpkgid = 0;
if (flags & DEBS_ADD_WITH_PKGID)
{
Chksum *chk = solv_chksum_create(REPOKEY_TYPE_MD5);
solv_chksum_add(chk, ctgz, clen);
solv_chksum_free(chk, pkgid);
gotpkgid = 1;
}
if (ctgz[0] != 0x1f || ctgz[1] != 0x8b)
{
pool_error(pool, -1, "%s: control.tar.gz is not gzipped", deb);
solv_free(ctgz);
return 0;
}
if (ctgz[2] != 8 || (ctgz[3] & 0xe0) != 0)
{
pool_error(pool, -1, "%s: control.tar.gz is compressed in a strange way", deb);
solv_free(ctgz);
return 0;
}
bp = ctgz + 4;
bp += 6; /* skip time, xflags and OS code */
if (ctgz[3] & 0x04)
{
/* skip extra field */
l = bp[0] | bp[1] << 8;
bp += l + 2;
if (bp >= ctgz + clen)
{
pool_error(pool, -1, "%s: control.tar.gz is corrupt", deb);
solv_free(ctgz);
return 0;
}
}
if (ctgz[3] & 0x08) /* orig filename */
while (*bp)
bp++;
if (ctgz[3] & 0x10) /* file comment */
while (*bp)
bp++;
if (ctgz[3] & 0x02) /* header crc */
bp += 2;
if (bp >= ctgz + clen)
{
pool_error(pool, -1, "%s: control.tar.gz is corrupt", deb);
solv_free(ctgz);
return 0;
}
ctar = decompress(bp, ctgz + clen - bp, &ctarlen);
solv_free(ctgz);
if (!ctar)
{
pool_error(pool, -1, "%s: control.tar.gz is corrupt", deb);
return 0;
}
bp = ctar;
l = ctarlen;
while (l > 512)
{
int j;
l2 = 0;
for (j = 124; j < 124 + 12; j++)
if (bp[j] >= '0' && bp[j] <= '7')
l2 = l2 * 8 + (bp[j] - '0');
if (!strcmp((char *)bp, "./control") || !strcmp((char *)bp, "control"))
break;
l2 = 512 + ((l2 + 511) & ~511);
l -= l2;
bp += l2;
}
if (l <= 512 || l - 512 - l2 <= 0 || l2 <= 0)
{
pool_error(pool, -1, "%s: control.tar.gz contains no control file", deb);
free(ctar);
return 0;
}
memmove(ctar, bp + 512, l2);
ctar = solv_realloc(ctar, l2 + 1);
ctar[l2] = 0;
s = pool_id2solvable(pool, repo_add_solvable(repo));
control2solvable(s, data, (char *)ctar);
if (!(flags & REPO_NO_LOCATION))
repodata_set_location(data, s - pool->solvables, 0, 0, deb);
if (S_ISREG(stb.st_mode))
repodata_set_num(data, s - pool->solvables, SOLVABLE_DOWNLOADSIZE, (unsigned long long)stb.st_size);
if (gotpkgid)
repodata_set_bin_checksum(data, s - pool->solvables, SOLVABLE_PKGID, REPOKEY_TYPE_MD5, pkgid);
solv_free(ctar);
if (!(flags & REPO_NO_INTERNALIZE))
repodata_internalize(data);
return s - pool->solvables;
}
term_ps_zzn* Ps_ZZn::addterm(const ZZn& a,int power,term_ps_zzn* pos)
{
term_ps_zzn* newone;
term_ps_zzn* ptr;
term_ps_zzn *t,*iptr;
int dc,pw;
ptr=start;
iptr=NULL;
//
// intelligently determine the most compressed form to use
// for example if coefficient a=1 always, and power = -7 -5 -3 -1 1 3....
// then set pwr=2, offset=7 and PS = 1 + x + x^2
//
pw=power+offset;
if (one_term() && pw!=0)
{ // when PS has only one term, pwr is undefined
if (pw<0)
pwr=-pw;
else pwr=pw;
}
dc=igcd(pw,pwr);
if (dc != pwr) decompress(pwr/dc);
power=pw/pwr;
// quick scan through to detect if term exists already
// and to find insertion point
if (pos!=NULL) ptr=pos;
while (ptr!=NULL)
{
if (ptr->n==power)
{
ptr->an+=a;
if (ptr->an.iszero())
{ // delete term
if (ptr==start)
{ // delete first one
start=ptr->next;
delete ptr;
norm();
return start;
}
iptr=ptr;
ptr=start;
while (ptr->next!=iptr)ptr=ptr->next;
ptr->next=iptr->next;
delete iptr;
return ptr;
}
return ptr;
}
if (ptr->n<power) iptr=ptr; // determines order
else break;
ptr=ptr->next;
}
newone=new term_ps_zzn;
newone->next=NULL;
newone->an=a;
newone->n=power;
pos=newone;
if (start==NULL)
{
start=newone;
norm();
return pos;
}
// insert at the start
if (iptr==NULL)
{
t=start;
start=newone;
newone->next=t;
norm();
return pos;
}
// insert new term
t=iptr->next;
iptr->next=newone;
newone->next=t;
return pos;
}
Ps_ZZn& Ps_ZZn::operator*=(Ps_ZZn& b)
{
term_ps_zzn *ptr,*bptr;
int g,d,deg,ka,kb;
if (IsInt())
{
if (start!=NULL) *this = start->an*b;
return *this;
}
if (b.IsInt())
{
if (b.start==NULL) clear();
else *this *=(b.start->an);
return *this;
}
g=igcd(pwr,b.pwr);
deg=psN/g;
#ifdef FFT
if (deg>=FFT_BREAK_EVEN)
{
big *A,*B;
A=(big *)mr_alloc(deg,sizeof(big));
B=(big *)mr_alloc(deg,sizeof(big));
ka=pwr/g;
decompress(ka);
pad();
ptr=start;
while (ptr!=NULL)
{
d=ptr->n;
if (d>=deg) break;
A[d]=getbig(ptr->an);
ptr=ptr->next;
}
kb=b.pwr/g;
b.decompress(kb);
bptr=b.start;
while (bptr!=NULL)
{
d=bptr->n;
if (d>=deg) break;
B[d]=getbig(bptr->an);
bptr=bptr->next;
}
mr_ps_zzn_mul(deg,A,B,A);
mr_free(B); mr_free(A);
b.compress(kb);
}
else
{
#endif
*this=*this*b;
return *this;
#ifdef FFT
}
#endif
norm();
offset+=b.offset;
return *this;
}
void ParseXpck(FILE *infile, bool quietMode)
{
ArchiveHeaderXpck header = { 0 };
int archiveSize = 0;
fseek(infile,0,SEEK_END);
archiveSize = ftell(infile);
rewind(infile);
fread(header.magic, 1, 4, infile);
fread(&header.fileCount, 1, 1, infile);
fread(&header.unknown, 1, 1, infile);
fread(&header.fileInfoOffset, 1, 2, infile);
fread(&header.filenameTableOffset, 1, 2, infile);
fread(&header.dataOffset, 1, 2, infile);
fread(&header.fileInfoSize, 1, 2, infile);
fread(&header.filenameTableSize, 1, 2, infile);
fread(&header.dataSize, 1, 4, infile);
header.fileInfoOffset *= 4;
header.filenameTableOffset *= 4;
header.dataOffset *= 4;
header.fileInfoSize *= 4;
header.filenameTableSize *= 4;
header.dataSize *= 4;
ParseFilenames(infile, header);
ParseFileEntries(infile, header);
// Create all of the possible directory paths in one go instead of trying to create
// each full path during the file dumping loop
for(int i = 0; i < foldernames.size(); i++)
{
CreateFullPath(foldernames[i]);
}
std::string currentFolder = "";
int dumped = 0;
char *buffer = NULL;
int bufferSize = 0;
for(int i = 0; i < filenames.size(); i++)
{
if(filenames[i][filenames[i].size() - 1] == '/')
{
currentFolder = filenames[i];
}
else
{
std::string fullpath = currentFolder + filenames[i];
unsigned int crc32 = CalculateCrc32((char*)filenames[i].c_str(), filenames[i].size());
if(file_index.find(crc32) != file_index.end())
{
int idx = file_index[crc32];
if(file_entries[idx].filesize > bufferSize)
{
bufferSize = file_entries[idx].filesize * 2;
if(buffer == NULL)
{
buffer = (char*)calloc(bufferSize, sizeof(char));
}
else
{
buffer = (char*)realloc(buffer, bufferSize);
}
}
if(header.dataOffset + file_entries[idx].offset > archiveSize)
{
//printf("Couldn't seek passed end of file\n");
continue;
}
fseek(infile,header.dataOffset + file_entries[idx].offset,SEEK_SET);
fread(buffer, 1, file_entries[idx].filesize, infile);
// Try to decompress data
if(memcmp(buffer + 4, "\0DVLB", 5) == 0 ||
memcmp(buffer + 4, "\0XPCK", 5) == 0 ||
memcmp(buffer + 4, "\0CHRC", 5) == 0 ||
memcmp(buffer + 4, "\0RESC", 5) == 0)
{
int length = *(int*)buffer;
length /= 8;
int output_len = 0;
char *data = decompress(buffer, file_entries[idx].filesize, &output_len);
// Replace the running buffer with the decompressed data's buffer.
// Lazy way to handle this without changign the code after.
free(buffer);
buffer = data;
file_entries[idx].filesize = output_len;
bufferSize = output_len;
}
if(!quietMode)
{
printf("[%06d] %-38s size[%08x] offset[%08x]\n", dumped, fullpath.c_str(), file_entries[idx].filesize, file_entries[idx].offset);
}
dumped++;
FILE *outfile = fopen(fullpath.c_str(), "wb");
if(!outfile)
{
printf("Could not open %s\n", fullpath.c_str());
exit(-1);
}
fwrite(buffer, 1, file_entries[idx].filesize, outfile);
fclose(outfile);
}
else
{
printf("Could not find crc32 %08x: %s\n",crc32, filenames[i].c_str());
exit(1);
}
}
}
if(buffer != NULL)
free(buffer);
}
int main(int argc, char* argv[])
{
if (argc != 4)
{
puts("Usage: ");
printf(" Compress : %s c sourcefile destfile\n", argv[0]);
printf(" Decompress : %s d sourcefile destfile\n", argv[0]);
return 0;
}
BYTE soubuf[65536];
BYTE destbuf[65536 + 16];
FILE* in;
FILE* out;
//打开源文件
in = fopen(argv[2], "rb");
if (in == NULL)
{
puts("Can't open source file");
return 0;
}
//打开目标文件
out = fopen(argv[3], "wb");
if (out == NULL)
{
puts("Can't open dest file");
fclose(in);
return 0;
}
fseek(in, 0, SEEK_END);
long soulen = ftell(in);
fseek(in, 0, SEEK_SET);
SortHeap = new struct STIDXNODE[_MAX_WINDOW_SIZE];
WORD flag1, flag2;
if (argv[1][0] == 'c') // 对源文件进行压缩
{
int last = soulen, act;
while ( last > 0 )
{
act = min(65536, last);
fread(soubuf, act, 1, in);
last -= act;
if (act == 65536) // 超过65536字节
flag1 = 0;
else
flag1 = act;
fwrite(&flag1, sizeof(WORD), 1, out);
int destlen = compress((BYTE*)soubuf, act, (BYTE*)destbuf);
if (destlen == 0) // 不能压缩块
{
flag2 = flag1;
fwrite(&flag2, sizeof(WORD), 1, out);
fwrite(soubuf, act, 1, out);
}
else
{
flag2 = (WORD)destlen;
fwrite(&flag2, sizeof(WORD), 1, out);
fwrite(destbuf, destlen, 1, out);
}
}
}
else if (argv[1][0] == 'd') // 解压缩源文件
{
int last = soulen, act;
while (last > 0)
{
fread(&flag1, sizeof(WORD), 1, in);
fread(&flag2, sizeof(WORD), 1, in);
last -= 2 * sizeof(WORD);
if (flag1 == 0)
act = 65536;
else
act = flag1;
last-= flag2 ? (flag2) : act;
if (flag2 == flag1)
{
fread(soubuf, act, 1, in);
}
else
{
fread(destbuf, flag2, 1, in);
if (!decompress((BYTE*)soubuf, act, (BYTE*)destbuf))
{
puts("Decompress error");
fclose(in);
fclose(out);
return 0;
}
}
fwrite((BYTE*)soubuf, act, 1, out);
}
}
else
{
puts("Usage: ");
printf(" Compress : %s c sourcefile destfile\n", argv[0]);
printf(" Decompress : %s d sourcefile destfile\n", argv[0]);
}
//关闭源文件和目标文件
fclose(in);
fclose(out);
return 0;
}
BinaryInput::BinaryInput(
const std::string& filename,
G3DEndian fileEndian,
bool compressed) :
m_filename(filename),
m_bitPos(0),
m_bitString(0),
m_beginEndBits(0),
m_alreadyRead(0),
m_length(0),
m_bufferLength(0),
m_buffer(NULL),
m_pos(0),
m_freeBuffer(true) {
setEndian(fileEndian);
// Update global file tracker
_internal::currentFilesUsed.insert(m_filename);
if (! fileExists(m_filename, false)) {
std::string zipfile;
std::string internalfile;
if (zipfileExists(m_filename, zipfile, internalfile)) {
// Load from zipfile
void* v;
size_t s;
zipRead(filename, v, s);
m_buffer = reinterpret_cast<uint8*>(v);
m_bufferLength = m_length = s;
if (compressed) {
decompress();
}
m_freeBuffer = true;
} else {
Log::common()->printf("Warning: File not found: %s\n", m_filename.c_str());
}
return;
}
// Figure out how big the file is and verify that it exists.
m_length = fileLength(m_filename);
// Read the file into memory
FILE* file = fopen(m_filename.c_str(), "rb");
if (! file || (m_length == -1)) {
throw format("File not found: \"%s\"", m_filename.c_str());
return;
}
if (! compressed && (m_length > INITIAL_BUFFER_LENGTH)) {
// Read only a subset of the file so we don't consume
// all available memory.
m_bufferLength = INITIAL_BUFFER_LENGTH;
} else {
// Either the length is fine or the file is compressed
// and requires us to read the whole thing for zlib.
m_bufferLength = m_length;
}
debugAssert(m_freeBuffer);
m_buffer = (uint8*)System::alignedMalloc(m_bufferLength, 16);
if (m_buffer == NULL) {
if (compressed) {
throw "Not enough memory to load compressed file. (1)";
}
// Try to allocate a small array; not much memory is available.
// Give up if we can't allocate even 1k.
while ((m_buffer == NULL) && (m_bufferLength > 1024)) {
m_bufferLength /= 2;
m_buffer = (uint8*)System::alignedMalloc(m_bufferLength, 16);
}
}
debugAssert(m_buffer);
fread(m_buffer, m_bufferLength, sizeof(int8), file);
fclose(file);
file = NULL;
if (compressed) {
if (m_bufferLength != m_length) {
throw "Not enough memory to load compressed file. (2)";
}
decompress();
}
}
void pause_cmdline( int argc, char **argv )
{
char lzo_file[MAX_PATH];
char files[MAX_PATH];
bool brecurse;
bool bpath;
if( argc == 1 )
{
show_help();
}
else if( argc == 2 )
{
if( strcmp( argv[1], "-h" ) == 0 || strcmp( argv[1], "-help" ) == 0 )
{
show_help();
}
}
else if( argc == 3 )
{
if( strcmp( argv[1], "-l" ) == 0 )
{
/// display the file info in the archive
get_archive_info( argv[2] );
}
else if( strcmp( argv[1], "-d" ) == 0 )
{
/// decompress files in the current directory
decompress( argv[2], "./" );
}
else
{
/// normal compress files
strncpy( files, argv[1], MAX_PATH );
strncpy( lzo_file, argv[2], MAX_PATH );
compress( lzo_file, files, false, false );
}
}
else if( argc == 4 )
{
if( strcmp( argv[1], "-d" ) == 0 )
{
/// decompress files in the specified directory
decompress( argv[2], argv[3] );
}
else if( strcmp( argv[1], "-vcprj" ) == 0 )
{
/// compress vcprj file
compress_vcproj( argv[3], argv[2] );
}
else if( strcmp( argv[1], "-sln" ) == 0 )
{
/// compress solution file
compress_sln( argv[3], argv[2] );
}
else
{
/// compress files
char cmd[64];
strncpy( cmd, argv[1], 63 );
bpath = strcmp( cmd, "-p" ) == 0 ? true : false;
brecurse = strcmp( cmd, "-r" ) == 0 ? true : false;
if( strcmp( cmd, "-rp" ) == 0 || strcmp( cmd, "-pr" ) == 0 )
{
bpath = brecurse = true;
}
strncpy( files, argv[2], MAX_PATH );
strncpy( lzo_file, argv[3], MAX_PATH );
compress( lzo_file, files, bpath, brecurse );
}
}
}
void MapBlock::deSerialize(std::istream &is, u8 version)
{
if(!ser_ver_supported(version))
throw VersionMismatchException("ERROR: MapBlock format not supported");
// These have no lighting info
if(version <= 1)
{
setLightingExpired(true);
}
// These have no "generated" field
if(version < 18)
{
m_generated = true;
}
// These have no compression
if(version <= 3 || version == 5 || version == 6)
{
u32 nodecount = MAP_BLOCKSIZE*MAP_BLOCKSIZE*MAP_BLOCKSIZE;
char tmp;
is.read(&tmp, 1);
if(is.gcount() != 1)
throw SerializationError
("MapBlock::deSerialize: no enough input data");
is_underground = tmp;
for(u32 i=0; i<nodecount; i++)
{
s32 len = MapNode::serializedLength(version);
SharedBuffer<u8> d(len);
is.read((char*)*d, len);
if(is.gcount() != len)
throw SerializationError
("MapBlock::deSerialize: no enough input data");
data[i].deSerialize(*d, version);
}
}
else if(version <= 10)
{
u32 nodecount = MAP_BLOCKSIZE*MAP_BLOCKSIZE*MAP_BLOCKSIZE;
u8 t8;
is.read((char*)&t8, 1);
is_underground = t8;
{
// Uncompress and set material data
std::ostringstream os(std::ios_base::binary);
decompress(is, os, version);
std::string s = os.str();
if(s.size() != nodecount)
throw SerializationError
("MapBlock::deSerialize: invalid format");
for(u32 i=0; i<s.size(); i++)
{
data[i].param0 = s[i];
}
}
{
// Uncompress and set param data
std::ostringstream os(std::ios_base::binary);
decompress(is, os, version);
std::string s = os.str();
if(s.size() != nodecount)
throw SerializationError
("MapBlock::deSerialize: invalid format");
for(u32 i=0; i<s.size(); i++)
{
data[i].param1 = s[i];
}
}
if(version >= 10)
{
// Uncompress and set param2 data
std::ostringstream os(std::ios_base::binary);
decompress(is, os, version);
std::string s = os.str();
if(s.size() != nodecount)
throw SerializationError
("MapBlock::deSerialize: invalid format");
for(u32 i=0; i<s.size(); i++)
{
data[i].param2 = s[i];
}
}
}
// All other versions (newest)
else
{
u32 nodecount = MAP_BLOCKSIZE*MAP_BLOCKSIZE*MAP_BLOCKSIZE;
u8 flags;
is.read((char*)&flags, 1);
is_underground = (flags & 0x01) ? true : false;
m_day_night_differs = (flags & 0x02) ? true : false;
m_lighting_expired = (flags & 0x04) ? true : false;
if(version >= 18)
m_generated = (flags & 0x08) ? false : true;
// Uncompress data
std::ostringstream os(std::ios_base::binary);
decompress(is, os, version);
std::string s = os.str();
if(s.size() != nodecount*3)
throw SerializationError
("MapBlock::deSerialize: decompress resulted in size"
" other than nodecount*3");
// deserialize nodes from buffer
for(u32 i=0; i<nodecount; i++)
{
u8 buf[3];
buf[0] = s[i];
buf[1] = s[i+nodecount];
buf[2] = s[i+nodecount*2];
data[i].deSerialize(buf, version);
}
/*
NodeMetadata
*/
if(version >= 14)
{
// Ignore errors
try{
if(version <= 15)
{
std::string data = deSerializeString(is);
std::istringstream iss(data, std::ios_base::binary);
m_node_metadata->deSerialize(iss, m_gamedef);
}
else
{
//std::string data = deSerializeLongString(is);
std::ostringstream oss(std::ios_base::binary);
decompressZlib(is, oss);
std::istringstream iss(oss.str(), std::ios_base::binary);
m_node_metadata->deSerialize(iss, m_gamedef);
}
}
catch(SerializationError &e)
{
errorstream<<"WARNING: MapBlock::deSerialize(): Ignoring an error"
<<" while deserializing node metadata"<<std::endl;
}
}
}
}
int S3D_Init(s3d_object *s3dobj, FILE *fp)
{
struct_header s3d_header;
struct_directory_header s3d_dir_header;
struct_directory s3d_dir;
struct_data_block s3d_data;
struct_fn_header *s3d_fn_header;
struct_fn_entry *s3d_fn_entry;
uint32 *offsets;
char *temp, *temp2;
int i, j, pos, inf, tmp, running = 0;
s3dobj->fp = fp;
fread(&s3d_header, sizeof(struct_header), 1, fp);
if(s3d_header.magicCookie[0] != 'P' || s3d_header.magicCookie[1] != 'F' || s3d_header.magicCookie[2] != 'S' || s3d_header.magicCookie[3] != ' ')
{
return -1;
}
fseek(fp, s3d_header.offset, SEEK_SET);
fread(&s3d_dir_header, sizeof(struct_directory_header), 1, fp);
s3dobj->count = s3d_dir_header.count;
s3dobj->filenames = (char **) malloc(s3d_dir_header.count * sizeof(char *));
s3dobj->files = (uint32 *) malloc((s3d_dir_header.count - 1) * sizeof(uint32));
offsets = (uint32 *) malloc((s3d_dir_header.count - 1) * sizeof(uint32));
for(i = 0; i < s3d_dir_header.count; ++i)
{
fread(&s3d_dir, sizeof(struct_directory), 1, fp);
if(s3d_dir.crc == 0x61580AC9)
{
pos = ftell(fp);
fseek(fp, s3d_dir.offset, SEEK_SET);
temp = (char *) malloc(s3d_dir.size);
memset(temp, 0, s3d_dir.size);
inf = 0;
while(inf < s3d_dir.size)
{
fread(&s3d_data, sizeof(struct_data_block), 1, fp);
temp2 = (char *) malloc(s3d_data.deflen);
fread(temp2, s3d_data.deflen, 1, fp);
decompress(temp2, temp + inf, s3d_data.deflen, s3d_data.inflen);
free(temp2);
inf += s3d_data.inflen;
//Yeahlight: Zone freeze debug
if(ZONE_FREEZE_DEBUG && rand()%ZONE_FREEZE_DEBUG == 1)
EQC_FREEZE_DEBUG(__LINE__, __FILE__);
}
fseek(fp, pos, SEEK_SET);
s3d_fn_header = (struct_fn_header *) temp;
pos = sizeof(struct_fn_header);
for(j = 0; j < s3d_fn_header->fncount; ++j) //TODO: using arena.s3d, s3d_fn_header->fncount is 10, however it crashes after 8, possible the s3d_fn_header->fncount is wrong due to this being older s3d files
{
s3d_fn_entry = (struct_fn_entry *) &temp[pos];
s3d_fn_entry->fnlen = strlen(&temp[pos + sizeof(struct_fn_entry)]) +1; // Dark-Prince - Added this to update the length
s3dobj->filenames[j] = (char *) malloc(s3d_fn_entry->fnlen + 1);
s3dobj->filenames[j][s3d_fn_entry->fnlen] = 0;
memcpy(s3dobj->filenames[j], &temp[pos + sizeof(struct_fn_entry)], s3d_fn_entry->fnlen);
pos += sizeof(struct_fn_entry) + s3d_fn_entry->fnlen;
}
}
else
{
s3dobj->files[running] = ftell(fp) - 12;
offsets[running] = s3d_dir.offset;
++running;
}
}
for(i = s3d_dir_header.count - 2; i > 0; i--) {
for(j = 0; j < i; j++) {
if(offsets[j] > offsets[j+1]) {
tmp = offsets[j];
offsets[j] = offsets[j + 1];
offsets[j + 1] = tmp;
tmp = s3dobj->files[j];
s3dobj->files[j] = s3dobj->files[j + 1];
s3dobj->files[j + 1] = tmp;
}
}
}
return 0;
}
void splash_logo(u32 splash_id)
{
bmp_t *bmp;
int rval = 0;
u32 out_ptr = 0;
u32 gzip_logo;
int gzip_logo_len;
/* Select the free memory for store the decomprssed data */
if (logo_addr == NULL)
#if (CHIP_REV == I1)
logo_addr = (u8 *)BSB_RAM_START;
#else
logo_addr = (u8 *)0xc2000000;
#endif
if (splash_id == 0) {
gzip_logo = (u32)&__gzip_logo_1;
gzip_logo_len = __gzip_logo_1_len;
} else if (splash_id == 1) {
gzip_logo = (u32)&__gzip_logo_2;
gzip_logo_len = __gzip_logo_2_len;
} else if (splash_id == 2) {
gzip_logo = (u32)&__gzip_logo_3;
gzip_logo_len = __gzip_logo_3_len;
} else if (splash_id == 3) {
gzip_logo = (u32)&__gzip_logo_4;
gzip_logo_len = __gzip_logo_4_len;
} else if (splash_id == 4) {
gzip_logo = (u32)&__gzip_logo_5;
gzip_logo_len = __gzip_logo_5_len;
} else {
putstr("Splash_Err : splash_id should be between 1 and 5!");
putstr("\r\n");
/* If splash id is not valid, VOUT_BG_BLUE */
//vout_set_monitor_bg_color(chan, 0x29, 0xf0, 0x6e);
return;
}
if (gzip_logo_len == 0) {
putstr("Splash_Err : splash logo length = 0!");
return;
}
out_ptr = decompress("splash",
(u32) gzip_logo,
((u32) gzip_logo) + gzip_logo_len,
(u32) logo_addr,
(u32) &__heap_start,
(u32) &__heap_end);
/** Use the golbal variable to store the BMP info for VOUT OSD used.
*/
bmp = bld_get_splash_bmp();
/* Parse the BMP file header */
rval = bld_parse_bmp(logo_addr, bmp);
if (rval == 0) {
/* Assign the the BMP raw data offset to the bmp buffer */
bmp->buf = (logo_addr + bmp->header.fheader.offset);
/* Assign the the BMP width, height, pitch from the info header */
bmp->width = bmp->header.iheader.width;
/* BMP pitch need be the mutilple of 4 */
if ((bmp->width % 4) != 0)
bmp->width = ((bmp->header.iheader.width / 4) + 1) * 4;
bmp->height = bmp->header.iheader.height;
bmp->pitch = bmp->width;
bmp->size = bmp->header.iheader.bmp_data_size;
} else {
putstr("Splash_Err : Parsing BMP header!");
putstr("\r\n");
return;
}
}
/**
* Deserialize meta-data. Initializes md.
*
* @param input buffer with the serialized metadata
* @param size number of bytes available in input
* @return MD on success, NULL on error (i.e.
* bad format)
*/
struct GNUNET_CONTAINER_MetaData *
GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size)
{
struct GNUNET_CONTAINER_MetaData *md;
struct MetaDataHeader hdr;
struct MetaDataEntry ent;
uint32_t ic;
uint32_t i;
char *data;
const char *cdata;
uint32_t version;
uint32_t dataSize;
int compressed;
size_t left;
uint32_t mlen;
uint32_t plen;
uint32_t dlen;
const char *mdata;
const char *meta_data;
const char *plugin_name;
const char *mime_type;
enum EXTRACTOR_MetaFormat format;
if (size < sizeof (struct MetaDataHeader))
return NULL;
memcpy (&hdr, input, sizeof (struct MetaDataHeader));
version = ntohl (hdr.version) & HEADER_VERSION_MASK;
compressed = (ntohl (hdr.version) & HEADER_COMPRESSED) != 0;
if (version == 1)
return NULL; /* null pointer */
if (version != 2)
{
GNUNET_break_op (0); /* unsupported version */
return NULL;
}
ic = ntohl (hdr.entries);
dataSize = ntohl (hdr.size);
if ((sizeof (struct MetaDataEntry) * ic) > dataSize)
{
GNUNET_break_op (0);
return NULL;
}
if (compressed)
{
if (dataSize >= GNUNET_MAX_MALLOC_CHECKED)
{
/* make sure we don't blow our memory limit because of a mal-formed
* message... */
GNUNET_break_op (0);
return NULL;
}
data =
decompress ((const char *) &input[sizeof (struct MetaDataHeader)],
size - sizeof (struct MetaDataHeader), dataSize);
if (data == NULL)
{
GNUNET_break_op (0);
return NULL;
}
cdata = data;
}
else
{
data = NULL;
cdata = (const char *) &input[sizeof (struct MetaDataHeader)];
if (dataSize != size - sizeof (struct MetaDataHeader))
{
GNUNET_break_op (0);
return NULL;
}
}
md = GNUNET_CONTAINER_meta_data_create ();
left = dataSize - ic * sizeof (struct MetaDataEntry);
mdata = &cdata[ic * sizeof (struct MetaDataEntry)];
for (i = 0; i < ic; i++)
{
memcpy (&ent, &cdata[i * sizeof (struct MetaDataEntry)],
sizeof (struct MetaDataEntry));
format = (enum EXTRACTOR_MetaFormat) ntohl (ent.format);
if ((format != EXTRACTOR_METAFORMAT_UTF8) &&
(format != EXTRACTOR_METAFORMAT_C_STRING) &&
(format != EXTRACTOR_METAFORMAT_BINARY))
{
GNUNET_break_op (0);
break;
}
dlen = ntohl (ent.data_size);
plen = ntohl (ent.plugin_name_len);
mlen = ntohl (ent.mime_type_len);
if (dlen > left)
{
GNUNET_break_op (0);
break;
}
left -= dlen;
meta_data = &mdata[left];
if ((format == EXTRACTOR_METAFORMAT_UTF8) ||
(format == EXTRACTOR_METAFORMAT_C_STRING))
{
if ((dlen == 0) || (mdata[left + dlen - 1] != '\0'))
{
GNUNET_break_op (0);
break;
}
}
if (plen > left)
{
GNUNET_break_op (0);
break;
}
left -= plen;
if ((plen > 0) && (mdata[left + plen - 1] != '\0'))
{
GNUNET_break_op (0);
break;
}
if (plen == 0)
plugin_name = NULL;
else
plugin_name = &mdata[left];
if (mlen > left)
{
GNUNET_break_op (0);
break;
}
left -= mlen;
if ((mlen > 0) && (mdata[left + mlen - 1] != '\0'))
{
GNUNET_break_op (0);
break;
}
if (mlen == 0)
mime_type = NULL;
else
mime_type = &mdata[left];
GNUNET_CONTAINER_meta_data_insert (md, plugin_name,
(enum EXTRACTOR_MetaType)
ntohl (ent.type), format, mime_type,
meta_data, dlen);
}
GNUNET_free_non_null (data);
return md;
}
static int copy_one_extent(struct btrfs_root *root, int fd,
struct extent_buffer *leaf,
struct btrfs_file_extent_item *fi, u64 pos)
{
struct btrfs_multi_bio *multi = NULL;
struct btrfs_device *device;
char *inbuf, *outbuf = NULL;
ssize_t done, total = 0;
u64 bytenr;
u64 ram_size;
u64 disk_size;
u64 num_bytes;
u64 length;
u64 size_left;
u64 dev_bytenr;
u64 offset;
u64 count = 0;
int compress;
int ret;
int dev_fd;
int mirror_num = 1;
int num_copies;
compress = btrfs_file_extent_compression(leaf, fi);
bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
disk_size = btrfs_file_extent_disk_num_bytes(leaf, fi);
ram_size = btrfs_file_extent_ram_bytes(leaf, fi);
offset = btrfs_file_extent_offset(leaf, fi);
num_bytes = btrfs_file_extent_num_bytes(leaf, fi);
size_left = disk_size;
if (compress == BTRFS_COMPRESS_NONE)
bytenr += offset;
if (verbose && offset)
printf("offset is %Lu\n", offset);
/* we found a hole */
if (disk_size == 0)
return 0;
inbuf = malloc(size_left);
if (!inbuf) {
error("not enough memory");
return -ENOMEM;
}
if (compress != BTRFS_COMPRESS_NONE) {
outbuf = calloc(1, ram_size);
if (!outbuf) {
error("not enough memory");
free(inbuf);
return -ENOMEM;
}
}
again:
length = size_left;
ret = btrfs_map_block(root->fs_info, READ, bytenr, &length, &multi,
mirror_num, NULL);
if (ret) {
error("cannot map block logical %llu length %llu: %d",
(unsigned long long)bytenr,
(unsigned long long)length, ret);
goto out;
}
device = multi->stripes[0].dev;
dev_fd = device->fd;
device->total_ios++;
dev_bytenr = multi->stripes[0].physical;
free(multi);
if (size_left < length)
length = size_left;
done = pread(dev_fd, inbuf+count, length, dev_bytenr);
/* Need both checks, or we miss negative values due to u64 conversion */
if (done < 0 || done < length) {
num_copies = btrfs_num_copies(root->fs_info, bytenr, length);
mirror_num++;
/* mirror_num is 1-indexed, so num_copies is a valid mirror. */
if (mirror_num > num_copies) {
ret = -1;
error("exhausted mirrors trying to read (%d > %d)",
mirror_num, num_copies);
goto out;
}
fprintf(stderr, "Trying another mirror\n");
goto again;
}
mirror_num = 1;
size_left -= length;
count += length;
bytenr += length;
if (size_left)
goto again;
if (compress == BTRFS_COMPRESS_NONE) {
while (total < num_bytes) {
done = pwrite(fd, inbuf+total, num_bytes-total,
pos+total);
if (done < 0) {
ret = -1;
error("cannot write data: %d %m", errno);
goto out;
}
total += done;
}
ret = 0;
goto out;
}
ret = decompress(root, inbuf, outbuf, disk_size, &ram_size, compress);
if (ret) {
num_copies = btrfs_num_copies(root->fs_info, bytenr, length);
mirror_num++;
if (mirror_num >= num_copies) {
ret = -1;
goto out;
}
fprintf(stderr, "Trying another mirror\n");
goto again;
}
while (total < num_bytes) {
done = pwrite(fd, outbuf + offset + total,
num_bytes - total,
pos + total);
if (done < 0) {
ret = -1;
goto out;
}
total += done;
}
out:
free(inbuf);
free(outbuf);
return ret;
}
int main(int argc, char* argv[])
{
int type = FPZIP_TYPE_FLOAT;
int prec = 0;
int nx = 1;
int ny = 1;
int nz = 1;
int nf = 1;
char* infile = 0;
char* outfile = 0;
char* reconfile = 0;
// parse command-line options
switch (argc) {
case 10:
reconfile = argv[9];
/*FALLTHROUGH*/
case 9:
outfile = argv[8];
/*FALLTHROUGH*/
case 8:
infile = argv[7];
/*FALLTHROUGH*/
case 7:
if (sscanf(argv[6], "%d", &prec) != 1)
goto usage;
/*FALLTHROUGH*/
case 6:
if (sscanf(argv[5], "%d", &nf) != 1)
goto usage;
/*FALLTHROUGH*/
case 5:
if (sscanf(argv[4], "%d", &nz) != 1)
goto usage;
/*FALLTHROUGH*/
case 4:
if (sscanf(argv[3], "%d", &ny) != 1)
goto usage;
/*FALLTHROUGH*/
case 3:
if (sscanf(argv[2], "%d", &nx) != 1)
goto usage;
/*FALLTHROUGH*/
case 2:
if (!strcmp(argv[1], "-f"))
type = FPZIP_TYPE_FLOAT;
else if (!strcmp(argv[1], "-d"))
type = FPZIP_TYPE_DOUBLE;
else
goto usage;
break;
default:
usage:
fprintf(stderr, "Usage: testfpzip <-f|-d> [nx [ny [nz [nf [prec [infile [outfile [reconfile]]]]]]]]\n");
return EXIT_FAILURE;
}
// initialize
fprintf(stderr, "testing '%s a[%d][%d][%d][%d]'\n", type == FPZIP_TYPE_FLOAT ? "float" : "double", nf, nz, ny, nx);
size_t count = (size_t)nx * ny * nz * nf;
size_t size = (type == FPZIP_TYPE_FLOAT ? sizeof(float) : sizeof(double));
size_t inbytes = count * size;
size_t bufbytes = 1024 + inbytes;
if (prec == 0)
prec = CHAR_BIT * size;
int lossless = (prec == CHAR_BIT * size);
// allocate buffers
#ifdef __cplusplus
void* data = (type == FPZIP_TYPE_FLOAT ? static_cast<void*>(new float[count]) : static_cast<void*>(new double[count]));
void* copy = (type == FPZIP_TYPE_FLOAT ? static_cast<void*>(new float[count]) : static_cast<void*>(new double[count]));
void* buffer = static_cast<void*>(new unsigned char[bufbytes]);
#else
void* data = malloc(inbytes);
void* copy = malloc(inbytes);
void* buffer = malloc(bufbytes);
#endif
// initialize scalar field
if (infile) {
// read raw data
fprintf(stderr, "reading input file\n");
FILE* file = fopen(infile, "rb");
if (!file) {
fprintf(stderr, "cannot open input file\n");
return EXIT_FAILURE;
}
if (fread(data, size, count, file) != count) {
fprintf(stderr, "read failed\n");
return EXIT_FAILURE;
}
fclose(file);
}
else {
// set scalar field to product of cosines
const double pi = acos(-1.0);
if (type == FPZIP_TYPE_FLOAT) {
float* p = (float*)data;
for (int f = 0; f < nf; f++)
for (int z = 0; z < nz; z++)
for (int y = 0; y < ny; y++)
for (int x = 0; x < nx; x++)
*p++ = cos(2 * pi * x / nx) * cos(2 * pi * y / ny) * cos(2 * pi * z / nz);
}
else {
double* p = (double*)data;
for (int f = 0; f < nf; f++)
for (int z = 0; z < nz; z++)
for (int y = 0; y < ny; y++)
for (int x = 0; x < nx; x++)
*p++ = cos(2 * pi * x / nx) * cos(2 * pi * y / ny) * cos(2 * pi * z / nz);
}
}
// compress to memory
FPZ* fpz = fpzip_write_to_buffer(buffer, bufbytes);
fpz->type = type;
fpz->prec = prec;
fpz->nx = nx;
fpz->ny = ny;
fpz->nz = nz;
fpz->nf = nf;
if (!compress(fpz, data, inbytes, "memory"))
return EXIT_FAILURE;
fpzip_write_close(fpz);
// decompress from memory
fpz = fpzip_read_from_buffer(buffer);
#ifdef WITHOUT_HEADER
// manually set array parameters since header is not stored
fpz->type = type;
fpz->prec = prec;
fpz->nx = nx;
fpz->ny = ny;
fpz->nz = nz;
fpz->nf = nf;
#endif
if (!decompress(fpz, copy, inbytes, "memory"))
return EXIT_FAILURE;
fpzip_read_close(fpz);
// perform validation
if (!validate(data, copy, count, type, lossless))
return EXIT_FAILURE;
if (outfile) {
// compress to file
FILE* file = fopen(outfile, "wb");
if (!file) {
fprintf(stderr, "cannot create compressed file\n");
return EXIT_FAILURE;
}
fpz = fpzip_write_to_file(file);
fpz->type = type;
fpz->prec = prec;
fpz->nx = nx;
fpz->ny = ny;
fpz->nz = nz;
fpz->nf = nf;
if (!compress(fpz, data, inbytes, "file"))
return EXIT_FAILURE;
fpzip_write_close(fpz);
fclose(file);
// decompress from file
file = fopen(outfile, "rb");
if (!file) {
fprintf(stderr, "cannot open compressed file\n");
return EXIT_FAILURE;
}
fpz = fpzip_read_from_file(file);
#ifdef WITHOUT_HEADER
// manually set array parameters since header is not stored
fpz->type = type;
fpz->prec = prec;
fpz->nx = nx;
fpz->ny = ny;
fpz->nz = nz;
fpz->nf = nf;
#endif
if (!decompress(fpz, copy, inbytes, "file"))
return EXIT_FAILURE;
fpzip_read_close(fpz);
fclose(file);
// perform validation
if (!validate(data, copy, count, type, lossless))
return EXIT_FAILURE;
if (reconfile) {
// write reconstructed data to file
FILE* file = fopen(reconfile, "wb");
if (!file) {
fprintf(stderr, "cannot open reconstructed file\n");
return EXIT_FAILURE;
}
if (fwrite(copy, size, count, file) != count) {
fprintf(stderr, "write failed\n");
return EXIT_FAILURE;
}
fclose(file);
}
}
// deallocate buffers
#ifdef __cplusplus
if (type == FPZIP_TYPE_FLOAT) {
delete[] static_cast<float*>(data);
delete[] static_cast<float*>(copy);
}
else {
delete[] static_cast<double*>(data);
delete[] static_cast<double*>(copy);
}
delete[] static_cast<unsigned char*>(buffer);
#else
free(data);
free(copy);
free(buffer);
#endif
fprintf(stderr, "OK\n");
return 0;
}
void search_file(const char *file_full_path) {
int fd;
off_t f_len = 0;
char *buf = NULL;
struct stat statbuf;
int rv = 0;
FILE *pipe = NULL;
fd = open(file_full_path, O_RDONLY);
if (fd < 0) {
log_err("Error opening file %s: %s Skipping...", file_full_path, strerror(errno));
goto cleanup;
}
rv = fstat(fd, &statbuf);
if (rv != 0) {
log_err("Error fstat()ing file %s. Skipping...", file_full_path);
goto cleanup;
}
if (opts.stdout_inode != 0 && opts.stdout_inode == statbuf.st_ino) {
log_debug("Skipping %s because stdout is redirected to it", file_full_path);
goto cleanup;
}
if ((statbuf.st_mode & S_IFMT) == 0) {
log_err("%s is not a file. Mode %u. Skipping...", file_full_path, statbuf.st_mode);
goto cleanup;
}
if (statbuf.st_mode & S_IFIFO) {
log_debug("%s is a named pipe. stream searching", file_full_path);
pipe = fdopen(fd, "r");
search_stream(pipe, file_full_path);
fclose(pipe);
} else {
f_len = statbuf.st_size;
if (f_len == 0) {
log_debug("File %s is empty, skipping.", file_full_path);
goto cleanup;
}
#ifdef _WIN32
{
HANDLE hmmap = CreateFileMapping(
(HANDLE)_get_osfhandle(fd), 0, PAGE_READONLY, 0, f_len, NULL);
buf = (char*) MapViewOfFile(hmmap, FILE_SHARE_READ, 0, 0, f_len);
if (hmmap != NULL)
CloseHandle(hmmap);
}
if (buf == NULL) {
FormatMessageA(
FORMAT_MESSAGE_ALLOCATE_BUFFER |
FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, GetLastError(), 0, (void*) &buf, 0, NULL);
log_err("File %s failed to load: %s.", file_full_path, buf);
LocalFree((void*)buf);
goto cleanup;
}
#else
buf = mmap(0, f_len, PROT_READ, MAP_SHARED, fd, 0);
if (buf == MAP_FAILED) {
log_err("File %s failed to load: %s.", file_full_path, strerror(errno));
goto cleanup;
}
#endif
if (opts.search_zip_files) {
ag_compression_type zip_type = is_zipped(buf, f_len);
if (zip_type != AG_NO_COMPRESSION) {
int _buf_len = (int)f_len;
char *_buf = decompress(zip_type, buf, f_len, file_full_path, &_buf_len);
if (_buf == NULL || _buf_len == 0) {
log_err("Cannot decompress zipped file %s", file_full_path);
goto cleanup;
}
search_buf(_buf, _buf_len, file_full_path);
free(_buf);
goto cleanup;
}
}
search_buf(buf, (int)f_len, file_full_path);
}
cleanup:;
if (fd != -1) {
#ifdef _WIN32
UnmapViewOfFile(buf);
#else
munmap(buf, f_len);
#endif
close(fd);
}
}
