// ---------------------------------------------------------------------------- // Tokenizer::openMem // // Opens text from a MemChunk [mc] // ---------------------------------------------------------------------------- bool Tokenizer::openMem(const MemChunk& mc, const string& source) { source_ = source; data_.assign(mc.getData(), mc.getData() + mc.getSize()); reset(); return true; }
/* AudioEntryPanel::openMod * Opens a Module file for playback *******************************************************************/ bool AudioEntryPanel::openMod(MemChunk& data) { // Attempt to load the mod if (mod->loadFromMemory(data.getData(), data.getSize())) { audio_type = AUTYPE_MOD; // Enable playback controls slider_volume->Enable(); btn_play->Enable(); btn_pause->Enable(); btn_stop->Enable(); setAudioDuration(mod->getDuration().asMilliseconds()); return true; } else { // Disable playback controls slider_volume->Enable(); btn_play->Enable(); btn_pause->Enable(); btn_stop->Enable(); setAudioDuration(0); return false; } return false; }
bool isThisFormat(MemChunk& mc) { FIMEMORY* mem = FreeImage_OpenMemory((BYTE*)mc.getData(), mc.getSize()); FREE_IMAGE_FORMAT fif = FreeImage_GetFileTypeFromMemory(mem, 0); FreeImage_CloseMemory(mem); if (fif == FIF_UNKNOWN) return false; else return true; }
/* ArchiveEntry::importMemChunk * Imports data from a MemChunk object into the entry, resizing it * and clearing any currently existing data. * Returns false if the MemChunk has no data, or true otherwise. *******************************************************************/ bool ArchiveEntry::importMemChunk(MemChunk& mc) { // Check that the given MemChunk has data if (mc.hasData()) { // Copy the data from the MemChunk into the entry return importMem(mc.getData(), mc.getSize()); } else return false; }
/* GLTexture::loadImage * Loads SImage data to the texture. If the dimensions are invalid * for the system opengl implementation, the data will be split into * 128x128 squares. Returns false if the given data is invalid, true * otherwise *******************************************************************/ bool GLTexture::loadImage(SImage* image, Palette8bit* pal) { // Check image was given if (!image) return false; // Check image is valid if (!image->isValid()) return false; // Clear current texture clear(); // Check image dimensions if (OpenGL::validTexDimension(image->getWidth()) && OpenGL::validTexDimension(image->getHeight())) { // If the image dimensions are valid for OpenGL on this system, just load it as a single texture // Get RGBA image data MemChunk rgba; image->getRGBAData(rgba, pal); // Generate GL texture from rgba data return loadData(rgba.getData(), image->getWidth(), image->getHeight()); } else { // Otherwise split the image into 128x128 chunks int top = 0; while (top < image->getHeight()) { int left = 0; while (left < image->getWidth()) { // Load 128x128 portion of image loadImagePortion(image, rect_t(left, top, left + 128, top + 128), pal, true); // Move right 128px left += 128; } // Move down 128px top += 128; } // Update variables width = image->getWidth(); height = image->getHeight(); scale_x = scale_y = 1.0; return true; } }
/* DecodeTXB * TXB files are text files with a bit shift xor cipher. It makes an exception * for the newline character probably so that standard string functions will * continue to work. As an extension we also except the encoded version of 0xA * in order to produce a lossless conversion. This allows us to semi-effectively * handle this at the archive level instead of as a filter at the text editor. *******************************************************************/ void DecodeTXB(MemChunk &mc) { const uint8_t* data = mc.getData(); const uint8_t* const dataend = data + mc.getSize(); uint8_t* odata = new uint8_t[mc.getSize()]; uint8_t* const ostart = odata; while (data != dataend) { if (*data != 0xA && *data != 0x8F) { *odata++ = (((*data&0x3F)<<2)|((*data&0xC0)>>6))^0xA7; ++data; } else
/* PaletteEntryPanel::saveEntry * Writes all loaded palettes in the palette entry *******************************************************************/ bool PaletteEntryPanel::saveEntry() { MemChunk full; MemChunk mc; // Write each palette as raw data for (size_t a = 0; a < palettes.size(); a++) { palettes[a]->saveMem(mc, Palette8bit::FORMAT_RAW); full.write(mc.getData(), 768); } entry->importMemChunk(full); setModified(false); return true; }
/* DatArchive::open * Reads wad format data from a MemChunk * Returns true if successful, false otherwise *******************************************************************/ bool DatArchive::open(MemChunk& mc) { // Check data was given if (!mc.hasData()) return false; const uint8_t* mcdata = mc.getData(); // Read dat header mc.seek(0, SEEK_SET); uint16_t num_lumps; uint32_t dir_offset, unknown; mc.read(&num_lumps, 2); // Size mc.read(&dir_offset, 4); // Directory offset mc.read(&unknown, 4); // Unknown value num_lumps = wxINT16_SWAP_ON_BE(num_lumps); dir_offset = wxINT32_SWAP_ON_BE(dir_offset); unknown = wxINT32_SWAP_ON_BE(unknown); string lastname(wxString::FromAscii("-noname-")); size_t namecount = 0; // Stop announcements (don't want to be announcing modification due to entries being added etc) setMuted(true); // Read the directory mc.seek(dir_offset, SEEK_SET); theSplashWindow->setProgressMessage("Reading dat archive data"); for (uint32_t d = 0; d < num_lumps; d++) { // Update splash window progress theSplashWindow->setProgress(((float)d / (float)num_lumps)); // Read lump info uint32_t offset = 0; uint32_t size = 0; uint16_t nameofs = 0; uint16_t flags = 0; mc.read(&offset, 4); // Offset mc.read(&size, 4); // Size mc.read(&nameofs, 2); // Name offset mc.read(&flags, 2); // Flags (only one: RLE encoded) // Byteswap values for big endian if needed offset = wxINT32_SWAP_ON_BE(offset); size = wxINT32_SWAP_ON_BE(size); nameofs = wxINT16_SWAP_ON_BE(nameofs); flags = wxINT16_SWAP_ON_BE(flags); // If the lump data goes past the directory, // the data file is invalid if (offset + size > mc.getSize()) { wxLogMessage("DatArchive::open: Dat archive is invalid or corrupt at entry %i", d); Global::error = "Archive is invalid and/or corrupt"; setMuted(false); return false; } string myname; if (nameofs != 0) { size_t len = 1; size_t start = nameofs+dir_offset; for (size_t i = start; mcdata[i] != 0; ++i) { ++len; } lastname = myname = wxString::FromAscii(mcdata+start, len); namecount = 0; } else { myname = S_FMT("%s+%d", lastname, ++namecount); } // Create & setup lump ArchiveEntry* nlump = new ArchiveEntry(myname, size); nlump->setLoaded(false); nlump->exProp("Offset") = (int)offset; nlump->setState(0); if (flags & 1) nlump->setEncryption(ENC_SCRLE0); // Check for markers if (!nlump->getName().Cmp("startflats")) flats[0] = d; if (!nlump->getName().Cmp("endflats")) flats[1] = d; if (!nlump->getName().Cmp("startsprites")) sprites[0] = d; if (!nlump->getName().Cmp("endmonsters")) sprites[1] = d; if (!nlump->getName().Cmp("startwalls")) walls[0] = d; if (!nlump->getName().Cmp("endwalls")) walls[1] = d; // Add to entry list getRoot()->addEntry(nlump); } // Detect all entry types MemChunk edata; theSplashWindow->setProgressMessage("Detecting entry types"); for (size_t a = 0; a < numEntries(); a++) { // Update splash window progress theSplashWindow->setProgress((((float)a / (float)num_lumps))); // Get entry ArchiveEntry* entry = getEntry(a); // Read entry data if it isn't zero-sized if (entry->getSize() > 0) { // Read the entry data mc.exportMemChunk(edata, getEntryOffset(entry), entry->getSize()); entry->importMemChunk(edata); } // Detect entry type EntryType::detectEntryType(entry); // Set entry to unchanged entry->setState(0); } // Detect maps (will detect map entry types) //theSplashWindow->setProgressMessage("Detecting maps"); //detectMaps(); // Setup variables setMuted(false); setModified(false); announce("opened"); theSplashWindow->setProgressMessage(""); return true; }
/* AudioEntryPanel::openAudio * Opens an audio file for playback (SFML 2.x+) *******************************************************************/ bool AudioEntryPanel::openAudio(MemChunk& audio, string filename) { // Stop if sound currently playing resetStream(); // (Re)create sound buffer if (sound_buffer) delete sound_buffer; sound_buffer = new sf::SoundBuffer(); audio_type = AUTYPE_INVALID; // Load into buffer if (sound_buffer->loadFromMemory((const char*)audio.getData(), audio.getSize())) { LOG_MESSAGE(3, "opened as sound"); // Bind to sound sound->setBuffer(*sound_buffer); audio_type = AUTYPE_SOUND; // Enable play controls #if (SFML_VERSION_MAJOR == 2 && SFML_VERSION_MINOR < 2) // SFML before 2.2 has a bug where it reports an incorrect value for long sounds, so compute it ourselves then setAudioDuration((sound_buffer->getSampleCount() / sound_buffer->getSampleRate())*(1000/sound_buffer->getChannelCount())); #else setAudioDuration(sound_buffer->getDuration().asMilliseconds()); #endif btn_play->Enable(); btn_pause->Enable(); btn_stop->Enable(); return true; } else if (music->openFromMemory((const char*)audio.getData(), audio.getSize())) { LOG_MESSAGE(3, "opened as music"); // Couldn't open the audio as a sf::SoundBuffer, try sf::Music instead audio_type = AUTYPE_MUSIC; // Enable play controls setAudioDuration(music->getDuration().asMilliseconds()); btn_play->Enable(); btn_stop->Enable(); return true; } else { // Couldn't open as sound or music, try the wxMediaCtrl LOG_MESSAGE(3, "opened as media"); // Dump audio to temp file audio.exportFile(filename); if (openMedia(filename)) return true; } // Unable to open audio, disable play controls setAudioDuration(0); btn_play->Enable(false); btn_pause->Enable(false); btn_stop->Enable(false); return false; }
/* ADatArchive::write * Writes the dat archive to a MemChunk * Returns true if successful, false otherwise *******************************************************************/ bool ADatArchive::write(MemChunk& mc, bool update) { // Clear current data mc.clear(); MemChunk directory; MemChunk compressed; // Get archive tree as a list vector<ArchiveEntry*> entries; getEntryTreeAsList(entries); // Write header long dir_offset = wxINT32_SWAP_ON_BE(16); long dir_size = wxINT32_SWAP_ON_BE(0); char pack[4] = { 'A', 'D', 'A', 'T' }; uint32_t version = wxINT32_SWAP_ON_BE(9); mc.seek(0, SEEK_SET); mc.write(pack, 4); mc.write(&dir_offset, 4); mc.write(&dir_size, 4); mc.write(&version, 4); // Write entry data for (unsigned a = 0; a < entries.size(); a++) { // Skip folders if (entries[a]->getType() == EntryType::folderType()) continue; // Create compressed version of the lump MemChunk * entry = NULL; if (Compression::ZlibDeflate(entries[a]->getMCData(), compressed, 9)) { entry = &compressed; } else { entry = &(entries[a]->getMCData()); wxLogMessage("Entry %s couldn't be deflated", CHR(entries[a]->getName())); } // Update entry int offset = mc.currentPos(); if (update) { entries[a]->setState(0); entries[a]->exProp("Offset") = (int)offset; } /////////////////////////////////// // Step 1: Write directory entry // /////////////////////////////////// // Check entry name string name = entries[a]->getPath(true); name.Remove(0, 1); // Remove leading / if (name.Len() > 128) { wxLogMessage("Warning: Entry %s path is too long (> 128 characters), putting it in the root directory", CHR(name)); wxFileName fn(name); name = fn.GetFullName(); if (name.Len() > 128) name.Truncate(128); } // Write entry name char name_data[128]; memset(name_data, 0, 128); memcpy(name_data, CHR(name), name.Length()); directory.write(name_data, 128); // Write entry offset long myoffset = wxINT32_SWAP_ON_BE(offset); directory.write(&myoffset, 4); // Write full entry size long decsize = wxINT32_SWAP_ON_BE(entries[a]->getSize()); directory.write(&decsize, 4); // Write compressed entry size long compsize = wxINT32_SWAP_ON_BE(entry->getSize()); directory.write(&compsize, 4); // Write whatever it is that should be there // TODO: Reverse engineer what exactly it is // and implement something valid for the game. long whatever = 0; directory.write(&whatever, 4); ////////////////////////////// // Step 2: Write entry data // ////////////////////////////// mc.write(entry->getData(), entry->getSize()); } // Write directory dir_offset = wxINT32_SWAP_ON_BE(mc.currentPos()); dir_size = wxINT32_SWAP_ON_BE(directory.getSize()); mc.write(directory.getData(), directory.getSize()); // Update directory offset and size in header mc.seek(4, SEEK_SET); mc.write(&dir_offset, 4); mc.write(&dir_size, 4); // Yay! Finished! return true; }
/* RffArchive::open * Reads grp format data from a MemChunk * Returns true if successful, false otherwise *******************************************************************/ bool RffArchive::open(MemChunk& mc) { // Check data was given if (!mc.hasData()) return false; // Read grp header uint8_t magic[4]; uint32_t version, dir_offset, num_lumps; mc.seek(0, SEEK_SET); mc.read(magic, 4); // Should be "RFF\x18" mc.read(&version, 4); // 0x01 0x03 \x00 \x00 mc.read(&dir_offset, 4); // Offset to directory mc.read(&num_lumps, 4); // No. of lumps in rff // Byteswap values for big endian if needed dir_offset = wxINT32_SWAP_ON_BE(dir_offset); num_lumps = wxINT32_SWAP_ON_BE(num_lumps); version = wxINT32_SWAP_ON_BE(version); // Check the header if (magic[0] != 'R' || magic[1] != 'F' || magic[2] != 'F' || magic[3] != 0x1A || version != 0x301) { wxLogMessage("RffArchive::openFile: File %s has invalid header", filename); Global::error = "Invalid rff header"; return false; } // Stop announcements (don't want to be announcing modification due to entries being added etc) setMuted(true); // Read the directory RFFLump* lumps = new RFFLump[num_lumps]; mc.seek(dir_offset, SEEK_SET); theSplashWindow->setProgressMessage("Reading rff archive data"); mc.read (lumps, num_lumps * sizeof(RFFLump)); BloodCrypt (lumps, dir_offset, num_lumps * sizeof(RFFLump)); for (uint32_t d = 0; d < num_lumps; d++) { // Update splash window progress theSplashWindow->setProgress(((float)d / (float)num_lumps)); // Read lump info char name[13] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; uint32_t offset = wxINT32_SWAP_ON_BE(lumps[d].FilePos); uint32_t size = wxINT32_SWAP_ON_BE(lumps[d].Size); // Reconstruct name int i, j = 0; for (i = 0; i < 8; ++i) { if (lumps[d].Name[i] == 0) break; name[i] = lumps[d].Name[i]; } for (name[i++] = '.'; j < 3; ++j) name[i+j] = lumps[d].Extension[j]; // If the lump data goes past the end of the file, // the rfffile is invalid if (offset + size > mc.getSize()) { wxLogMessage("RffArchive::open: rff archive is invalid or corrupt"); Global::error = "Archive is invalid and/or corrupt"; setMuted(false); return false; } // Create & setup lump ArchiveEntry* nlump = new ArchiveEntry(wxString::FromAscii(name), size); nlump->setLoaded(false); nlump->exProp("Offset") = (int)offset; nlump->setState(0); // Is the entry encrypted? if (lumps[d].Flags & 0x10) nlump->setEncryption(ENC_BLOOD); // Add to entry list getRoot()->addEntry(nlump); } delete[] lumps; // Detect all entry types MemChunk edata; theSplashWindow->setProgressMessage("Detecting entry types"); for (size_t a = 0; a < numEntries(); a++) { // Update splash window progress theSplashWindow->setProgress((((float)a / (float)num_lumps))); // Get entry ArchiveEntry* entry = getEntry(a); // Read entry data if it isn't zero-sized if (entry->getSize() > 0) { // Read the entry data mc.exportMemChunk(edata, getEntryOffset(entry), entry->getSize()); // If the entry is encrypted, decrypt it if (entry->isEncrypted()) { uint8_t* cdata = new uint8_t[entry->getSize()]; memcpy(cdata, edata.getData(), entry->getSize()); int cryptlen = entry->getSize() < 256 ? entry->getSize() : 256; BloodCrypt(cdata, 0, cryptlen); edata.importMem(cdata, entry->getSize()); delete[] cdata; } // Import data entry->importMemChunk(edata); } // Detect entry type EntryType::detectEntryType(entry); // Unload entry data if needed if (!archive_load_data) entry->unloadData(); // Set entry to unchanged entry->setState(0); } // Detect maps (will detect map entry types) //theSplashWindow->setProgressMessage("Detecting maps"); //detectMaps(); // Setup variables setMuted(false); setModified(false); announce("opened"); theSplashWindow->setProgressMessage(""); return true; }
bool PaletteEntryPanel::generateColormaps() { if (!entry || !entry->getParent() || ! palettes[0]) return false; MemChunk mc; SImage img; MemChunk imc; mc.reSize(34*256); mc.seek(0, SEEK_SET); imc.reSize(34*256*4); imc.seek(0, SEEK_SET); uint8_t rgba[4]; rgba[3] = 255; rgba_t rgb; float grey; // Generate 34 maps: the first 32 for diminishing light levels, // the 33th for the inverted grey map used by invulnerability. // The 34th colormap remains empty and black. for (size_t l = 0; l < 34; ++l) { for (size_t c = 0; c < 256; ++c) { rgb = palettes[0]->colour(c); if (l < 32) { // Generate light maps DIMINISH(rgb.r, l); DIMINISH(rgb.g, l); DIMINISH(rgb.b, l); #if (0) } else if (l == GREENMAP) { // Point of mostly useless trivia: the green "light amp" colormap in the Press Release beta // have colors that, on average, correspond to a bit less than (R*75/256, G*225/256, B*115/256) #endif } else if (l == GRAYMAP) { // Generate inverse map grey = ((float)rgb.r/256.0 * col_greyscale_r) + ((float)rgb.g/256.0 * col_greyscale_g) + ((float)rgb.b/256.0 * col_greyscale_b); grey = 1.0 - grey; // Clamp value: with Id Software's values, the sum is greater than 1.0 (0.299+0.587+0.144=1.030) // This means the negation above can give a negative value (for example, with RGB values of 247 or more), // which will not be converted correctly to unsigned 8-bit int in the rgba_t struct. if (grey < 0.0) grey = 0; rgb.r = rgb.g = rgb.b = grey*255; } else { // Fill with 0 rgb = palettes[0]->colour(0); } rgba[0] = rgb.r; rgba[1] = rgb.g; rgba[2] = rgb.b; imc.write(&rgba, 4); mc[(256*l)+c] = palettes[0]->nearestColour(rgb); } } #if 0 // Create truecolor image uint8_t* imd = new uint8_t[256*34*4]; memcpy(imd, imc.getData(), 256*34*4); img.setImageData(imd, 256, 34, RGBA); // imd will be freed by img's destructor ArchiveEntry* tcolormap; string name = entry->getName(true) + "-tcm.png"; tcolormap = new ArchiveEntry(name); if (tcolormap) { entry->getParent()->addEntry(tcolormap); SIFormat::getFormat("png")->saveImage(img, tcolormap->getMCData()); EntryType::detectEntryType(tcolormap); } #endif // Now override or create new entry ArchiveEntry* colormap; colormap = entry->getParent()->getEntry("COLORMAP", true); bool preexisting = colormap != NULL; if (!colormap) { // We need to create this entry colormap = new ArchiveEntry("COLORMAP.lmp", 34*256); } if (!colormap) return false; colormap->importMemChunk(mc); if (!preexisting) { entry->getParent()->addEntry(colormap); } return true; }
/* GLTexture::loadImagePortion * Loads a portion of a SImage to the texture. Only used internally, * the portion must be 128x128 in size *******************************************************************/ bool GLTexture::loadImagePortion(SImage* image, rect_t rect, Palette8bit* pal, bool add) { // Check image was given if (!image) return false; // Check image is valid if (!image->isValid()) return false; // Check portion rect is valid if (rect.width() <= 0 || rect.height() <= 0) return false; // Get RGBA image data MemChunk rgba; image->getRGBAData(rgba, pal); // Init texture data MemChunk portion; portion.reSize(rect.width() * rect.height() * 4, false); portion.fillData(0); // Read portion of image if rect isn't completely outside the image if (!(rect.left() >= image->getWidth() || rect.right() < 0 || rect.top() >= image->getHeight() || rect.bottom() < 0)) { // Determine start of each row to read uint32_t row_start = 0; if (rect.left() > 0) row_start = rect.left(); // Determine width of each row to read uint32_t row_width = rect.right() - row_start; if (rect.right() >= image->getWidth()) row_width = image->getWidth() - row_start; // Determine difference between the left of the portion and the left of the image uint32_t skip = 0; if (rect.left() < 0) skip = (0 - rect.left()) * 4; // Create temp row buffer uint8_t* buf = new uint8_t[rect.width() * 4]; // Go through each row for (int32_t row = rect.top(); row < rect.bottom(); row++) { // Clear row buffer memset(buf, 0, rect.width() * 4); // Check that the current row is within the image if (row >= 0 && row < image->getHeight()) { // Seek to current row in image data rgba.seek((row * image->getWidth() + row_start) * 4, SEEK_SET); // Copy the row data rgba.read(buf + skip, row_width * 4); } // Write the row portion.write(buf, rect.width() * 4); } // Free buffer delete[] buf; } scale_x = scale_y = 1.0; // Generate texture from rgba data return loadData(portion.getData(), rect.width(), rect.height(), add); }
/* GZipArchive::open * Reads gzip format data from a MemChunk * Returns true if successful, false otherwise *******************************************************************/ bool GZipArchive::open(MemChunk& mc) { // Minimal metadata size is 18: 10 for header, 8 for footer size_t mds = 18; size_t size = mc.getSize(); if (mds > size) return false; // Read header uint8_t header[4]; mc.read(header, 4); // Check for GZip header; we'll only accept deflated gzip files // and reject any field using unknown flags if ((!(header[0] == GZIP_ID1 && header[1] == GZIP_ID2 && header[2] == GZIP_DEFLATE)) || (header[3] & GZIP_FLG_FUNKN)) return false; bool ftext, fhcrc, fxtra, fname, fcmnt; ftext = (header[3] & GZIP_FLG_FTEXT) ? true : false; fhcrc = (header[3] & GZIP_FLG_FHCRC) ? true : false; fxtra = (header[3] & GZIP_FLG_FXTRA) ? true : false; fname = (header[3] & GZIP_FLG_FNAME) ? true : false; fcmnt = (header[3] & GZIP_FLG_FCMNT) ? true : false; flags = header[3]; mc.read(&mtime, 4); mtime = wxUINT32_SWAP_ON_BE(mtime); mc.read(&xfl, 1); mc.read(&os, 1); // Skip extra fields which may be there if (fxtra) { uint16_t xlen; mc.read(&xlen, 2); xlen = wxUINT16_SWAP_ON_BE(xlen); mds += xlen + 2; if (mds > size) return false; mc.exportMemChunk(xtra, mc.currentPos(), xlen); mc.seek(xlen, SEEK_CUR); } // Skip past name, if any string name; if (fname) { char c; do { mc.read(&c, 1); if (c) name += c; ++mds; } while (c != 0 && size > mds); } else { // Build name from filename name = getFilename(false); wxFileName fn(name); if (!fn.GetExt().CmpNoCase("tgz")) fn.SetExt("tar"); else if (!fn.GetExt().CmpNoCase("gz")) fn.ClearExt(); name = fn.GetFullName(); } // Skip past comment if (fcmnt) { char c; do { mc.read(&c, 1); if (c) comment += c; ++mds; } while (c != 0 && size > mds); wxLogMessage("Archive %s says:\n %s", CHR(getFilename(true)), CHR(comment)); } // Skip past CRC 16 check if (fhcrc) { uint8_t* crcbuffer = new uint8_t[mc.currentPos()]; memcpy(crcbuffer, mc.getData(), mc.currentPos()); uint32_t fullcrc = Misc::crc(crcbuffer, mc.currentPos()); delete[] crcbuffer; uint16_t hcrc; mc.read(&hcrc, 2); hcrc = wxUINT16_SWAP_ON_BE(hcrc); mds += 2; if (hcrc != (fullcrc & 0x0000FFFF)) { wxLogMessage("CRC-16 mismatch for GZip header"); } } // Header is over if (mds > size || mc.currentPos() + 8 > size) return false; // Let's create the entry setMuted(true); ArchiveEntry* entry = new ArchiveEntry(name, size - mds); MemChunk xdata; if (Compression::GZipInflate(mc, xdata)) { entry->importMemChunk(xdata); } else { delete entry; setMuted(false); return false; } getRoot()->addEntry(entry); EntryType::detectEntryType(entry); entry->setState(0); setMuted(false); setModified(false); announce("opened"); // Finish return true; }
/* GZipArchive::write * Writes the gzip archive to a MemChunk * Returns true if successful, false otherwise *******************************************************************/ bool GZipArchive::write(MemChunk& mc, bool update) { // Clear current data mc.clear(); if (numEntries() == 1) { MemChunk stream; if (Compression::GZipDeflate(getEntry(0)->getMCData(), stream, 9)) { const uint8_t* data = stream.getData(); uint32_t working = 0; size_t size = stream.getSize(); if (size < 18) return false; // zlib will have given us a minimal header, so we make our own uint8_t header[4]; header[0] = GZIP_ID1; header[1] = GZIP_ID2; header[2] = GZIP_DEFLATE; header[3] = flags; mc.write(header, 4); // Update mtime if the file was modified if (getEntry(0)->getState()) { mtime = ::wxGetLocalTime(); } // Write mtime working = wxUINT32_SWAP_ON_BE(mtime); mc.write(&working, 4); // Write other stuff mc.write(&xfl, 1); mc.write(&os, 1); // Any extra content that may have been there if (flags & GZIP_FLG_FXTRA) { uint16_t xlen = wxUINT16_SWAP_ON_BE(xtra.getSize()); mc.write(&xlen, 2); mc.write(xtra.getData(), xtra.getSize()); } // File name, if not extrapolated from archive name if (flags & GZIP_FLG_FNAME) { mc.write(CHR(getEntry(0)->getName()), getEntry(0)->getName().length()); uint8_t zero = 0; mc.write(&zero, 1); // Terminate string } // Comment, if there were actually one if (flags & GZIP_FLG_FCMNT) { mc.write(CHR(comment), comment.length()); uint8_t zero = 0; mc.write(&zero, 1); // Terminate string } // And finally, the half CRC, which we recalculate if (flags & GZIP_FLG_FHCRC) { uint32_t fullcrc = Misc::crc(mc.getData(), mc.getSize()); uint16_t hcrc = (fullcrc & 0x0000FFFF); hcrc = wxUINT16_SWAP_ON_BE(hcrc); mc.write(&hcrc, 2); } // Now that the pleasantries are dispensed with, // let's get with the meat of the matter return mc.write(data + 10, size - 10); } } return false; }
void MessageHandleJob::_handleChunkOrAggregateChunk(bool isAggregateChunk) { boost::shared_ptr<scidb_msg::Chunk> chunkRecord = _messageDesc->getRecord<scidb_msg::Chunk>(); assert(!chunkRecord->eof()); assert(_query); RWLock::ErrorChecker noopEc; ScopedRWLockRead shared(_query->queryLock, noopEc); try { if (! isAggregateChunk) { // TODO: Apply it to statistics of current SG currentStatistics->receivedSize += _messageDesc->getMessageSize(); currentStatistics->receivedMessages++; } LOG4CXX_TRACE(logger, "Next chunk message was received") const int compMethod = chunkRecord->compression_method(); const size_t decompressedSize = chunkRecord->decompressed_size(); const AttributeID attributeID = chunkRecord->attribute_id(); const size_t count = chunkRecord->count(); boost::shared_ptr<SGContext> sgCtx = dynamic_pointer_cast<SGContext>(_query->getOperatorContext()); if (sgCtx == NULL) { throw (SYSTEM_EXCEPTION(SCIDB_SE_INTERNAL, SCIDB_LE_UNKNOWN_CTX) << typeid(*_query->getOperatorContext()).name()); } boost::shared_ptr<Array> outputArray = sgCtx->_resultSG; ScopedMutexLock cs(_query->resultCS); boost::shared_ptr<ArrayIterator> outputIter = outputArray->getIterator(attributeID); const bool shouldCacheEmptyBitmap = sgCtx->_shouldCacheEmptyBitmap; const ArrayDesc& desc = outputArray->getArrayDesc(); const size_t sourceId = _query->mapPhysicalToLogical(_messageDesc->getSourceInstanceID()); const bool isEmptyable = (desc.getEmptyBitmapAttribute() != NULL); const bool isEmptyIndicator = isEmptyable && (attributeID+1==desc.getAttributes().size()); const bool rle = chunkRecord->rle(); if (isAggregateChunk) { assert(! isEmptyIndicator); } Coordinates coordinates; for (int i = 0; i < chunkRecord->coordinates_size(); i++) { coordinates.push_back(chunkRecord->coordinates(i)); } if (!isAggregateChunk && sgCtx->_targetVersioned) { sgCtx->_newChunks.insert(coordinates); } boost::shared_ptr<CompressedBuffer> compressedBuffer = dynamic_pointer_cast<CompressedBuffer>(_messageDesc->getBinary()); if (compressedBuffer) { assert(compressedBuffer->getData()); PinBuffer pin(*compressedBuffer); // this line protects compressedBuffer->data from being freed, allowing MemChunk::decompressed(*compressedBuffer) to be called multiple times. compressedBuffer->setCompressionMethod(compMethod); compressedBuffer->setDecompressedSize(decompressedSize); Chunk* outChunk; // temporary MemArray objects shared_ptr<MemChunk> pTmpChunk = make_shared<MemChunk>(); // make it a shared pointer, because the bitmap chunk needs to be preserved across threads. MemChunk closure; // the PinBuffer objects protect tmpChunk and closure, respectively. shared_ptr<PinBuffer> pinTmpChunk, pinClosure; if (outputIter->setPosition(coordinates)) { // existing chunk outChunk = &outputIter->updateChunk(); if (! isAggregateChunk) { if (outChunk->getDiskChunk() != NULL) { throw SYSTEM_EXCEPTION(SCIDB_SE_MERGE, SCIDB_LE_CANT_UPDATE_CHUNK); } outChunk->setCount(0); // unknown } // if (a) either dest is NULL or merge by bitwise-or is possible; and (b) src is not compressed char* dst = static_cast<char*>(outChunk->getData()); if ( (dst == NULL || (outChunk->isPossibleToMergeByBitwiseOr() && !chunkRecord->sparse() && !chunkRecord->rle())) && compMethod == 0 ) { char const* src = (char const*)compressedBuffer->getData(); // Special care is needed if shouldCacheEmptyBitmap. // - If this is the empty bitmap, store it in the SGContext. // - Otherwise, add the empty bitmap from the SGContext to the chunk's data. if (shouldCacheEmptyBitmap) { initMemChunkFromNetwork(pTmpChunk, pinTmpChunk, outputArray, coordinates, attributeID, compMethod, chunkRecord->sparse() || outChunk->isSparse(), rle, compressedBuffer); if (isEmptyIndicator) { sgCtx->setCachedEmptyBitmapChunk(sourceId, pTmpChunk, coordinates); } else { shared_ptr<ConstRLEEmptyBitmap> cachedBitmap = sgCtx->getCachedEmptyBitmap(sourceId, coordinates); assert(cachedBitmap); pinClosure = make_shared<PinBuffer>(closure); closure.initialize(*pTmpChunk); pTmpChunk->makeClosure(closure, cachedBitmap); src = static_cast<char const*>(closure.getData()); } } if (dst == NULL) { outChunk->allocateAndCopy(src, decompressedSize, chunkRecord->sparse(), chunkRecord->rle(), count, _query); } else { outChunk->mergeByBitwiseOr(src, decompressedSize, _query); } } else { initMemChunkFromNetwork(pTmpChunk, pinTmpChunk, outputArray, coordinates, attributeID, compMethod, chunkRecord->sparse() || outChunk->isSparse(), rle, compressedBuffer); ConstChunk const* srcChunk = &(*pTmpChunk); // Special care is needed if shouldCacheEmptyBitmap. // - If this is the empty bitmap, store it in the SGContext. // - Otherwise, add the empty bitmap from the SGContext to the chunk's data. if (shouldCacheEmptyBitmap) { if (isEmptyIndicator) { sgCtx->setCachedEmptyBitmapChunk(sourceId, pTmpChunk, coordinates); } else { shared_ptr<ConstRLEEmptyBitmap> cachedBitmap = sgCtx->getCachedEmptyBitmap(sourceId, coordinates); assert(cachedBitmap); pinClosure = make_shared<PinBuffer>(closure); closure.initialize(*pTmpChunk); pTmpChunk->makeClosure(closure, cachedBitmap); srcChunk = &closure; } } if (isAggregateChunk) { AggregatePtr aggregate = sgCtx->_aggregateList[attributeID]; if (!isEmptyable && rle) { assert(!shouldCacheEmptyBitmap); assert(srcChunk==&(*pTmpChunk)); outChunk->nonEmptyableAggregateMerge(*srcChunk, aggregate, _query); } else { outChunk->aggregateMerge(*srcChunk, aggregate, _query); } } else { outChunk->merge(*srcChunk, _query); } } } else { // new chunk outChunk = &outputIter->newChunk(coordinates, compMethod); outChunk->setSparse(chunkRecord->sparse()); outChunk->setRLE(rle); shared_ptr<CompressedBuffer> myCompressedBuffer = compressedBuffer; // Special care is needed if shouldCacheEmptyBitmap. // - If this is the empty bitmap, store it in the SGContext. // - Otherwise, add the empty bitmap from the SGContext to the chunk's data. if (shouldCacheEmptyBitmap) { initMemChunkFromNetwork(pTmpChunk, pinTmpChunk, outputArray, coordinates, attributeID, compMethod, chunkRecord->sparse() || outChunk->isSparse(), rle, compressedBuffer); if (isEmptyIndicator) { sgCtx->setCachedEmptyBitmapChunk(sourceId, pTmpChunk, coordinates); } else { shared_ptr<ConstRLEEmptyBitmap> cachedBitmap = sgCtx->getCachedEmptyBitmap(sourceId, coordinates); assert(cachedBitmap); myCompressedBuffer = make_shared<CompressedBuffer>(); pTmpChunk->compress(*myCompressedBuffer, cachedBitmap); } } outChunk->decompress(*myCompressedBuffer); outChunk->setCount(count); outChunk->write(_query); } // end if (outputIter->setPosition(coordinates)) assert(checkChunkMagic(*outChunk)); } // end if (compressedBuffer) sgSync(); LOG4CXX_TRACE(logger, "Chunk was stored") } catch(const Exception& e) { sgSync(); throw; } }