// ----------------------------------------------------------------------------
// Tokenizer::openMem
//
// Opens text from a MemChunk [mc]
// ----------------------------------------------------------------------------
bool Tokenizer::openMem(const MemChunk& mc, const string& source)
{
source_ = source;
data_.assign(mc.getData(), mc.getData() + mc.getSize());
reset();
return true;
}
/* AudioEntryPanel::openMod
* Opens a Module file for playback
*******************************************************************/
bool AudioEntryPanel::openMod(MemChunk& data)
{
// Attempt to load the mod
if (mod->loadFromMemory(data.getData(), data.getSize()))
{
audio_type = AUTYPE_MOD;
// Enable playback controls
slider_volume->Enable();
btn_play->Enable();
btn_pause->Enable();
btn_stop->Enable();
setAudioDuration(mod->getDuration().asMilliseconds());
return true;
}
else
{
// Disable playback controls
slider_volume->Enable();
btn_play->Enable();
btn_pause->Enable();
btn_stop->Enable();
setAudioDuration(0);
return false;
}
return false;
}
bool isThisFormat(MemChunk& mc)
{
FIMEMORY* mem = FreeImage_OpenMemory((BYTE*)mc.getData(), mc.getSize());
FREE_IMAGE_FORMAT fif = FreeImage_GetFileTypeFromMemory(mem, 0);
FreeImage_CloseMemory(mem);
if (fif == FIF_UNKNOWN)
return false;
else
return true;
}
/* ArchiveEntry::importMemChunk
* Imports data from a MemChunk object into the entry, resizing it
* and clearing any currently existing data.
* Returns false if the MemChunk has no data, or true otherwise.
*******************************************************************/
bool ArchiveEntry::importMemChunk(MemChunk& mc)
{
// Check that the given MemChunk has data
if (mc.hasData())
{
// Copy the data from the MemChunk into the entry
return importMem(mc.getData(), mc.getSize());
}
else
return false;
}
/* GLTexture::loadImage
* Loads SImage data to the texture. If the dimensions are invalid
* for the system opengl implementation, the data will be split into
* 128x128 squares. Returns false if the given data is invalid, true
* otherwise
*******************************************************************/
bool GLTexture::loadImage(SImage* image, Palette8bit* pal)
{
// Check image was given
if (!image)
return false;
// Check image is valid
if (!image->isValid())
return false;
// Clear current texture
clear();
// Check image dimensions
if (OpenGL::validTexDimension(image->getWidth()) && OpenGL::validTexDimension(image->getHeight()))
{
// If the image dimensions are valid for OpenGL on this system, just load it as a single texture
// Get RGBA image data
MemChunk rgba;
image->getRGBAData(rgba, pal);
// Generate GL texture from rgba data
return loadData(rgba.getData(), image->getWidth(), image->getHeight());
}
else
{
// Otherwise split the image into 128x128 chunks
int top = 0;
while (top < image->getHeight())
{
int left = 0;
while (left < image->getWidth())
{
// Load 128x128 portion of image
loadImagePortion(image, rect_t(left, top, left + 128, top + 128), pal, true);
// Move right 128px
left += 128;
}
// Move down 128px
top += 128;
}
// Update variables
width = image->getWidth();
height = image->getHeight();
scale_x = scale_y = 1.0;
return true;
}
}
/* DecodeTXB
* TXB files are text files with a bit shift xor cipher. It makes an exception
* for the newline character probably so that standard string functions will
* continue to work. As an extension we also except the encoded version of 0xA
* in order to produce a lossless conversion. This allows us to semi-effectively
* handle this at the archive level instead of as a filter at the text editor.
*******************************************************************/
void DecodeTXB(MemChunk &mc)
{
const uint8_t* data = mc.getData();
const uint8_t* const dataend = data + mc.getSize();
uint8_t* odata = new uint8_t[mc.getSize()];
uint8_t* const ostart = odata;
while (data != dataend)
{
if (*data != 0xA && *data != 0x8F)
{
*odata++ = (((*data&0x3F)<<2)|((*data&0xC0)>>6))^0xA7;
++data;
}
else
/* PaletteEntryPanel::saveEntry
* Writes all loaded palettes in the palette entry
*******************************************************************/
bool PaletteEntryPanel::saveEntry()
{
MemChunk full;
MemChunk mc;
// Write each palette as raw data
for (size_t a = 0; a < palettes.size(); a++)
{
palettes[a]->saveMem(mc, Palette8bit::FORMAT_RAW);
full.write(mc.getData(), 768);
}
entry->importMemChunk(full);
setModified(false);
return true;
}
/* DatArchive::open
* Reads wad format data from a MemChunk
* Returns true if successful, false otherwise
*******************************************************************/
bool DatArchive::open(MemChunk& mc)
{
// Check data was given
if (!mc.hasData())
return false;
const uint8_t* mcdata = mc.getData();
// Read dat header
mc.seek(0, SEEK_SET);
uint16_t num_lumps;
uint32_t dir_offset, unknown;
mc.read(&num_lumps, 2); // Size
mc.read(&dir_offset, 4); // Directory offset
mc.read(&unknown, 4); // Unknown value
num_lumps = wxINT16_SWAP_ON_BE(num_lumps);
dir_offset = wxINT32_SWAP_ON_BE(dir_offset);
unknown = wxINT32_SWAP_ON_BE(unknown);
string lastname(wxString::FromAscii("-noname-"));
size_t namecount = 0;
// Stop announcements (don't want to be announcing modification due to entries being added etc)
setMuted(true);
// Read the directory
mc.seek(dir_offset, SEEK_SET);
theSplashWindow->setProgressMessage("Reading dat archive data");
for (uint32_t d = 0; d < num_lumps; d++)
{
// Update splash window progress
theSplashWindow->setProgress(((float)d / (float)num_lumps));
// Read lump info
uint32_t offset = 0;
uint32_t size = 0;
uint16_t nameofs = 0;
uint16_t flags = 0;
mc.read(&offset, 4); // Offset
mc.read(&size, 4); // Size
mc.read(&nameofs, 2); // Name offset
mc.read(&flags, 2); // Flags (only one: RLE encoded)
// Byteswap values for big endian if needed
offset = wxINT32_SWAP_ON_BE(offset);
size = wxINT32_SWAP_ON_BE(size);
nameofs = wxINT16_SWAP_ON_BE(nameofs);
flags = wxINT16_SWAP_ON_BE(flags);
// If the lump data goes past the directory,
// the data file is invalid
if (offset + size > mc.getSize())
{
wxLogMessage("DatArchive::open: Dat archive is invalid or corrupt at entry %i", d);
Global::error = "Archive is invalid and/or corrupt";
setMuted(false);
return false;
}
string myname;
if (nameofs != 0)
{
size_t len = 1;
size_t start = nameofs+dir_offset;
for (size_t i = start; mcdata[i] != 0; ++i) { ++len; }
lastname = myname = wxString::FromAscii(mcdata+start, len);
namecount = 0;
}
else
{
myname = S_FMT("%s+%d", lastname, ++namecount);
}
// Create & setup lump
ArchiveEntry* nlump = new ArchiveEntry(myname, size);
nlump->setLoaded(false);
nlump->exProp("Offset") = (int)offset;
nlump->setState(0);
if (flags & 1) nlump->setEncryption(ENC_SCRLE0);
// Check for markers
if (!nlump->getName().Cmp("startflats"))
flats[0] = d;
if (!nlump->getName().Cmp("endflats"))
flats[1] = d;
if (!nlump->getName().Cmp("startsprites"))
sprites[0] = d;
if (!nlump->getName().Cmp("endmonsters"))
sprites[1] = d;
if (!nlump->getName().Cmp("startwalls"))
walls[0] = d;
if (!nlump->getName().Cmp("endwalls"))
walls[1] = d;
// Add to entry list
getRoot()->addEntry(nlump);
}
// Detect all entry types
MemChunk edata;
theSplashWindow->setProgressMessage("Detecting entry types");
for (size_t a = 0; a < numEntries(); a++)
{
// Update splash window progress
theSplashWindow->setProgress((((float)a / (float)num_lumps)));
// Get entry
ArchiveEntry* entry = getEntry(a);
// Read entry data if it isn't zero-sized
if (entry->getSize() > 0)
{
// Read the entry data
mc.exportMemChunk(edata, getEntryOffset(entry), entry->getSize());
entry->importMemChunk(edata);
}
// Detect entry type
EntryType::detectEntryType(entry);
// Set entry to unchanged
entry->setState(0);
}
// Detect maps (will detect map entry types)
//theSplashWindow->setProgressMessage("Detecting maps");
//detectMaps();
// Setup variables
setMuted(false);
setModified(false);
announce("opened");
theSplashWindow->setProgressMessage("");
return true;
}
/* AudioEntryPanel::openAudio
* Opens an audio file for playback (SFML 2.x+)
*******************************************************************/
bool AudioEntryPanel::openAudio(MemChunk& audio, string filename)
{
// Stop if sound currently playing
resetStream();
// (Re)create sound buffer
if (sound_buffer)
delete sound_buffer;
sound_buffer = new sf::SoundBuffer();
audio_type = AUTYPE_INVALID;
// Load into buffer
if (sound_buffer->loadFromMemory((const char*)audio.getData(), audio.getSize()))
{
LOG_MESSAGE(3, "opened as sound");
// Bind to sound
sound->setBuffer(*sound_buffer);
audio_type = AUTYPE_SOUND;
// Enable play controls
#if (SFML_VERSION_MAJOR == 2 && SFML_VERSION_MINOR < 2)
// SFML before 2.2 has a bug where it reports an incorrect value for long sounds, so compute it ourselves then
setAudioDuration((sound_buffer->getSampleCount() / sound_buffer->getSampleRate())*(1000/sound_buffer->getChannelCount()));
#else
setAudioDuration(sound_buffer->getDuration().asMilliseconds());
#endif
btn_play->Enable();
btn_pause->Enable();
btn_stop->Enable();
return true;
}
else if (music->openFromMemory((const char*)audio.getData(), audio.getSize()))
{
LOG_MESSAGE(3, "opened as music");
// Couldn't open the audio as a sf::SoundBuffer, try sf::Music instead
audio_type = AUTYPE_MUSIC;
// Enable play controls
setAudioDuration(music->getDuration().asMilliseconds());
btn_play->Enable();
btn_stop->Enable();
return true;
}
else
{
// Couldn't open as sound or music, try the wxMediaCtrl
LOG_MESSAGE(3, "opened as media");
// Dump audio to temp file
audio.exportFile(filename);
if (openMedia(filename))
return true;
}
// Unable to open audio, disable play controls
setAudioDuration(0);
btn_play->Enable(false);
btn_pause->Enable(false);
btn_stop->Enable(false);
return false;
}
/* ADatArchive::write
* Writes the dat archive to a MemChunk
* Returns true if successful, false otherwise
*******************************************************************/
bool ADatArchive::write(MemChunk& mc, bool update) {
// Clear current data
mc.clear();
MemChunk directory;
MemChunk compressed;
// Get archive tree as a list
vector<ArchiveEntry*> entries;
getEntryTreeAsList(entries);
// Write header
long dir_offset = wxINT32_SWAP_ON_BE(16);
long dir_size = wxINT32_SWAP_ON_BE(0);
char pack[4] = { 'A', 'D', 'A', 'T' };
uint32_t version = wxINT32_SWAP_ON_BE(9);
mc.seek(0, SEEK_SET);
mc.write(pack, 4);
mc.write(&dir_offset, 4);
mc.write(&dir_size, 4);
mc.write(&version, 4);
// Write entry data
for (unsigned a = 0; a < entries.size(); a++) {
// Skip folders
if (entries[a]->getType() == EntryType::folderType())
continue;
// Create compressed version of the lump
MemChunk * entry = NULL;
if (Compression::ZlibDeflate(entries[a]->getMCData(), compressed, 9)) {
entry = &compressed;
} else {
entry = &(entries[a]->getMCData());
wxLogMessage("Entry %s couldn't be deflated", CHR(entries[a]->getName()));
}
// Update entry
int offset = mc.currentPos();
if (update) {
entries[a]->setState(0);
entries[a]->exProp("Offset") = (int)offset;
}
///////////////////////////////////
// Step 1: Write directory entry //
///////////////////////////////////
// Check entry name
string name = entries[a]->getPath(true);
name.Remove(0, 1); // Remove leading /
if (name.Len() > 128) {
wxLogMessage("Warning: Entry %s path is too long (> 128 characters), putting it in the root directory", CHR(name));
wxFileName fn(name);
name = fn.GetFullName();
if (name.Len() > 128)
name.Truncate(128);
}
// Write entry name
char name_data[128];
memset(name_data, 0, 128);
memcpy(name_data, CHR(name), name.Length());
directory.write(name_data, 128);
// Write entry offset
long myoffset = wxINT32_SWAP_ON_BE(offset);
directory.write(&myoffset, 4);
// Write full entry size
long decsize = wxINT32_SWAP_ON_BE(entries[a]->getSize());
directory.write(&decsize, 4);
// Write compressed entry size
long compsize = wxINT32_SWAP_ON_BE(entry->getSize());
directory.write(&compsize, 4);
// Write whatever it is that should be there
// TODO: Reverse engineer what exactly it is
// and implement something valid for the game.
long whatever = 0;
directory.write(&whatever, 4);
//////////////////////////////
// Step 2: Write entry data //
//////////////////////////////
mc.write(entry->getData(), entry->getSize());
}
// Write directory
dir_offset = wxINT32_SWAP_ON_BE(mc.currentPos());
dir_size = wxINT32_SWAP_ON_BE(directory.getSize());
mc.write(directory.getData(), directory.getSize());
// Update directory offset and size in header
mc.seek(4, SEEK_SET);
mc.write(&dir_offset, 4);
mc.write(&dir_size, 4);
// Yay! Finished!
return true;
}
/* RffArchive::open
* Reads grp format data from a MemChunk
* Returns true if successful, false otherwise
*******************************************************************/
bool RffArchive::open(MemChunk& mc)
{
// Check data was given
if (!mc.hasData())
return false;
// Read grp header
uint8_t magic[4];
uint32_t version, dir_offset, num_lumps;
mc.seek(0, SEEK_SET);
mc.read(magic, 4); // Should be "RFF\x18"
mc.read(&version, 4); // 0x01 0x03 \x00 \x00
mc.read(&dir_offset, 4); // Offset to directory
mc.read(&num_lumps, 4); // No. of lumps in rff
// Byteswap values for big endian if needed
dir_offset = wxINT32_SWAP_ON_BE(dir_offset);
num_lumps = wxINT32_SWAP_ON_BE(num_lumps);
version = wxINT32_SWAP_ON_BE(version);
// Check the header
if (magic[0] != 'R' || magic[1] != 'F' || magic[2] != 'F' || magic[3] != 0x1A || version != 0x301)
{
wxLogMessage("RffArchive::openFile: File %s has invalid header", filename);
Global::error = "Invalid rff header";
return false;
}
// Stop announcements (don't want to be announcing modification due to entries being added etc)
setMuted(true);
// Read the directory
RFFLump* lumps = new RFFLump[num_lumps];
mc.seek(dir_offset, SEEK_SET);
theSplashWindow->setProgressMessage("Reading rff archive data");
mc.read (lumps, num_lumps * sizeof(RFFLump));
BloodCrypt (lumps, dir_offset, num_lumps * sizeof(RFFLump));
for (uint32_t d = 0; d < num_lumps; d++)
{
// Update splash window progress
theSplashWindow->setProgress(((float)d / (float)num_lumps));
// Read lump info
char name[13] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
uint32_t offset = wxINT32_SWAP_ON_BE(lumps[d].FilePos);
uint32_t size = wxINT32_SWAP_ON_BE(lumps[d].Size);
// Reconstruct name
int i, j = 0;
for (i = 0; i < 8; ++i)
{
if (lumps[d].Name[i] == 0) break;
name[i] = lumps[d].Name[i];
}
for (name[i++] = '.'; j < 3; ++j)
name[i+j] = lumps[d].Extension[j];
// If the lump data goes past the end of the file,
// the rfffile is invalid
if (offset + size > mc.getSize())
{
wxLogMessage("RffArchive::open: rff archive is invalid or corrupt");
Global::error = "Archive is invalid and/or corrupt";
setMuted(false);
return false;
}
// Create & setup lump
ArchiveEntry* nlump = new ArchiveEntry(wxString::FromAscii(name), size);
nlump->setLoaded(false);
nlump->exProp("Offset") = (int)offset;
nlump->setState(0);
// Is the entry encrypted?
if (lumps[d].Flags & 0x10)
nlump->setEncryption(ENC_BLOOD);
// Add to entry list
getRoot()->addEntry(nlump);
}
delete[] lumps;
// Detect all entry types
MemChunk edata;
theSplashWindow->setProgressMessage("Detecting entry types");
for (size_t a = 0; a < numEntries(); a++)
{
// Update splash window progress
theSplashWindow->setProgress((((float)a / (float)num_lumps)));
// Get entry
ArchiveEntry* entry = getEntry(a);
// Read entry data if it isn't zero-sized
if (entry->getSize() > 0)
{
// Read the entry data
mc.exportMemChunk(edata, getEntryOffset(entry), entry->getSize());
// If the entry is encrypted, decrypt it
if (entry->isEncrypted())
{
uint8_t* cdata = new uint8_t[entry->getSize()];
memcpy(cdata, edata.getData(), entry->getSize());
int cryptlen = entry->getSize() < 256 ? entry->getSize() : 256;
BloodCrypt(cdata, 0, cryptlen);
edata.importMem(cdata, entry->getSize());
delete[] cdata;
}
// Import data
entry->importMemChunk(edata);
}
// Detect entry type
EntryType::detectEntryType(entry);
// Unload entry data if needed
if (!archive_load_data)
entry->unloadData();
// Set entry to unchanged
entry->setState(0);
}
// Detect maps (will detect map entry types)
//theSplashWindow->setProgressMessage("Detecting maps");
//detectMaps();
// Setup variables
setMuted(false);
setModified(false);
announce("opened");
theSplashWindow->setProgressMessage("");
return true;
}
bool PaletteEntryPanel::generateColormaps()
{
if (!entry || !entry->getParent() || ! palettes[0])
return false;
MemChunk mc;
SImage img;
MemChunk imc;
mc.reSize(34*256);
mc.seek(0, SEEK_SET);
imc.reSize(34*256*4);
imc.seek(0, SEEK_SET);
uint8_t rgba[4];
rgba[3] = 255;
rgba_t rgb;
float grey;
// Generate 34 maps: the first 32 for diminishing light levels,
// the 33th for the inverted grey map used by invulnerability.
// The 34th colormap remains empty and black.
for (size_t l = 0; l < 34; ++l)
{
for (size_t c = 0; c < 256; ++c)
{
rgb = palettes[0]->colour(c);
if (l < 32)
{
// Generate light maps
DIMINISH(rgb.r, l);
DIMINISH(rgb.g, l);
DIMINISH(rgb.b, l);
#if (0)
}
else if (l == GREENMAP)
{
// Point of mostly useless trivia: the green "light amp" colormap in the Press Release beta
// have colors that, on average, correspond to a bit less than (R*75/256, G*225/256, B*115/256)
#endif
}
else if (l == GRAYMAP)
{
// Generate inverse map
grey = ((float)rgb.r/256.0 * col_greyscale_r) + ((float)rgb.g/256.0 * col_greyscale_g) + ((float)rgb.b/256.0 * col_greyscale_b);
grey = 1.0 - grey;
// Clamp value: with Id Software's values, the sum is greater than 1.0 (0.299+0.587+0.144=1.030)
// This means the negation above can give a negative value (for example, with RGB values of 247 or more),
// which will not be converted correctly to unsigned 8-bit int in the rgba_t struct.
if (grey < 0.0) grey = 0;
rgb.r = rgb.g = rgb.b = grey*255;
}
else
{
// Fill with 0
rgb = palettes[0]->colour(0);
}
rgba[0] = rgb.r; rgba[1] = rgb.g; rgba[2] = rgb.b;
imc.write(&rgba, 4);
mc[(256*l)+c] = palettes[0]->nearestColour(rgb);
}
}
#if 0
// Create truecolor image
uint8_t* imd = new uint8_t[256*34*4];
memcpy(imd, imc.getData(), 256*34*4);
img.setImageData(imd, 256, 34, RGBA);
// imd will be freed by img's destructor
ArchiveEntry* tcolormap;
string name = entry->getName(true) + "-tcm.png";
tcolormap = new ArchiveEntry(name);
if (tcolormap)
{
entry->getParent()->addEntry(tcolormap);
SIFormat::getFormat("png")->saveImage(img, tcolormap->getMCData());
EntryType::detectEntryType(tcolormap);
}
#endif
// Now override or create new entry
ArchiveEntry* colormap;
colormap = entry->getParent()->getEntry("COLORMAP", true);
bool preexisting = colormap != NULL;
if (!colormap)
{
// We need to create this entry
colormap = new ArchiveEntry("COLORMAP.lmp", 34*256);
}
if (!colormap)
return false;
colormap->importMemChunk(mc);
if (!preexisting)
{
entry->getParent()->addEntry(colormap);
}
return true;
}
/* GLTexture::loadImagePortion
* Loads a portion of a SImage to the texture. Only used internally,
* the portion must be 128x128 in size
*******************************************************************/
bool GLTexture::loadImagePortion(SImage* image, rect_t rect, Palette8bit* pal, bool add)
{
// Check image was given
if (!image)
return false;
// Check image is valid
if (!image->isValid())
return false;
// Check portion rect is valid
if (rect.width() <= 0 || rect.height() <= 0)
return false;
// Get RGBA image data
MemChunk rgba;
image->getRGBAData(rgba, pal);
// Init texture data
MemChunk portion;
portion.reSize(rect.width() * rect.height() * 4, false);
portion.fillData(0);
// Read portion of image if rect isn't completely outside the image
if (!(rect.left() >= image->getWidth() || rect.right() < 0 || rect.top() >= image->getHeight() || rect.bottom() < 0))
{
// Determine start of each row to read
uint32_t row_start = 0;
if (rect.left() > 0)
row_start = rect.left();
// Determine width of each row to read
uint32_t row_width = rect.right() - row_start;
if (rect.right() >= image->getWidth())
row_width = image->getWidth() - row_start;
// Determine difference between the left of the portion and the left of the image
uint32_t skip = 0;
if (rect.left() < 0)
skip = (0 - rect.left()) * 4;
// Create temp row buffer
uint8_t* buf = new uint8_t[rect.width() * 4];
// Go through each row
for (int32_t row = rect.top(); row < rect.bottom(); row++)
{
// Clear row buffer
memset(buf, 0, rect.width() * 4);
// Check that the current row is within the image
if (row >= 0 && row < image->getHeight())
{
// Seek to current row in image data
rgba.seek((row * image->getWidth() + row_start) * 4, SEEK_SET);
// Copy the row data
rgba.read(buf + skip, row_width * 4);
}
// Write the row
portion.write(buf, rect.width() * 4);
}
// Free buffer
delete[] buf;
}
scale_x = scale_y = 1.0;
// Generate texture from rgba data
return loadData(portion.getData(), rect.width(), rect.height(), add);
}
/* GZipArchive::open
* Reads gzip format data from a MemChunk
* Returns true if successful, false otherwise
*******************************************************************/
bool GZipArchive::open(MemChunk& mc)
{
// Minimal metadata size is 18: 10 for header, 8 for footer
size_t mds = 18;
size_t size = mc.getSize();
if (mds > size)
return false;
// Read header
uint8_t header[4];
mc.read(header, 4);
// Check for GZip header; we'll only accept deflated gzip files
// and reject any field using unknown flags
if ((!(header[0] == GZIP_ID1 && header[1] == GZIP_ID2 && header[2] == GZIP_DEFLATE))
|| (header[3] & GZIP_FLG_FUNKN))
return false;
bool ftext, fhcrc, fxtra, fname, fcmnt;
ftext = (header[3] & GZIP_FLG_FTEXT) ? true : false;
fhcrc = (header[3] & GZIP_FLG_FHCRC) ? true : false;
fxtra = (header[3] & GZIP_FLG_FXTRA) ? true : false;
fname = (header[3] & GZIP_FLG_FNAME) ? true : false;
fcmnt = (header[3] & GZIP_FLG_FCMNT) ? true : false;
flags = header[3];
mc.read(&mtime, 4);
mtime = wxUINT32_SWAP_ON_BE(mtime);
mc.read(&xfl, 1);
mc.read(&os, 1);
// Skip extra fields which may be there
if (fxtra)
{
uint16_t xlen;
mc.read(&xlen, 2);
xlen = wxUINT16_SWAP_ON_BE(xlen);
mds += xlen + 2;
if (mds > size)
return false;
mc.exportMemChunk(xtra, mc.currentPos(), xlen);
mc.seek(xlen, SEEK_CUR);
}
// Skip past name, if any
string name;
if (fname)
{
char c;
do
{
mc.read(&c, 1);
if (c) name += c;
++mds;
}
while (c != 0 && size > mds);
}
else
{
// Build name from filename
name = getFilename(false);
wxFileName fn(name);
if (!fn.GetExt().CmpNoCase("tgz"))
fn.SetExt("tar");
else if (!fn.GetExt().CmpNoCase("gz"))
fn.ClearExt();
name = fn.GetFullName();
}
// Skip past comment
if (fcmnt)
{
char c;
do
{
mc.read(&c, 1);
if (c) comment += c;
++mds;
}
while (c != 0 && size > mds);
wxLogMessage("Archive %s says:\n %s", CHR(getFilename(true)), CHR(comment));
}
// Skip past CRC 16 check
if (fhcrc)
{
uint8_t* crcbuffer = new uint8_t[mc.currentPos()];
memcpy(crcbuffer, mc.getData(), mc.currentPos());
uint32_t fullcrc = Misc::crc(crcbuffer, mc.currentPos());
delete[] crcbuffer;
uint16_t hcrc;
mc.read(&hcrc, 2);
hcrc = wxUINT16_SWAP_ON_BE(hcrc);
mds += 2;
if (hcrc != (fullcrc & 0x0000FFFF))
{
wxLogMessage("CRC-16 mismatch for GZip header");
}
}
// Header is over
if (mds > size || mc.currentPos() + 8 > size)
return false;
// Let's create the entry
setMuted(true);
ArchiveEntry* entry = new ArchiveEntry(name, size - mds);
MemChunk xdata;
if (Compression::GZipInflate(mc, xdata))
{
entry->importMemChunk(xdata);
}
else
{
delete entry;
setMuted(false);
return false;
}
getRoot()->addEntry(entry);
EntryType::detectEntryType(entry);
entry->setState(0);
setMuted(false);
setModified(false);
announce("opened");
// Finish
return true;
}
/* GZipArchive::write
* Writes the gzip archive to a MemChunk
* Returns true if successful, false otherwise
*******************************************************************/
bool GZipArchive::write(MemChunk& mc, bool update)
{
// Clear current data
mc.clear();
if (numEntries() == 1)
{
MemChunk stream;
if (Compression::GZipDeflate(getEntry(0)->getMCData(), stream, 9))
{
const uint8_t* data = stream.getData();
uint32_t working = 0;
size_t size = stream.getSize();
if (size < 18) return false;
// zlib will have given us a minimal header, so we make our own
uint8_t header[4];
header[0] = GZIP_ID1; header[1] = GZIP_ID2;
header[2] = GZIP_DEFLATE; header[3] = flags;
mc.write(header, 4);
// Update mtime if the file was modified
if (getEntry(0)->getState())
{
mtime = ::wxGetLocalTime();
}
// Write mtime
working = wxUINT32_SWAP_ON_BE(mtime);
mc.write(&working, 4);
// Write other stuff
mc.write(&xfl, 1);
mc.write(&os, 1);
// Any extra content that may have been there
if (flags & GZIP_FLG_FXTRA)
{
uint16_t xlen = wxUINT16_SWAP_ON_BE(xtra.getSize());
mc.write(&xlen, 2);
mc.write(xtra.getData(), xtra.getSize());
}
// File name, if not extrapolated from archive name
if (flags & GZIP_FLG_FNAME)
{
mc.write(CHR(getEntry(0)->getName()), getEntry(0)->getName().length());
uint8_t zero = 0; mc.write(&zero, 1); // Terminate string
}
// Comment, if there were actually one
if (flags & GZIP_FLG_FCMNT)
{
mc.write(CHR(comment), comment.length());
uint8_t zero = 0; mc.write(&zero, 1); // Terminate string
}
// And finally, the half CRC, which we recalculate
if (flags & GZIP_FLG_FHCRC)
{
uint32_t fullcrc = Misc::crc(mc.getData(), mc.getSize());
uint16_t hcrc = (fullcrc & 0x0000FFFF);
hcrc = wxUINT16_SWAP_ON_BE(hcrc);
mc.write(&hcrc, 2);
}
// Now that the pleasantries are dispensed with,
// let's get with the meat of the matter
return mc.write(data + 10, size - 10);
}
}
return false;
}
void MessageHandleJob::_handleChunkOrAggregateChunk(bool isAggregateChunk)
{
boost::shared_ptr<scidb_msg::Chunk> chunkRecord = _messageDesc->getRecord<scidb_msg::Chunk>();
assert(!chunkRecord->eof());
assert(_query);
RWLock::ErrorChecker noopEc;
ScopedRWLockRead shared(_query->queryLock, noopEc);
try
{
if (! isAggregateChunk) {
// TODO: Apply it to statistics of current SG
currentStatistics->receivedSize += _messageDesc->getMessageSize();
currentStatistics->receivedMessages++;
}
LOG4CXX_TRACE(logger, "Next chunk message was received")
const int compMethod = chunkRecord->compression_method();
const size_t decompressedSize = chunkRecord->decompressed_size();
const AttributeID attributeID = chunkRecord->attribute_id();
const size_t count = chunkRecord->count();
boost::shared_ptr<SGContext> sgCtx = dynamic_pointer_cast<SGContext>(_query->getOperatorContext());
if (sgCtx == NULL) {
throw (SYSTEM_EXCEPTION(SCIDB_SE_INTERNAL, SCIDB_LE_UNKNOWN_CTX)
<< typeid(*_query->getOperatorContext()).name());
}
boost::shared_ptr<Array> outputArray = sgCtx->_resultSG;
ScopedMutexLock cs(_query->resultCS);
boost::shared_ptr<ArrayIterator> outputIter = outputArray->getIterator(attributeID);
const bool shouldCacheEmptyBitmap = sgCtx->_shouldCacheEmptyBitmap;
const ArrayDesc& desc = outputArray->getArrayDesc();
const size_t sourceId = _query->mapPhysicalToLogical(_messageDesc->getSourceInstanceID());
const bool isEmptyable = (desc.getEmptyBitmapAttribute() != NULL);
const bool isEmptyIndicator = isEmptyable && (attributeID+1==desc.getAttributes().size());
const bool rle = chunkRecord->rle();
if (isAggregateChunk) {
assert(! isEmptyIndicator);
}
Coordinates coordinates;
for (int i = 0; i < chunkRecord->coordinates_size(); i++) {
coordinates.push_back(chunkRecord->coordinates(i));
}
if (!isAggregateChunk && sgCtx->_targetVersioned)
{
sgCtx->_newChunks.insert(coordinates);
}
boost::shared_ptr<CompressedBuffer> compressedBuffer = dynamic_pointer_cast<CompressedBuffer>(_messageDesc->getBinary());
if (compressedBuffer) {
assert(compressedBuffer->getData());
PinBuffer pin(*compressedBuffer); // this line protects compressedBuffer->data from being freed, allowing MemChunk::decompressed(*compressedBuffer) to be called multiple times.
compressedBuffer->setCompressionMethod(compMethod);
compressedBuffer->setDecompressedSize(decompressedSize);
Chunk* outChunk;
// temporary MemArray objects
shared_ptr<MemChunk> pTmpChunk = make_shared<MemChunk>(); // make it a shared pointer, because the bitmap chunk needs to be preserved across threads.
MemChunk closure;
// the PinBuffer objects protect tmpChunk and closure, respectively.
shared_ptr<PinBuffer> pinTmpChunk, pinClosure;
if (outputIter->setPosition(coordinates)) { // existing chunk
outChunk = &outputIter->updateChunk();
if (! isAggregateChunk) {
if (outChunk->getDiskChunk() != NULL) {
throw SYSTEM_EXCEPTION(SCIDB_SE_MERGE, SCIDB_LE_CANT_UPDATE_CHUNK);
}
outChunk->setCount(0); // unknown
}
// if (a) either dest is NULL or merge by bitwise-or is possible; and (b) src is not compressed
char* dst = static_cast<char*>(outChunk->getData());
if ( (dst == NULL || (outChunk->isPossibleToMergeByBitwiseOr() && !chunkRecord->sparse() && !chunkRecord->rle()))
&&
compMethod == 0 )
{
char const* src = (char const*)compressedBuffer->getData();
// Special care is needed if shouldCacheEmptyBitmap.
// - If this is the empty bitmap, store it in the SGContext.
// - Otherwise, add the empty bitmap from the SGContext to the chunk's data.
if (shouldCacheEmptyBitmap) {
initMemChunkFromNetwork(pTmpChunk, pinTmpChunk, outputArray, coordinates, attributeID,
compMethod, chunkRecord->sparse() || outChunk->isSparse(), rle, compressedBuffer);
if (isEmptyIndicator) {
sgCtx->setCachedEmptyBitmapChunk(sourceId, pTmpChunk, coordinates);
} else {
shared_ptr<ConstRLEEmptyBitmap> cachedBitmap = sgCtx->getCachedEmptyBitmap(sourceId, coordinates);
assert(cachedBitmap);
pinClosure = make_shared<PinBuffer>(closure);
closure.initialize(*pTmpChunk);
pTmpChunk->makeClosure(closure, cachedBitmap);
src = static_cast<char const*>(closure.getData());
}
}
if (dst == NULL) {
outChunk->allocateAndCopy(src, decompressedSize, chunkRecord->sparse(), chunkRecord->rle(), count, _query);
} else {
outChunk->mergeByBitwiseOr(src, decompressedSize, _query);
}
} else {
initMemChunkFromNetwork(pTmpChunk, pinTmpChunk, outputArray, coordinates, attributeID,
compMethod, chunkRecord->sparse() || outChunk->isSparse(), rle, compressedBuffer);
ConstChunk const* srcChunk = &(*pTmpChunk);
// Special care is needed if shouldCacheEmptyBitmap.
// - If this is the empty bitmap, store it in the SGContext.
// - Otherwise, add the empty bitmap from the SGContext to the chunk's data.
if (shouldCacheEmptyBitmap) {
if (isEmptyIndicator) {
sgCtx->setCachedEmptyBitmapChunk(sourceId, pTmpChunk, coordinates);
} else {
shared_ptr<ConstRLEEmptyBitmap> cachedBitmap = sgCtx->getCachedEmptyBitmap(sourceId, coordinates);
assert(cachedBitmap);
pinClosure = make_shared<PinBuffer>(closure);
closure.initialize(*pTmpChunk);
pTmpChunk->makeClosure(closure, cachedBitmap);
srcChunk = &closure;
}
}
if (isAggregateChunk) {
AggregatePtr aggregate = sgCtx->_aggregateList[attributeID];
if (!isEmptyable && rle) {
assert(!shouldCacheEmptyBitmap);
assert(srcChunk==&(*pTmpChunk));
outChunk->nonEmptyableAggregateMerge(*srcChunk, aggregate, _query);
} else {
outChunk->aggregateMerge(*srcChunk, aggregate, _query);
}
} else {
outChunk->merge(*srcChunk, _query);
}
}
} else { // new chunk
outChunk = &outputIter->newChunk(coordinates, compMethod);
outChunk->setSparse(chunkRecord->sparse());
outChunk->setRLE(rle);
shared_ptr<CompressedBuffer> myCompressedBuffer = compressedBuffer;
// Special care is needed if shouldCacheEmptyBitmap.
// - If this is the empty bitmap, store it in the SGContext.
// - Otherwise, add the empty bitmap from the SGContext to the chunk's data.
if (shouldCacheEmptyBitmap) {
initMemChunkFromNetwork(pTmpChunk, pinTmpChunk, outputArray, coordinates, attributeID,
compMethod, chunkRecord->sparse() || outChunk->isSparse(), rle, compressedBuffer);
if (isEmptyIndicator) {
sgCtx->setCachedEmptyBitmapChunk(sourceId, pTmpChunk, coordinates);
} else {
shared_ptr<ConstRLEEmptyBitmap> cachedBitmap = sgCtx->getCachedEmptyBitmap(sourceId, coordinates);
assert(cachedBitmap);
myCompressedBuffer = make_shared<CompressedBuffer>();
pTmpChunk->compress(*myCompressedBuffer, cachedBitmap);
}
}
outChunk->decompress(*myCompressedBuffer);
outChunk->setCount(count);
outChunk->write(_query);
} // end if (outputIter->setPosition(coordinates))
assert(checkChunkMagic(*outChunk));
} // end if (compressedBuffer)
sgSync();
LOG4CXX_TRACE(logger, "Chunk was stored")
}
catch(const Exception& e)
{
sgSync();
throw;
}
}
