void BlockFileInputStreamTestCase::testSeekAndRead()
{
OutputStreamPtr pOutput = m_pBlockFileSystem->createFile("testfile1");
size_t dataSize = 10000;
stringstream content;
for (size_t i = 0; i < dataSize; ++i)
{
content << (i % 10);
}
pOutput->write((const void*)content.str().c_str(), content.str().size());
pOutput.reset();
InputStreamPtr pInput = m_pBlockFileSystem->openFile("testfile1");
CPPUNIT_ASSERT(pInput != NULL);
CPPUNIT_ASSERT_EQUAL(dataSize, (size_t)pInput->getSize());
for (size_t i = 0; i < dataSize - 1; ++i)
{
string readCont;
readCont.resize(dataSize - i);
if (i % 147 == 0)
{
pInput->seek(i % 789);
}
pInput->seekAndRead(i, (void*)readCont.c_str(), dataSize - i);
string expStr = content.str().substr(i);
// assert(expStr == readCont);
CPPUNIT_ASSERT_EQUAL(expStr, readCont);
}
pInput.reset();
}
void
Ice::Object::__read(const InputStreamPtr& is)
{
is->startObject();
__readImpl(is);
is->endObject(false);
}
void BlockFileOutputStreamTestCase::testWriteFromInputStream()
{
const static size_t MAX_TEST_DATA = 10000;
for (size_t i = 1; i < MAX_TEST_DATA; i += 31)
{
OutputStreamPtr pOutput = m_pBlockFileSystem->createFile("testfile1");
string answer;
makeData(answer, *pOutput, i);
pOutput.reset();
InputStreamPtr pInput = m_pBlockFileSystem->openFile("testfile1");
pOutput = m_pBlockFileSystem->createFile("testfile2");
pOutput->write(*pInput);
pOutput.reset();
pInput = m_pBlockFileSystem->openFile("testfile2");
string expStr;
expStr.resize(i);
pInput->read((void*)expStr.c_str(), i);
assert(answer==expStr);
CPPUNIT_ASSERT_EQUAL(answer, expStr);
}
}
void BlockFileOutputStreamTestCase::testSeekAndWrite()
{
size_t dataSize = 10000;
stringstream content;
for (size_t i = 0; i < dataSize; ++i)
{
content << (i % 10);
}
for (size_t i = 0; i < dataSize - 1; ++i)
{
OutputStreamPtr pOutput = m_pBlockFileSystem->createFile("testfile1");
pOutput->write((const void*) content.str().c_str(), i);
if (i % 137)
{
pOutput->write((const void*) content.str().c_str(), i % dataSize);
}
pOutput->seek(i);
pOutput->write((const void*) (content.str().c_str() + i), dataSize - i);
pOutput->close();
InputStreamPtr pInput = m_pBlockFileSystem->openFile("testfile1");
string readCont;
readCont.resize(dataSize);
pInput->read((void*)readCont.c_str(), dataSize);
CPPUNIT_ASSERT_EQUAL(content.str(), readCont);
}
}
void XMLConfigurator::load(const string& sCfgFile, FileSystemPtr& pFileSys)
{
InputStreamPtr pInStream = pFileSys->openFile(sCfgFile);
string str;
str.resize((size_t)pInStream->getSize());
pInStream->read((void*)str.c_str(), str.length());
loadFromBuffer(str);
}
void Config::LoadConfiguration(const URL& path) {
NEX_THREAD_LOCK_GUARD_MUTEX(contentLock);
FileSystem& fileSys = FileSystem::Instance();
InputStreamPtr inputStream = fileSys.OpenRead(path);
if (inputStream) {
const void* readOnlyBuffer = 0;
size_t size = inputStream->AcquireBuffer(readOnlyBuffer);
StreamLexer input(static_cast<const char*>(readOnlyBuffer), size);
// read
String section;
NameValueMap* current = §ions[""];
input.SkipWhite();
while (!input.IsEndOfStream()) {
switch (input.Forward()) {
case '[': {
input.SkipWhite();
section = input.ReadWord("] \n\t\r");
current = §ions[section];
if (NEX_IS_DEBUG_MODE()) {
input.SkipWhite();
if (input.Current() != ']') {
Warn(
"expected ']', in line: "
+ Convert::ToString(input.Line())
+ " in config: "
+ path.GetRelativePath());
}
}
input.SkipLine();
}
break;
default: {
input.Backward();
String name = input.ReadWord("= \n\t\r");
input.SkipWhite();
if (input.Forward() == '=') {
const String& value = input.ReadStringLiteral();
(*current)[name] = value;
} else {
Warn(
"illegal expression in line: "
+ Convert::ToString(input.Line())
+ " in config: " + path.GetRelativePath());
input.SkipLine();
}
}
break;
}
input.SkipWhite();
}
inputStream->ReleaseBuffer(readOnlyBuffer);
}
}
InputStreamPtr InputStreamPool::getInputStream(const std::string& sFileName) const
{
{
ScopedLock<FastMutex> lock(m_mutex);
InputStreamPtr pInStream = doGetInputStream(sFileName);
if (pInStream.isNotNull())
{
return pInStream;
}
}
return m_pFileSystem->openFile(sFileName);
}
bool PNGImageCodec::TryLoad(InputStreamPtr& file, const ImageParams& params,
ImageCodecMetaInfo& metaInfo) {
char data[PNG_BYTES_TO_CHECK];
file->Read(data, PNG_BYTES_TO_CHECK);
return png_sig_cmp((png_bytep) data, (png_size_t) 0, PNG_BYTES_TO_CHECK)
== 0;
}
void InputStreamPool::releaseInputStream(InputStreamPtr& pInputStream) const
{
{
ScopedLock<FastMutex> lock(m_mutex);
--m_nOpenedStreamCount;
StreamVector& streams = m_streamMap[pInputStream->getFilePath()];
if (streams.size() < m_nPoolSize)
{
InputStreamPtr pTmpPtr = pInputStream;
streams.push_back(pTmpPtr);
return;
}
}
pInputStream.reset();
}
void BlockFileInputStreamTestCase::testReadInt()
{
OutputStreamPtr pOutput = m_pBlockFileSystem->createFile("testfile1");
uint8_t ui8v =81;
int32_t i32v = 321;
int64_t i64v = ((int64_t)1 << 32) + 641;
string str(129, '1');
pOutput->writeByte(ui8v);
pOutput->writeInt32(i32v);
pOutput->write(str.c_str(), str.size());
pOutput->writeVInt32(i32v);
pOutput->writeInt64(i64v);
pOutput->writeVInt64(i64v);
pOutput.reset();
InputStreamPtr pInput = m_pBlockFileSystem->openFile("testfile1");
CPPUNIT_ASSERT(pInput);
string expStr;
expStr.resize(str.size());
CPPUNIT_ASSERT_EQUAL(ui8v, pInput->readByte());
CPPUNIT_ASSERT_EQUAL(i32v, pInput->readInt32());
pInput->read((void*)expStr.c_str(), str.size());
CPPUNIT_ASSERT_EQUAL(str, expStr);
CPPUNIT_ASSERT_EQUAL(i32v, pInput->readVInt32());
CPPUNIT_ASSERT_EQUAL(i64v, pInput->readInt64());
CPPUNIT_ASSERT_EQUAL(i64v, pInput->readVInt64());
}
PostingDecoderPtr PosPostingWriter::createDecoder() const
{
InMemPosPostingDecoder* pDecoder = new InMemPosPostingDecoder();
PostingDecoderPtr pDecoderPtr(pDecoder);
DocSkipListReaderPtr pDocSkipReader;
if (m_streammer.m_pDocSkipListWriter)
{
DocSkipListReader* p = dynamic_cast<DocSkipListReader*>(
m_streammer.m_pDocSkipListWriter->createReader());
pDocSkipReader.assign(p);
}
PosSkipListReaderPtr pPosSkipReader;
if (m_streammer.m_pPosSkipListWriter)
{
PosSkipListReader* p = dynamic_cast<PosSkipListReader*>(
m_streammer.m_pPosSkipListWriter->createReader());
pPosSkipReader.assign(p);
}
InputStreamPtr pDocInStreamPtr;
InputStreamPtr pPosInStreamPtr;
if (m_streammer.m_pDocListWriter)
{
ByteSliceInputStream* pDocInStream = new ByteSliceInputStream(
m_streammer.m_pDocListWriter->getHeadSlice(), false);
pDocInStreamPtr.assign(pDocInStream);
}
if (m_streammer.m_pPosListWriter)
{
ByteSliceInputStream* pPosInStream = new ByteSliceInputStream(
m_streammer.m_pPosListWriter->getHeadSlice(), false);
pPosInStreamPtr.assign(pPosInStream);
}
pDecoder->init(m_termMeta, pDocInStreamPtr, pPosInStreamPtr,
pDocSkipReader, pPosSkipReader, this->m_lastDocId,
this->m_lastPosInDoc);
return pDecoderPtr;
}
void BlockFileInputStreamTestCase::testSeekInBuffer()
{
OutputStreamPtr pOutput = m_pBlockFileSystem->createFile("testfile1");
size_t dataSize = 10000;
stringstream content;
for (size_t i = 0; i < dataSize; ++i)
{
content << (i % 10);
}
pOutput->write((const void*)content.str().c_str(), content.str().size());
pOutput.reset();
InputStreamPtr pInput = m_pBlockFileSystem->openFile("testfile1");
CPPUNIT_ASSERT(pInput != NULL);
CPPUNIT_ASSERT_EQUAL(dataSize, (size_t)pInput->getSize());
off_t off = 100;
string readCont;
readCont.resize((size_t)off);
pInput->read((void*)readCont.c_str(), (size_t)off);
pInput->seek(off);
size_t readSize = 1000;
readCont.resize(readSize);
pInput->read((void*)readCont.c_str(), readSize);
string expStr = content.str().substr((size_t)off, readSize);
CPPUNIT_ASSERT_EQUAL(expStr, readCont);
pInput.reset();
}
ImageData PNGImageCodec::Load(InputStreamPtr& file, const ImageParams& params,
ImageCodecMetaInfo& metaInfo) {
InputStream* readr = file.GetPtr();
// check if png_ptr
if (!CheckIfPng(readr)) {
Warn(String("Image file format is not png: ") + params.name);
}
png_structp png_ptr;
png_infop info_ptr;
png_uint_32 width, height;
int bit_depth, color_type;
bool alpha;
Color32 keycolor;
int keycolor_index = -1;
bool hascolorkey = false;
ImageType imagetype;
png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, 0, 0, 0);
if (png_ptr == NULL) {
Error("Could not create read struct");
NEX_THROW_GracefulError(EXCEPT_INVALID_CALL);
}
info_ptr = png_create_info_struct(png_ptr);
if (info_ptr == NULL) {
png_destroy_read_struct(&png_ptr, NULL, NULL);
Error("Could not create info struct");
NEX_THROW_GracefulError(EXCEPT_INVALID_CALL);
}
png_set_error_fn(png_ptr, (void *) (readr), PngWarn, PngWarn);
png_set_read_fn(png_ptr, (void *) (readr), PngReadFile);
png_set_sig_bytes(png_ptr, PNG_BYTES_TO_CHECK);
png_read_info(png_ptr, info_ptr);
png_get_IHDR(png_ptr, info_ptr, &width, &height, &bit_depth, &color_type, 0,
NULL, NULL);
// todo Unstrip
png_set_strip_16(png_ptr);
if (bit_depth != 8 && bit_depth != 16) {
Error("Unsupported bit depth");
NEX_THROW_GracefulError(EXCEPT_INVALID_CALL);
}
switch (color_type) {
case PNG_COLOR_TYPE_GRAY:
case PNG_COLOR_TYPE_GRAY_ALPHA:
if (color_type & PNG_COLOR_MASK_ALPHA)
imagetype = GrayAlpha;
else {
imagetype = Palleted;
png_set_strip_alpha(png_ptr);
if (png_get_valid(png_ptr, info_ptr, PNG_INFO_tRNS)) {
png_color_16p trans_values;
png_get_tRNS(png_ptr, info_ptr, 0, 0, &trans_values);
hascolorkey = true;
keycolor.red = uint8(trans_values->gray) & 0xff;
keycolor.green = uint8(trans_values->gray) & 0xff;
keycolor.blue = uint8(trans_values->gray) & 0xff;
keycolor.alpha = uint8(trans_values->gray) & 0xff;
}
}
break;
case PNG_COLOR_TYPE_PALETTE:
imagetype = Palleted;
alpha = true;
if (png_get_valid(png_ptr, info_ptr, PNG_INFO_tRNS)) {
// tRNS chunk. Every palette entry gets its own alpha value.
png_bytep trans;
int num_trans;
png_get_tRNS(png_ptr, info_ptr, &trans, &num_trans, 0);
// see if there is a single entry w/ alpha==0 and all other 255.
// if yes use keycolor transparency.
bool only_binary_trans = true;
for (int32 i = 0; (i < num_trans) && only_binary_trans; i++) {
if (trans[i] != 0xff) {
only_binary_trans = only_binary_trans && (trans[i] == 0)
&& (keycolor_index == -1);
keycolor_index = i;
}
}
if (!only_binary_trans) {
keycolor_index = -1;
png_set_palette_to_rgb(png_ptr);
png_set_tRNS_to_alpha(png_ptr);
imagetype = RGBImage;
} else
alpha = false;
} else
alpha = false;
break;
case PNG_COLOR_TYPE_RGBA:
alpha = true;
case PNG_COLOR_TYPE_RGB:
imagetype = RGBImage;
if (!(color_type & PNG_COLOR_MASK_ALPHA)) {
alpha = false;
if (png_get_valid(png_ptr, info_ptr, PNG_INFO_tRNS)) {
png_color_16p trans_values;
png_get_tRNS(png_ptr, info_ptr, 0, 0, &trans_values);
hascolorkey = true;
keycolor.red = uint8((trans_values->red) & 0xff);
keycolor.green = uint8((trans_values->green) & 0xff);
keycolor.blue = uint8((trans_values->blue) & 0xff);
}
}
break;
}
// setup gamma
//@ TODO: add code to retrieve the gamma here
double screen_gamma = 0; //X_SharedPtr(iSystem)->getUserGamma();
// and comment this
screen_gamma = 2.2;
int intent;
if (png_get_sRGB(png_ptr, info_ptr, &intent))
png_set_gamma(png_ptr, screen_gamma, 0.45455);
else {
double image_gamma;
if (png_get_gAMA(png_ptr, info_ptr, &image_gamma))
png_set_gamma(png_ptr, screen_gamma, image_gamma);
else
png_set_gamma(png_ptr, screen_gamma, 0.45455);
}
/* todo Unstrip this */
if (bit_depth > 8) {
// tell libpng to strip 16 bit/color files down to 8 bits/color
png_set_strip_16(png_ptr);
bit_depth = 8;
} else if (bit_depth < 8)
// Expand pictures with less than 8bpp to 8bpp
png_set_packing(png_ptr);
// update
png_read_update_info(png_ptr, info_ptr);
png_uint_32 bytes_per_row = (png_uint_32)png_get_rowbytes(png_ptr, info_ptr);
//png_read_image
PixelFormat fmt;
uint32 bpp = 1;
// we dont support anything else
// other than RGBImage right now
// so!
if (imagetype == RGBImage) {
fmt = PixelFormat::RGBA8;
bpp = 4;
} else if (imagetype == GrayAlpha && bytes_per_row == 1)
fmt = PixelFormat::R8;
else { // pallete
// cant handle this
Warn(String("PixelFormat not supported.") + params.name);
NEX_THROW_GracefulError(EXCEPT_INVALID_CALL);
}
int numPass = png_set_interlace_handling(png_ptr);
ImageData img;
uint8* imgDest = (uint8*) NEX_ALLOC(bpp * height * width, MEMCAT_GENERAL);
img.data = imgDest;
img.format = fmt;
metaInfo.metaInfoInited = true;
metaInfo.mipLevelsToRead = 0;
metaInfo.metaInfo.maxDepth = img.depth = 1;
metaInfo.metaInfo.maxHeight = img.height = (uint16) height;
metaInfo.metaInfo.maxWidth = img.width = (uint16) width;
metaInfo.metaInfo.maxMipMapCount = img.numMipMaps = 1;
img.numFaces = 1;
uint8* imgRows = 0;
if (numPass > 1) {
imgRows = (uint8*) NEX_ALLOC(bytes_per_row * height, MEMCAT_GENERAL);
} else
imgRows = (uint8*) NEX_ALLOC(bytes_per_row, MEMCAT_GENERAL);
uint32 srcBpp = bytes_per_row / width;
for (uint32 i = 0; (int32) i < numPass; ++i) {
for (uint32 y = 0; y < height; y++) {
uint8* rowPtr = numPass > 1 ? imgRows + bytes_per_row * y : imgRows;
png_read_row(png_ptr, rowPtr, NULL);
// write only once
if (i == numPass - 1) {
for (uint32 x = 0; x < width; x++) {
// bgra
switch (fmt) {
case PixelFormat::RGBA8:
imgDest[0] = rowPtr[0];
imgDest[1] = rowPtr[1];
imgDest[2] = rowPtr[2];
if (alpha)
imgDest[3] = rowPtr[3];
else
imgDest[3] = 0xff;
break;
case PixelFormat::R8:
imgDest[0] = rowPtr[0];
break;
}
imgDest += bpp;
rowPtr += srcBpp;
}
}
}
}
png_read_end(png_ptr, (png_infop) 0);
png_destroy_read_struct(&png_ptr, NULL, NULL);
NEX_FREE(imgRows, MEMCAT_GENERAL);
if (hascolorkey) {
imgDest = (uint8*) img.data;
for (uint32 y = 0; y < height; y++) {
for (uint32 x = 0; x < width; x++) {
switch (fmt) {
case PixelFormat::RGBA8:
if (imgDest[0] == keycolor.red
&& imgDest[1] == keycolor.green
&& imgDest[2] == keycolor.blue)
imgDest[3] = 0;
break;
case PixelFormat::R8:
if (imgDest[0] == keycolor.alpha)
imgDest[0] = 0;
break;
}
imgDest += bpp;
}
}
}
return img;
}
void
Ice::ice_readObject(const InputStreamPtr& in, ObjectPtr& p)
{
in->read(p);
}
