void index(const tstring& sDir)
{
IndexWriterPtr pIndexWriter = m_pIndex->acquireWriter();
DirectoryIterator di(sDir, false);
while(di.hasNext())
{
const File& f = di.next();
if(f.isFile())
{
BinaryFile bf;
bf.open(f.getPath().c_str(), BinaryFile::READ);
if(bf.isFileOpen())
{
size_t nRead = (size_t)bf.getLength();
if (nRead > 0)
{
DocumentPtr pDoc = new Document(pIndexWriter->getDocSchema());
pDoc->addField(0, f.getPath().c_str());
char* buf = new char[nRead + 1];
bf.read(buf, nRead);
buf[nRead] = 0;
pDoc->addField(1, buf, nRead, false);
delete[] buf;
pIndexWriter->addDocument(pDoc);
}
}
}
}
docPool.commit();
pIndexWriter->close();
}
void XMLDocumentWrapper::parseFile(const std::string& sFile)
{
BinaryFile bf;
bf.open(sFile, BinaryFile::READ);
size_t len = (size_t)bf.getLength();
char* buf = m_doc.allocate_string(NULL, len + 1);
buf[len] = 0;
try
{
bf.read(buf, len);
}
catch (const FileIOException& fe)
{
bf.close();
clear();
FIRTEX_RETHROW(fe);
}
bf.close();
try
{
m_doc.parse<rapidxml::parse_full>(buf);
}
catch(rapidxml::parse_error& e)
{
clear();
FIRTEX_THROW(BadXmlFormatException, "Bad xml format: [%s]", e.what());
}
}
ExpressionValue expFuncRead(const std::vector<ExpressionValue>& parameters)
{
ExpressionValue result;
T buffer;
if (parameters[0].isString() == false || (parameters.size() >= 2 && parameters[1].isInt() == false))
{
Logger::queueError(Logger::Error,L"Invalid parameter");
return result;
}
std::wstring fileName = getFullPathName(parameters[0].strValue);
u64 pos = parameters.size() >= 2 ? parameters[1].intValue : 0;
BinaryFile file;
if (file.open(fileName,BinaryFile::Read) == false)
{
Logger::queueError(Logger::Error,L"Could not open %s",fileName);
return result;
}
file.setPos((long)pos);
if (file.read(&buffer,sizeof(T)) == sizeof(T))
{
result.type = ExpressionValueType::Integer;
result.intValue = (u64) buffer;
}
return result;
}
ExpressionValue expFuncRead(const std::wstring& funcName, const std::vector<ExpressionValue>& parameters)
{
const std::wstring* fileName;
u64 pos;
GET_PARAM(parameters,0,fileName);
GET_OPTIONAL_PARAM(parameters,1,pos,0);
std::wstring fullName = getFullPathName(*fileName);
BinaryFile file;
if (file.open(fullName,BinaryFile::Read) == false)
{
Logger::queueError(Logger::Error,L"Could not open %s",fileName);
return ExpressionValue();
}
file.setPos((long)pos);
T buffer;
if (file.read(&buffer,sizeof(T)) != sizeof(T))
return ExpressionValue();
return ExpressionValue((u64) buffer);
}
void BinaryFileTestCase::testRead()
{
tstring str = TestHelper::getTestDataPath(_T("BinaryFileTestWriteFile"), false);
BinaryFile bf;
bf.open(str.c_str(),BinaryFile::READ);
CPPUNIT_ASSERT( bf.isFileOpen() );
int32_t nData;
for(size_t i = 0; i < 10000;i += 1000)
{
bf.seek(i);
bf.read(&nData,sizeof(int32_t));
CPPUNIT_ASSERT_EQUAL(nData,(int32_t)i);
}
bf.close();
}
void BinaryFileTestCase::testLargeFileSeekAndRead()
{
#ifdef TEST_LARGE_FILE
tstring str = TestHelper::getTestDataPath(_T("BinaryFileTestLargeFile"), false);
BinaryFile bf;
bf.open(str.c_str(), BinaryFile::READ);
CPPUNIT_ASSERT( bf.isFileOpen() );
int64_t nPos = 1000000000;//1 GB
nPos <<= 4;//16 GB
int64_t nData = 123456789;
bf.seek(nPos);
bf.read(&nData,sizeof(int64_t));
CPPUNIT_ASSERT_EQUAL(nData,(int64_t)123456789);
#endif
}
void HTMLParser::parse(const tstring& sHtmlFile)
{
BinaryFile bf;
try
{
bf.open(sHtmlFile, BinaryFile::READ);
m_nFileSize = (size_t)bf.getLength();
if(m_nFileSize > MAX_FILESIZE - 1)
{
m_nFileSize = MAX_FILESIZE - 1;
}
if(!m_pReadBuffer)
{
m_nReadBufferSize = DEFAULT_READBUFFER_SIZE;
if(m_nReadBufferSize < m_nFileSize + 1)
m_nReadBufferSize = m_nFileSize + 1;
m_pReadBuffer = new char[m_nReadBufferSize];
}
else if(m_nFileSize + 1 > m_nReadBufferSize)
{
m_nReadBufferSize = m_nFileSize + 1;
delete[] m_pReadBuffer;
m_pReadBuffer = new char[m_nReadBufferSize];
}
size_t nRet = bf.read(m_pReadBuffer, m_nFileSize);
if(nRet != m_nFileSize)
{
FX_LOG(WARN, "Read file [%s] error", sHtmlFile.c_str());
bf.close();
return;
}
bf.close();
parse(m_pReadBuffer, m_nFileSize);
}
catch(const FirteXException& e)
{
FX_LOG(ERROR, "Parse file: [%s] FAILED. Error message: [%s]",
sHtmlFile.c_str(), e.what().c_str());
}
}