void PlaybackController::loadIndex()
{
if(m_reader->isCompressed()) {
emit indexLoadError(tr("Cannot index compressed recordings."), false);
return;
}
QFileInfo indexfile(indexFileName());
if(!indexfile.exists()) {
emit indexLoadError(tr("Index not yet generated"), true);
return;
}
m_indexloader = new IndexLoader(m_reader->filename(), indexFileName());
if(!m_indexloader->open()) {
delete m_indexloader;
m_indexloader = nullptr;
emit indexLoadError(tr("Error loading index!"), true);
return;
}
emit markersChanged();
emit indexLoaded();
}
virtual void init()
{
HashDistributeMasterBase::init();
// JCSMORE should common up some with indexread
IHThorKeyedDistributeArg *helper = (IHThorKeyedDistributeArg *)queryHelper();
StringBuffer scoped;
OwnedRoxieString indexFileName(helper->getIndexFileName());
queryThorFileManager().addScope(container.queryJob(), indexFileName, scoped);
file.setown(queryThorFileManager().lookup(container.queryJob(), indexFileName));
if (!file)
throw MakeActivityException(this, 0, "KeyedDistribute: Failed to find key: %s", scoped.str());
if (0 == file->numParts())
throw MakeActivityException(this, 0, "KeyedDistribute: Can't distribute based on an empty key: %s", scoped.str());
checkFormatCrc(this, file, helper->getFormatCrc(), true);
Owned<IFileDescriptor> fileDesc = file->getFileDescriptor();
Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
if (!tlkDesc->queryProperties().hasProp("@kind") || 0 != stricmp("topLevelKey", tlkDesc->queryProperties().queryProp("@kind")))
throw MakeActivityException(this, 0, "Cannot distribute using a non-distributed key: '%s'", scoped.str());
unsigned location;
OwnedIFile iFile;
StringBuffer filePath;
if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", file->queryLogicalName());
OwnedIFileIO iFileIO = iFile->open(IFOread);
assertex(iFileIO);
tlkMb.append(iFileIO->size());
::read(iFileIO, 0, (size32_t)iFileIO->size(), tlkMb);
queryThorFileManager().noteFileRead(container.queryJob(), file);
}
/**
* @brief (Re)build the index file
*/
void PlaybackController::buildIndex()
{
if(!m_indexbuilder.isNull()) {
qWarning("Index builder already running!");
return;
}
QThread *thread = new QThread;
m_indexbuilder = new IndexBuilder(m_reader->filename(), indexFileName());
m_indexbuilder->moveToThread(thread);
const qreal filesize = m_reader->filesize();
connect(m_indexbuilder.data(), &IndexBuilder::progress, [this, filesize](int progress) {
m_indexBuildProgress = progress / filesize;
emit indexBuildProgressed(m_indexBuildProgress);
});
connect(m_indexbuilder.data(), &IndexBuilder::done, [this](bool ok, const QString &error) {
if(!ok) {
m_indexBuildProgress = 0;
emit indexBuildProgressed(0);
emit indexLoadError(error, true);
} else {
loadIndex();
}
});
connect(thread, &QThread::started, m_indexbuilder.data(), &IndexBuilder::run);
connect(m_indexbuilder.data(), &IndexBuilder::done, thread, &QThread::quit);
connect(m_indexbuilder.data(), &IndexBuilder::done, m_indexbuilder.data(), &QThread::deleteLater);
connect(thread, &QThread::finished, thread, &QThread::deleteLater);
thread->start();
}
/**
* @brief (Re)build the index file
*/
void PlaybackController::buildIndex()
{
if(!m_indexbuilder.isNull()) {
qWarning("Index builder already running!");
return;
}
m_indexbuilder = new IndexBuilder(m_reader->filename(), indexFileName());
const qreal filesize = m_reader->filesize();
connect(m_indexbuilder, &IndexBuilder::progress, [this, filesize](int progress) {
m_indexBuildProgress = progress / filesize;
emit indexBuildProgressed(m_indexBuildProgress);
});
connect(m_indexbuilder, &IndexBuilder::done, [this](bool ok, const QString &error) {
if(!ok) {
m_indexBuildProgress = 0;
emit indexBuildProgressed(0);
emit indexLoadError(error, true);
} else {
loadIndex();
}
});
QThreadPool::globalInstance()->start(m_indexbuilder);
}
IndexFile::IndexFile(LogFile &logFile)
{
std::string indexFileName(logFile.getFileBaseName() + ".id2");
if(!loadIndexFile(indexFileName))
{
if(!createIndexFile(indexFileName, logFile))
{
throw std::runtime_error("Error building Index");
}
if(!loadIndexFile(indexFileName))
throw std::runtime_error("Internal Error, created index is corrupted");
}
}
// read from a file
int ibis::bak::read(const char* f) {
int ierr = -1;
try {
std::string fnm;
indexFileName(fnm, f);
if (ibis::index::isIndex(fnm.c_str(), ibis::index::BAK)) {
ierr = ibis::bin::read(f);
}
}
catch (...) {
clear();
throw;
}
return ierr;
} // ibis::bak::read
/// Read the index contained in the file f. This function always reads all
/// bitvectors.
int ibis::skive::read(const char* f) {
std::string fnm;
indexFileName(fnm, f);
int fdes = UnixOpen(fnm.c_str(), OPEN_READONLY);
if (fdes < 0) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- skive[" << col->partition()->name() << '.'
<< col->name() << "]::read failed to open " << fnm;
return -1; // can not do anything else
}
char header[8];
IBIS_BLOCK_GUARD(UnixClose, fdes);
#if defined(_WIN32) && defined(_MSC_VER)
(void)_setmode(fdes, _O_BINARY);
#endif
int ierr = UnixRead(fdes, static_cast<void*>(header), 8);
if (ierr != 8) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- skive[" << col->partition()->name() << '.'
<< col->name() << "]::read failed to read 8 bytes from "
<< fnm;
return -2;
}
if (!(header[0] == '#' && header[1] == 'I' &&
header[2] == 'B' && header[3] == 'I' &&
header[4] == 'S' &&
(header[5] == static_cast<char>(ibis::index::SKIVE) ||
header[5] == static_cast<char>(ibis::index::SLICE)) &&
(header[6] == 8 || header[6] == 4) &&
header[7] == static_cast<char>(0))) {
if (ibis::gVerbose > 0) {
ibis::util::logger lg;
lg() << "Warning -- skive[" << col->partition()->name() << '.'
<< col->name() << "]::read the header from " << fnm << " (";
printHeader(lg(), header);
lg() << ") does not contain the expected values";
}
return -3;
}
uint32_t dim[3]; // nrows, nobs, card
size_t begin, end;
clear(); // clear the current content
ierr = UnixRead(fdes, static_cast<void*>(dim), 3*sizeof(uint32_t));
nrows = dim[0];
// read vals
begin = 8*((3*sizeof(uint32_t) + 15) / 8);
end = begin + dim[2] * sizeof(double);
{
array_t<double> dbl(fnm.c_str(), fdes, begin, end);
vals.swap(dbl);
}
// read the offsets
begin = end;
end += header[6] * (dim[1] + 1);
ierr = initOffsets(fdes, header[6], begin, dim[1]);
if (ierr < 0)
return ierr;
// cnts
begin = end;
end += sizeof(uint32_t) * dim[2];
{
array_t<uint32_t> szt(fnm.c_str(), fdes, begin, end);
cnts.swap(szt);
}
ibis::fileManager::instance().recordPages(0, end);
initBitmaps(fdes);
activate();
return 0;
} // ibis::skive::read
// the argument is the name of the directory or the file name
int ibis::skive::write(const char* dt) const {
if (vals.empty()) return -1;
std::string fnm;
indexFileName(fnm, dt);
if (fnm.empty()) {
return 0;
}
else if (0 != str && 0 != str->filename() &&
0 == fnm.compare(str->filename())) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- skive::write can not overwrite the index file \""
<< fnm << "\" while it is used as a read-only file map";
return 0;
}
else if (fname != 0 && *fname != 0 && 0 == fnm.compare(fname)) {
activate(); // read everything into memory
fname = 0; // break the link with the named file
}
ibis::fileManager::instance().flushFile(fnm.c_str());
int fdes = UnixOpen(fnm.c_str(), OPEN_WRITENEW, OPEN_FILEMODE);
if (fdes < 0) {
ibis::fileManager::instance().flushFile(fnm.c_str());
fdes = UnixOpen(fnm.c_str(), OPEN_WRITENEW, OPEN_FILEMODE);
if (fdes < 0) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- skive[" << col->partition()->name() << '.'
<< col->name() << "]::write failed to open \"" << fnm
<< "\" for writing";
return -2;
}
}
IBIS_BLOCK_GUARD(UnixClose, fdes);
#if defined(_WIN32) && defined(_MSC_VER)
(void)_setmode(fdes, _O_BINARY);
#endif
#ifdef FASTBIT_USE_LONG_OFFSETS
const bool useoffset64 = true;
#else
const bool useoffset64 = (getSerialSize()+8 > 0x80000000UL);
#endif
char header[] = "#IBIS\11\0\0";
header[5] = (char)ibis::index::SKIVE;
header[6] = (char)(useoffset64 ? 8 : 4);
int ierr = UnixWrite(fdes, header, 8);
if (ierr < 8) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- skive[" << col->partition()->name() << "."
<< col->name() << "]::write(" << fnm
<< ") failed to write the 8-byte header, ierr = " << ierr;
return -3;
}
if (useoffset64)
ierr = write64(fdes);
else
ierr = write32(fdes);
if (ierr >= 0) {
#if defined(FASTBIT_SYNC_WRITE)
#if _POSIX_FSYNC+0 > 0
(void) UnixFlush(fdes); // write to disk
#elif defined(_WIN32) && defined(_MSC_VER)
(void) _commit(fdes);
#endif
#endif
LOGGER(ibis::gVerbose > 3)
<< "skive[" << col->partition()->name() << "." << col->name()
<< "]::write wrote " << bits.size() << " bitmap"
<< (bits.size()>1?"s":"") << " to file " << fnm;
}
return ierr;
} // ibis::skive::write
virtual void init()
{
CMasterActivity::init();
OwnedRoxieString indexFileName(helper->getIndexFileName());
Owned<IDistributedFile> dataFile;
Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true);
unsigned keyReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJKRR", 0);
if (!keyReadWidth || keyReadWidth>container.queryJob().querySlaves())
keyReadWidth = container.queryJob().querySlaves();
initMb.clear();
initMb.append(indexFileName.get());
if (helper->diskAccessRequired())
numTags += 2;
initMb.append(numTags);
unsigned t=0;
for (; t<numTags; t++)
{
tags[t] = container.queryJob().allocateMPTag();
initMb.append(tags[t]);
}
bool keyHasTlk = false;
if (indexFile)
{
unsigned numParts = 0;
localKey = indexFile->queryAttributes().getPropBool("@local");
if (container.queryLocalData() && !localKey)
throw MakeActivityException(this, 0, "Keyed Join cannot be LOCAL unless supplied index is local");
checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), true);
Owned<IFileDescriptor> indexFileDesc = indexFile->getFileDescriptor();
IDistributedSuperFile *superIndex = indexFile->querySuperFile();
unsigned superIndexWidth = 0;
unsigned numSuperIndexSubs = 0;
if (superIndex)
{
numSuperIndexSubs = superIndex->numSubFiles(true);
bool first=true;
// consistency check
Owned<IDistributedFileIterator> iter = superIndex->getSubFileIterator(true);
ForEach(*iter)
{
IDistributedFile &f = iter->query();
unsigned np = f.numParts()-1;
IDistributedFilePart &part = f.queryPart(np);
const char *kind = part.queryAttributes().queryProp("@kind");
bool hasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind); // if last part not tlk, then deemed local (might be singlePartKey)
if (first)
{
first = false;
keyHasTlk = hasTlk;
superIndexWidth = f.numParts();
if (keyHasTlk)
--superIndexWidth;
}
else
{
if (hasTlk != keyHasTlk)
throw MakeActivityException(this, 0, "Local/Single part keys cannot be mixed with distributed(tlk) keys in keyedjoin");
if (keyHasTlk && superIndexWidth != f.numParts()-1)
throw MakeActivityException(this, 0, "Super sub keys of different width cannot be mixed with distributed(tlk) keys in keyedjoin");
}
}
if (keyHasTlk)
numParts = superIndexWidth * numSuperIndexSubs;
else
numParts = superIndex->numParts();
}
else
{
numParts = indexFile->numParts();
if (numParts)
{
const char *kind = indexFile->queryPart(indexFile->numParts()-1).queryAttributes().queryProp("@kind");
keyHasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind);
if (keyHasTlk)
--numParts;
}
}
if (numParts)
{
initMb.append(numParts);
initMb.append(superIndexWidth); // 0 if not superIndex
initMb.append((superIndex && superIndex->isInterleaved()) ? numSuperIndexSubs : 0);
unsigned p=0;
UnsignedArray parts;
for (; p<numParts; p++)
parts.append(p);
indexFileDesc->serializeParts(initMb, parts);
if (localKey)
keyHasTlk = false; // not used
initMb.append(keyHasTlk);
if (keyHasTlk)
{
if (numSuperIndexSubs)
initMb.append(numSuperIndexSubs);
else
initMb.append((unsigned)1);
Owned<IDistributedFileIterator> iter;
IDistributedFile *f;
if (superIndex)
{
iter.setown(superIndex->getSubFileIterator(true));
f = &iter->query();
}
else
f = indexFile;
loop
{
unsigned location;
OwnedIFile iFile;
StringBuffer filePath;
Owned<IFileDescriptor> fileDesc = f->getFileDescriptor();
Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", f->queryLogicalName());
OwnedIFileIO iFileIO = iFile->open(IFOread);
assertex(iFileIO);
size32_t tlkSz = (size32_t)iFileIO->size();
initMb.append(tlkSz);
::read(iFileIO, 0, tlkSz, initMb);
if (!iter || !iter->next())
break;
f = &iter->query();
}
}
if (helper->diskAccessRequired())
{
OwnedRoxieString fetchFilename(helper->getFileName());
if (fetchFilename)
{
dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true));
if (dataFile)
{
if (superIndex)
throw MakeActivityException(this, 0, "Superkeys and full keyed joins are not supported");
Owned<IFileDescriptor> dataFileDesc = getConfiguredFileDescriptor(*dataFile);
void *ekey;
size32_t ekeylen;
helper->getFileEncryptKey(ekeylen,ekey);
bool encrypted = dataFileDesc->queryProperties().getPropBool("@encrypted");
if (0 != ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
if (!encrypted)
{
Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", helper->getFileName());
container.queryJob().fireException(e);
}
}
else if (encrypted)
throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fetchFilename.get());
unsigned dataReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJDRR", 0);
if (!dataReadWidth || dataReadWidth>container.queryJob().querySlaves())
dataReadWidth = container.queryJob().querySlaves();
Owned<IGroup> grp = container.queryJob().querySlaveGroup().subset((unsigned)0, dataReadWidth);
dataFileMapping.setown(getFileSlaveMaps(dataFile->queryLogicalName(), *dataFileDesc, container.queryJob().queryUserDescriptor(), *grp, false, false, NULL));
dataFileMapping->serializeFileOffsetMap(offsetMapMb.clear());
}
else
indexFile.clear();
}
}
}
else
int jb2()
{
JET_ERR err;
eseInstance instance;
if (instance.init(0) != JET_errSuccess)
{
return instance.getLastError();
}
eseSession session;
if (session.init(instance.id()) != JET_errSuccess)
{
return session.getLastError();;
}
eseDatabase db(session.id());
err = db.open("test2.edb");
if (err == JET_errDatabaseDuplicate)
{
err = db.create("test2.edb");
}
if (err != JET_errSuccess)
{
return err;
}
// eseColumns will be destroyed by table's destructor
eseColumn* col0 = new eseColumn("PK", JET_coltypLong, JET_bitColumnAutoincrement);
eseColumn* col1 = new eseColumn("filename", JET_coltypText, 0, 1024); // include path
eseColumn* col2 = new eseColumn("size", JET_coltypLongLong);
eseColumn* col3 = new eseColumn("changedDate", JET_coltypDateTime);
eseColumn* col4 = new eseColumn("status", JET_coltypLong);
eseIndex indexKey("PK_index", IndexKey("+PK"), JET_bitIndexPrimary);
eseIndex indexFileName("filename_index", IndexKey("+filename"), JET_bitIndexUnique);
eseTable table(db.id(), session.id());
switch (err = table.open("FileSystemLog"))
{
case JET_errSuccess:
break;
case JET_errObjectNotFound: // Table does not exists, open it
err = table.create("FileSystemLog", col0, col1, indexKey, indexFileName);
table.create_column(col2);
table.create_column(col3);
table.create_column(col4);
if (err != JET_errSuccess) return err;
break;
default:
return err;
}
// Inserting Values
wchar_t* stringsTolnsert[] = { L"FirstInsertedRow2", L"InSecondInsertedRow2", L"ThirdInsertedRow2", L"FourinsertedRow2", L"Fourinse" };
for (size_t n = 0; n != sizeof(stringsTolnsert) / sizeof(stringsTolnsert[0]); ++n)
{
err = table.insert(1, stringsTolnsert[n]); // insert to the second column (numbering starts from zero)
}
// List content of the table
err = table.setCurrentIndex(indexFileName);
if (err != JET_errSuccess) return err;
for (err = table.setCursor(JET_MoveFirst); JET_errSuccess == err; err = table.setCursor(JET_MoveNext))
{
int nPK = 0;
err = table.get_column(0, &nPK);
if (JET_errSuccess != err) break;
std::wcout << nPK;
wchar_t buffer[1024] = { 0 };
err = table.get_column(1, buffer, 1024);
std::wcout << L'\t' << buffer << std::endl;
}
if (JET_errNoCurrentRecord != err) return err;
std::wcout << std::endl;
return 0;
} /// jb2
