void PlaybackController::loadIndex() { if(m_reader->isCompressed()) { emit indexLoadError(tr("Cannot index compressed recordings."), false); return; } QFileInfo indexfile(indexFileName()); if(!indexfile.exists()) { emit indexLoadError(tr("Index not yet generated"), true); return; } m_indexloader = new IndexLoader(m_reader->filename(), indexFileName()); if(!m_indexloader->open()) { delete m_indexloader; m_indexloader = nullptr; emit indexLoadError(tr("Error loading index!"), true); return; } emit markersChanged(); emit indexLoaded(); }
virtual void init() { HashDistributeMasterBase::init(); // JCSMORE should common up some with indexread IHThorKeyedDistributeArg *helper = (IHThorKeyedDistributeArg *)queryHelper(); StringBuffer scoped; OwnedRoxieString indexFileName(helper->getIndexFileName()); queryThorFileManager().addScope(container.queryJob(), indexFileName, scoped); file.setown(queryThorFileManager().lookup(container.queryJob(), indexFileName)); if (!file) throw MakeActivityException(this, 0, "KeyedDistribute: Failed to find key: %s", scoped.str()); if (0 == file->numParts()) throw MakeActivityException(this, 0, "KeyedDistribute: Can't distribute based on an empty key: %s", scoped.str()); checkFormatCrc(this, file, helper->getFormatCrc(), true); Owned<IFileDescriptor> fileDesc = file->getFileDescriptor(); Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1); if (!tlkDesc->queryProperties().hasProp("@kind") || 0 != stricmp("topLevelKey", tlkDesc->queryProperties().queryProp("@kind"))) throw MakeActivityException(this, 0, "Cannot distribute using a non-distributed key: '%s'", scoped.str()); unsigned location; OwnedIFile iFile; StringBuffer filePath; if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath)) throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", file->queryLogicalName()); OwnedIFileIO iFileIO = iFile->open(IFOread); assertex(iFileIO); tlkMb.append(iFileIO->size()); ::read(iFileIO, 0, (size32_t)iFileIO->size(), tlkMb); queryThorFileManager().noteFileRead(container.queryJob(), file); }
/** * @brief (Re)build the index file */ void PlaybackController::buildIndex() { if(!m_indexbuilder.isNull()) { qWarning("Index builder already running!"); return; } QThread *thread = new QThread; m_indexbuilder = new IndexBuilder(m_reader->filename(), indexFileName()); m_indexbuilder->moveToThread(thread); const qreal filesize = m_reader->filesize(); connect(m_indexbuilder.data(), &IndexBuilder::progress, [this, filesize](int progress) { m_indexBuildProgress = progress / filesize; emit indexBuildProgressed(m_indexBuildProgress); }); connect(m_indexbuilder.data(), &IndexBuilder::done, [this](bool ok, const QString &error) { if(!ok) { m_indexBuildProgress = 0; emit indexBuildProgressed(0); emit indexLoadError(error, true); } else { loadIndex(); } }); connect(thread, &QThread::started, m_indexbuilder.data(), &IndexBuilder::run); connect(m_indexbuilder.data(), &IndexBuilder::done, thread, &QThread::quit); connect(m_indexbuilder.data(), &IndexBuilder::done, m_indexbuilder.data(), &QThread::deleteLater); connect(thread, &QThread::finished, thread, &QThread::deleteLater); thread->start(); }
/** * @brief (Re)build the index file */ void PlaybackController::buildIndex() { if(!m_indexbuilder.isNull()) { qWarning("Index builder already running!"); return; } m_indexbuilder = new IndexBuilder(m_reader->filename(), indexFileName()); const qreal filesize = m_reader->filesize(); connect(m_indexbuilder, &IndexBuilder::progress, [this, filesize](int progress) { m_indexBuildProgress = progress / filesize; emit indexBuildProgressed(m_indexBuildProgress); }); connect(m_indexbuilder, &IndexBuilder::done, [this](bool ok, const QString &error) { if(!ok) { m_indexBuildProgress = 0; emit indexBuildProgressed(0); emit indexLoadError(error, true); } else { loadIndex(); } }); QThreadPool::globalInstance()->start(m_indexbuilder); }
IndexFile::IndexFile(LogFile &logFile) { std::string indexFileName(logFile.getFileBaseName() + ".id2"); if(!loadIndexFile(indexFileName)) { if(!createIndexFile(indexFileName, logFile)) { throw std::runtime_error("Error building Index"); } if(!loadIndexFile(indexFileName)) throw std::runtime_error("Internal Error, created index is corrupted"); } }
// read from a file int ibis::bak::read(const char* f) { int ierr = -1; try { std::string fnm; indexFileName(fnm, f); if (ibis::index::isIndex(fnm.c_str(), ibis::index::BAK)) { ierr = ibis::bin::read(f); } } catch (...) { clear(); throw; } return ierr; } // ibis::bak::read
/// Read the index contained in the file f. This function always reads all /// bitvectors. int ibis::skive::read(const char* f) { std::string fnm; indexFileName(fnm, f); int fdes = UnixOpen(fnm.c_str(), OPEN_READONLY); if (fdes < 0) { LOGGER(ibis::gVerbose > 0) << "Warning -- skive[" << col->partition()->name() << '.' << col->name() << "]::read failed to open " << fnm; return -1; // can not do anything else } char header[8]; IBIS_BLOCK_GUARD(UnixClose, fdes); #if defined(_WIN32) && defined(_MSC_VER) (void)_setmode(fdes, _O_BINARY); #endif int ierr = UnixRead(fdes, static_cast<void*>(header), 8); if (ierr != 8) { LOGGER(ibis::gVerbose > 0) << "Warning -- skive[" << col->partition()->name() << '.' << col->name() << "]::read failed to read 8 bytes from " << fnm; return -2; } if (!(header[0] == '#' && header[1] == 'I' && header[2] == 'B' && header[3] == 'I' && header[4] == 'S' && (header[5] == static_cast<char>(ibis::index::SKIVE) || header[5] == static_cast<char>(ibis::index::SLICE)) && (header[6] == 8 || header[6] == 4) && header[7] == static_cast<char>(0))) { if (ibis::gVerbose > 0) { ibis::util::logger lg; lg() << "Warning -- skive[" << col->partition()->name() << '.' << col->name() << "]::read the header from " << fnm << " ("; printHeader(lg(), header); lg() << ") does not contain the expected values"; } return -3; } uint32_t dim[3]; // nrows, nobs, card size_t begin, end; clear(); // clear the current content ierr = UnixRead(fdes, static_cast<void*>(dim), 3*sizeof(uint32_t)); nrows = dim[0]; // read vals begin = 8*((3*sizeof(uint32_t) + 15) / 8); end = begin + dim[2] * sizeof(double); { array_t<double> dbl(fnm.c_str(), fdes, begin, end); vals.swap(dbl); } // read the offsets begin = end; end += header[6] * (dim[1] + 1); ierr = initOffsets(fdes, header[6], begin, dim[1]); if (ierr < 0) return ierr; // cnts begin = end; end += sizeof(uint32_t) * dim[2]; { array_t<uint32_t> szt(fnm.c_str(), fdes, begin, end); cnts.swap(szt); } ibis::fileManager::instance().recordPages(0, end); initBitmaps(fdes); activate(); return 0; } // ibis::skive::read
// the argument is the name of the directory or the file name int ibis::skive::write(const char* dt) const { if (vals.empty()) return -1; std::string fnm; indexFileName(fnm, dt); if (fnm.empty()) { return 0; } else if (0 != str && 0 != str->filename() && 0 == fnm.compare(str->filename())) { LOGGER(ibis::gVerbose > 0) << "Warning -- skive::write can not overwrite the index file \"" << fnm << "\" while it is used as a read-only file map"; return 0; } else if (fname != 0 && *fname != 0 && 0 == fnm.compare(fname)) { activate(); // read everything into memory fname = 0; // break the link with the named file } ibis::fileManager::instance().flushFile(fnm.c_str()); int fdes = UnixOpen(fnm.c_str(), OPEN_WRITENEW, OPEN_FILEMODE); if (fdes < 0) { ibis::fileManager::instance().flushFile(fnm.c_str()); fdes = UnixOpen(fnm.c_str(), OPEN_WRITENEW, OPEN_FILEMODE); if (fdes < 0) { LOGGER(ibis::gVerbose > 0) << "Warning -- skive[" << col->partition()->name() << '.' << col->name() << "]::write failed to open \"" << fnm << "\" for writing"; return -2; } } IBIS_BLOCK_GUARD(UnixClose, fdes); #if defined(_WIN32) && defined(_MSC_VER) (void)_setmode(fdes, _O_BINARY); #endif #ifdef FASTBIT_USE_LONG_OFFSETS const bool useoffset64 = true; #else const bool useoffset64 = (getSerialSize()+8 > 0x80000000UL); #endif char header[] = "#IBIS\11\0\0"; header[5] = (char)ibis::index::SKIVE; header[6] = (char)(useoffset64 ? 8 : 4); int ierr = UnixWrite(fdes, header, 8); if (ierr < 8) { LOGGER(ibis::gVerbose > 0) << "Warning -- skive[" << col->partition()->name() << "." << col->name() << "]::write(" << fnm << ") failed to write the 8-byte header, ierr = " << ierr; return -3; } if (useoffset64) ierr = write64(fdes); else ierr = write32(fdes); if (ierr >= 0) { #if defined(FASTBIT_SYNC_WRITE) #if _POSIX_FSYNC+0 > 0 (void) UnixFlush(fdes); // write to disk #elif defined(_WIN32) && defined(_MSC_VER) (void) _commit(fdes); #endif #endif LOGGER(ibis::gVerbose > 3) << "skive[" << col->partition()->name() << "." << col->name() << "]::write wrote " << bits.size() << " bitmap" << (bits.size()>1?"s":"") << " to file " << fnm; } return ierr; } // ibis::skive::write
virtual void init() { CMasterActivity::init(); OwnedRoxieString indexFileName(helper->getIndexFileName()); Owned<IDistributedFile> dataFile; Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true); unsigned keyReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJKRR", 0); if (!keyReadWidth || keyReadWidth>container.queryJob().querySlaves()) keyReadWidth = container.queryJob().querySlaves(); initMb.clear(); initMb.append(indexFileName.get()); if (helper->diskAccessRequired()) numTags += 2; initMb.append(numTags); unsigned t=0; for (; t<numTags; t++) { tags[t] = container.queryJob().allocateMPTag(); initMb.append(tags[t]); } bool keyHasTlk = false; if (indexFile) { unsigned numParts = 0; localKey = indexFile->queryAttributes().getPropBool("@local"); if (container.queryLocalData() && !localKey) throw MakeActivityException(this, 0, "Keyed Join cannot be LOCAL unless supplied index is local"); checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), true); Owned<IFileDescriptor> indexFileDesc = indexFile->getFileDescriptor(); IDistributedSuperFile *superIndex = indexFile->querySuperFile(); unsigned superIndexWidth = 0; unsigned numSuperIndexSubs = 0; if (superIndex) { numSuperIndexSubs = superIndex->numSubFiles(true); bool first=true; // consistency check Owned<IDistributedFileIterator> iter = superIndex->getSubFileIterator(true); ForEach(*iter) { IDistributedFile &f = iter->query(); unsigned np = f.numParts()-1; IDistributedFilePart &part = f.queryPart(np); const char *kind = part.queryAttributes().queryProp("@kind"); bool hasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind); // if last part not tlk, then deemed local (might be singlePartKey) if (first) { first = false; keyHasTlk = hasTlk; superIndexWidth = f.numParts(); if (keyHasTlk) --superIndexWidth; } else { if (hasTlk != keyHasTlk) throw MakeActivityException(this, 0, "Local/Single part keys cannot be mixed with distributed(tlk) keys in keyedjoin"); if (keyHasTlk && superIndexWidth != f.numParts()-1) throw MakeActivityException(this, 0, "Super sub keys of different width cannot be mixed with distributed(tlk) keys in keyedjoin"); } } if (keyHasTlk) numParts = superIndexWidth * numSuperIndexSubs; else numParts = superIndex->numParts(); } else { numParts = indexFile->numParts(); if (numParts) { const char *kind = indexFile->queryPart(indexFile->numParts()-1).queryAttributes().queryProp("@kind"); keyHasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind); if (keyHasTlk) --numParts; } } if (numParts) { initMb.append(numParts); initMb.append(superIndexWidth); // 0 if not superIndex initMb.append((superIndex && superIndex->isInterleaved()) ? numSuperIndexSubs : 0); unsigned p=0; UnsignedArray parts; for (; p<numParts; p++) parts.append(p); indexFileDesc->serializeParts(initMb, parts); if (localKey) keyHasTlk = false; // not used initMb.append(keyHasTlk); if (keyHasTlk) { if (numSuperIndexSubs) initMb.append(numSuperIndexSubs); else initMb.append((unsigned)1); Owned<IDistributedFileIterator> iter; IDistributedFile *f; if (superIndex) { iter.setown(superIndex->getSubFileIterator(true)); f = &iter->query(); } else f = indexFile; loop { unsigned location; OwnedIFile iFile; StringBuffer filePath; Owned<IFileDescriptor> fileDesc = f->getFileDescriptor(); Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1); if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath)) throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", f->queryLogicalName()); OwnedIFileIO iFileIO = iFile->open(IFOread); assertex(iFileIO); size32_t tlkSz = (size32_t)iFileIO->size(); initMb.append(tlkSz); ::read(iFileIO, 0, tlkSz, initMb); if (!iter || !iter->next()) break; f = &iter->query(); } } if (helper->diskAccessRequired()) { OwnedRoxieString fetchFilename(helper->getFileName()); if (fetchFilename) { dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true)); if (dataFile) { if (superIndex) throw MakeActivityException(this, 0, "Superkeys and full keyed joins are not supported"); Owned<IFileDescriptor> dataFileDesc = getConfiguredFileDescriptor(*dataFile); void *ekey; size32_t ekeylen; helper->getFileEncryptKey(ekeylen,ekey); bool encrypted = dataFileDesc->queryProperties().getPropBool("@encrypted"); if (0 != ekeylen) { memset(ekey,0,ekeylen); free(ekey); if (!encrypted) { Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", helper->getFileName()); container.queryJob().fireException(e); } } else if (encrypted) throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fetchFilename.get()); unsigned dataReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJDRR", 0); if (!dataReadWidth || dataReadWidth>container.queryJob().querySlaves()) dataReadWidth = container.queryJob().querySlaves(); Owned<IGroup> grp = container.queryJob().querySlaveGroup().subset((unsigned)0, dataReadWidth); dataFileMapping.setown(getFileSlaveMaps(dataFile->queryLogicalName(), *dataFileDesc, container.queryJob().queryUserDescriptor(), *grp, false, false, NULL)); dataFileMapping->serializeFileOffsetMap(offsetMapMb.clear()); } else indexFile.clear(); } } } else
int jb2() { JET_ERR err; eseInstance instance; if (instance.init(0) != JET_errSuccess) { return instance.getLastError(); } eseSession session; if (session.init(instance.id()) != JET_errSuccess) { return session.getLastError();; } eseDatabase db(session.id()); err = db.open("test2.edb"); if (err == JET_errDatabaseDuplicate) { err = db.create("test2.edb"); } if (err != JET_errSuccess) { return err; } // eseColumns will be destroyed by table's destructor eseColumn* col0 = new eseColumn("PK", JET_coltypLong, JET_bitColumnAutoincrement); eseColumn* col1 = new eseColumn("filename", JET_coltypText, 0, 1024); // include path eseColumn* col2 = new eseColumn("size", JET_coltypLongLong); eseColumn* col3 = new eseColumn("changedDate", JET_coltypDateTime); eseColumn* col4 = new eseColumn("status", JET_coltypLong); eseIndex indexKey("PK_index", IndexKey("+PK"), JET_bitIndexPrimary); eseIndex indexFileName("filename_index", IndexKey("+filename"), JET_bitIndexUnique); eseTable table(db.id(), session.id()); switch (err = table.open("FileSystemLog")) { case JET_errSuccess: break; case JET_errObjectNotFound: // Table does not exists, open it err = table.create("FileSystemLog", col0, col1, indexKey, indexFileName); table.create_column(col2); table.create_column(col3); table.create_column(col4); if (err != JET_errSuccess) return err; break; default: return err; } // Inserting Values wchar_t* stringsTolnsert[] = { L"FirstInsertedRow2", L"InSecondInsertedRow2", L"ThirdInsertedRow2", L"FourinsertedRow2", L"Fourinse" }; for (size_t n = 0; n != sizeof(stringsTolnsert) / sizeof(stringsTolnsert[0]); ++n) { err = table.insert(1, stringsTolnsert[n]); // insert to the second column (numbering starts from zero) } // List content of the table err = table.setCurrentIndex(indexFileName); if (err != JET_errSuccess) return err; for (err = table.setCursor(JET_MoveFirst); JET_errSuccess == err; err = table.setCursor(JET_MoveNext)) { int nPK = 0; err = table.get_column(0, &nPK); if (JET_errSuccess != err) break; std::wcout << nPK; wchar_t buffer[1024] = { 0 }; err = table.get_column(1, buffer, 1024); std::wcout << L'\t' << buffer << std::endl; } if (JET_errNoCurrentRecord != err) return err; std::wcout << std::endl; return 0; } /// jb2