RocksEngine::RocksEngine(const std::string& path, bool durable) : _path(path), _durable(durable) { { // create block cache uint64_t cacheSizeGB = 0; ProcessInfo pi; unsigned long long memSizeMB = pi.getMemSizeMB(); if (memSizeMB > 0) { double cacheMB = memSizeMB / 2; cacheSizeGB = static_cast<uint64_t>(cacheMB / 1024); } if (cacheSizeGB < 1) { cacheSizeGB = 1; } _block_cache = rocksdb::NewLRUCache(cacheSizeGB * 1024 * 1024 * 1024LL); } // open DB rocksdb::DB* db; auto s = rocksdb::DB::Open(_options(), path, &db); ROCKS_STATUS_OK(s); _db.reset(db); // open iterator boost::scoped_ptr<rocksdb::Iterator> _iter(_db->NewIterator(rocksdb::ReadOptions())); // find maxPrefix _maxPrefix = 0; _iter->SeekToLast(); if (_iter->Valid()) { // otherwise the DB is empty, so we just keep it at 0 bool ok = extractPrefix(_iter->key(), &_maxPrefix); // this is DB corruption here invariant(ok); } // load ident to prefix map { boost::mutex::scoped_lock lk(_identPrefixMapMutex); for (_iter->Seek(kMetadataPrefix); _iter->Valid() && _iter->key().starts_with(kMetadataPrefix); _iter->Next()) { rocksdb::Slice ident(_iter->key()); ident.remove_prefix(kMetadataPrefix.size()); // this could throw DBException, which then means DB corruption. We just let it fly // to the caller BSONObj identConfig(_iter->value().data()); BSONElement element = identConfig.getField("prefix"); // TODO: SERVER-16979 Correctly handle errors returned by RocksDB // This is DB corruption invariant(!element.eoo() || !element.isNumber()); uint32_t identPrefix = static_cast<uint32_t>(element.numberInt()); _identPrefixMap[StringData(ident.data(), ident.size())] = identPrefix; } } }
// cannot be rolled back Status RocksEngine::dropIdent(OperationContext* opCtx, StringData ident) { rocksdb::WriteBatch wb; wb.Delete(kMetadataPrefix + ident.toString()); // calculate which prefixes we need to drop std::vector<std::string> prefixesToDrop; prefixesToDrop.push_back(_getIdentPrefix(ident)); if (_oplogIdent == ident.toString()) { // if we're dropping oplog, we also need to drop keys from RocksOplogKeyTracker (they // are stored at prefix+1) prefixesToDrop.push_back(rocksGetNextPrefix(prefixesToDrop[0])); } // We record the fact that we're deleting this prefix. That way we ensure that the prefix is // always deleted for (const auto& prefix : prefixesToDrop) { wb.Put(kDroppedPrefix + prefix, ""); } // we need to make sure this is on disk before starting to delete data in compactions rocksdb::WriteOptions syncOptions; syncOptions.sync = true; auto s = _db->Write(syncOptions, &wb); if (!s.ok()) { return rocksToMongoStatus(s); } // remove from map { boost::lock_guard<boost::mutex> lk(_identPrefixMapMutex); _identPrefixMap.erase(ident); } // instruct compaction filter to start deleting { boost::lock_guard<boost::mutex> lk(_droppedPrefixesMutex); for (const auto& prefix : prefixesToDrop) { uint32_t int_prefix; bool ok = extractPrefix(prefix, &int_prefix); invariant(ok); _droppedPrefixes.insert(int_prefix); } } return Status::OK(); }
static void Word_postProcessHTML(WordConverter *conv, DFNode *node) { DFNode *next; for (DFNode *child = node->first; child != NULL; child = next) { next = child->next; switch (child->tag) { case HTML_SPAN: { const char *className = DFGetAttribute(child,HTML_CLASS); if (DFStringEquals(className,DFBookmarkClass)) { if (child->first != NULL) next = child->first; DFRemoveNodeButKeepChildren(child); } break; } case HTML_CAPTION: { const char *counterName = NULL; if ((child->prev != NULL) && (child->prev->tag == HTML_FIGURE) && (DFChildWithTag(child->prev,HTML_FIGCAPTION) == NULL)) { child->tag = HTML_FIGCAPTION; counterName = "figure"; DFAppendChild(child->prev,child); } else if ((child->prev != NULL) && (child->prev->tag == HTML_TABLE) && (DFChildWithTag(child->prev,HTML_CAPTION) == NULL)) { counterName = "table"; DFInsertBefore(child->prev,child,child->prev->first); } else if ((child->next != NULL) && (child->next->tag == HTML_FIGURE) && (DFChildWithTag(child->next,HTML_FIGCAPTION) == NULL)) { child->tag = HTML_FIGCAPTION; counterName = "figure"; DFInsertBefore(child->next,child,child->next->first); } else if ((child->next != NULL) && (child->next->tag == HTML_TABLE) && (DFChildWithTag(child->next,HTML_CAPTION) == NULL)) { counterName = "table"; DFSetAttribute(child,HTML_STYLE,"caption-side: top"); DFInsertBefore(child->next,child,child->next->first); } if (counterName != NULL) { char *beforeText = extractPrefix(child,counterName); if (beforeText != NULL) { CSSStyle *style = CSSSheetLookupElement(conv->styleSheet,DFNodeName(child),NULL,1,0); if (CSSGet(CSSStyleBefore(style),"content") == NULL) { CSSPut(CSSStyleRule(style),"counter-increment",counterName); CSSPut(CSSStyleBefore(style),"content",beforeText); } } free(beforeText); } break; } case HTML_NAV: { if (HTML_isParagraphTag(node->tag)) { if (child->prev != NULL) { DFNode *beforeP = DFCreateElement(conv->package->document,node->tag); while (child->prev != NULL) DFInsertBefore(beforeP,child->prev,beforeP->first); DFInsertBefore(node->parent,beforeP,node); } DFInsertBefore(node->parent,child,node); if ((node->first == NULL) || ((node->first->tag == HTML_BR) && (node->first->next == NULL))) { DFRemoveNode(node); return; } next = NULL; } break; } } } for (DFNode *child = node->first; child != NULL; child = next) { next = child->next; Word_postProcessHTML(conv,child); } }
RocksEngine::RocksEngine(const std::string& path, bool durable) : _path(path), _durable(durable), _maxPrefix(0) { { // create block cache uint64_t cacheSizeGB = rocksGlobalOptions.cacheSizeGB; if (cacheSizeGB == 0) { ProcessInfo pi; unsigned long long memSizeMB = pi.getMemSizeMB(); if (memSizeMB > 0) { double cacheMB = memSizeMB / 2; cacheSizeGB = static_cast<uint64_t>(cacheMB / 1024); } if (cacheSizeGB < 1) { cacheSizeGB = 1; } } _block_cache = rocksdb::NewLRUCache(cacheSizeGB * 1024 * 1024 * 1024LL, 6); } _maxWriteMBPerSec = rocksGlobalOptions.maxWriteMBPerSec; _rateLimiter.reset( rocksdb::NewGenericRateLimiter(static_cast<int64_t>(_maxWriteMBPerSec) * 1024 * 1024)); // open DB rocksdb::DB* db; auto s = rocksdb::DB::Open(_options(), path, &db); invariantRocksOK(s); _db.reset(db); _counterManager.reset( new RocksCounterManager(_db.get(), rocksGlobalOptions.crashSafeCounters)); _compactionScheduler.reset(new RocksCompactionScheduler(_db.get())); // open iterator boost::scoped_ptr<rocksdb::Iterator> iter(_db->NewIterator(rocksdb::ReadOptions())); // find maxPrefix iter->SeekToLast(); if (iter->Valid()) { // otherwise the DB is empty, so we just keep it at 0 bool ok = extractPrefix(iter->key(), &_maxPrefix); // this is DB corruption here invariant(ok); } // load ident to prefix map. also update _maxPrefix if there's any prefix bigger than // current _maxPrefix { boost::lock_guard<boost::mutex> lk(_identPrefixMapMutex); for (iter->Seek(kMetadataPrefix); iter->Valid() && iter->key().starts_with(kMetadataPrefix); iter->Next()) { invariantRocksOK(iter->status()); rocksdb::Slice ident(iter->key()); ident.remove_prefix(kMetadataPrefix.size()); // this could throw DBException, which then means DB corruption. We just let it fly // to the caller BSONObj identConfig(iter->value().data()); BSONElement element = identConfig.getField("prefix"); if (element.eoo() || !element.isNumber()) { log() << "Mongo metadata in RocksDB database is corrupted."; invariant(false); } uint32_t identPrefix = static_cast<uint32_t>(element.numberInt()); _identPrefixMap[StringData(ident.data(), ident.size())] = identPrefix; _maxPrefix = std::max(_maxPrefix, identPrefix); } } // just to be extra sure. we need this if last collection is oplog -- in that case we // reserve prefix+1 for oplog key tracker ++_maxPrefix; // load dropped prefixes { rocksdb::WriteBatch wb; // we will use this iter to check if prefixes are still alive boost::scoped_ptr<rocksdb::Iterator> prefixIter( _db->NewIterator(rocksdb::ReadOptions())); for (iter->Seek(kDroppedPrefix); iter->Valid() && iter->key().starts_with(kDroppedPrefix); iter->Next()) { invariantRocksOK(iter->status()); rocksdb::Slice prefix(iter->key()); prefix.remove_prefix(kDroppedPrefix.size()); prefixIter->Seek(prefix); invariantRocksOK(iter->status()); if (prefixIter->Valid() && prefixIter->key().starts_with(prefix)) { // prefix is still alive, let's instruct the compaction filter to clear it up uint32_t int_prefix; bool ok = extractPrefix(prefix, &int_prefix); invariant(ok); { boost::lock_guard<boost::mutex> lk(_droppedPrefixesMutex); _droppedPrefixes.insert(int_prefix); } } else { // prefix is no longer alive. let's remove the prefix from our dropped prefixes // list wb.Delete(iter->key()); } } if (wb.Count() > 0) { auto s = _db->Write(rocksdb::WriteOptions(), &wb); invariantRocksOK(s); } } }