NEVER_INLINE StringData* StringData::MakeSVSlowPath(APCString* shared, uint32_t len) { auto const data = shared->getStringData(); auto const hash = data->m_hash & STRHASH_MASK; auto const capAndHash = static_cast<uint64_t>(hash) << 32; auto const sd = static_cast<StringData*>( MM().smartMallocSize(sizeof(StringData) + sizeof(SharedPayload)) ); sd->m_data = const_cast<char*>(data->m_data); sd->m_lenAndCount = len; sd->m_capAndHash = capAndHash; sd->sharedPayload()->shared = shared; sd->enlist(); shared->getHandle()->reference(); assert(sd->m_len == len); assert(sd->m_count == 0); assert(sd->m_cap == 0); // cap == 0 means shared assert(sd->m_hash == hash); assert(sd->checkSane()); return sd; }
NEVER_INLINE void StringData::releaseDataSlowPath() { assert(!isFlat()); assert(isShared()); assert(checkSane()); sharedPayload()->shared->getHandle()->unreference(); delist(); MM().freeSmallSize(this, sizeof(StringData) + sizeof(SharedPayload)); }
ALWAYS_INLINE void StringData::delist() { assert(isShared()); auto& payload = *sharedPayload(); auto const next = payload.node.next; auto const prev = payload.node.prev; assert(uintptr_t(next) != kMallocFreeWord); assert(uintptr_t(prev) != kMallocFreeWord); next->prev = prev; prev->next = next; }
// Defined here for inlining into MakeSVSlowPath ALWAYS_INLINE void StringData::enlist() { assert(isShared()); auto& head = MM().m_strings; // insert after head auto const next = head.next; auto& payload = *sharedPayload(); assert(uintptr_t(next) != kMallocFreeWord); payload.node.next = next; payload.node.prev = &head; next->prev = head.next = &payload.node; }
unsigned StringData::sweepAll() { auto& head = MM().getStringList(); auto count = 0; for (StringDataNode *next, *n = head.next; n != &head; n = next) { count++; next = n->next; assert(next && uintptr_t(next) != kSmallFreeWord); assert(next && uintptr_t(next) != kMallocFreeWord); auto const s = node2str(n); assert(s->isShared()); s->sharedPayload()->shared->getHandle()->unreference(); } head.next = head.prev = &head; return count; }
unsigned StringData::sweepAll() { auto& head = MM().getStringList(); auto count = 0; for (StringDataNode *next, *n = head.next; n != &head; n = next) { count++; next = n->next; assert(next && uintptr_t(next) != kSmartFreeWord); assert(next && uintptr_t(next) != kMallocFreeWord); auto const s = reinterpret_cast<StringData*>( uintptr_t(n) - offsetof(SharedPayload, node) - sizeof(StringData) ); assert(s->isShared()); s->sharedPayload()->shared->getHandle()->unreference(); } head.next = head.prev = &head; return count; }
NEVER_INLINE StringData* StringData::MakeAPCSlowPath(const APCString* shared) { auto const sd = static_cast<StringData*>( MM().mallocSmallSize(sizeof(StringData) + sizeof(SharedPayload)) ); auto const data = shared->getStringData(); sd->m_data = const_cast<char*>(data->m_data); sd->m_hdr.init(data->m_hdr, 1); sd->m_lenAndHash = data->m_lenAndHash; sd->sharedPayload()->shared = shared; sd->enlist(); shared->getHandle()->reference(); assert(sd->m_len == data->size()); assert(sd->m_hdr.aux == data->m_hdr.aux); assert(sd->m_hdr.kind == HeaderKind::String); assert(sd->hasExactlyOneRef()); assert(sd->m_hash == data->m_hash); assert(sd->isShared()); assert(sd->checkSane()); return sd; }
NEVER_INLINE StringData* StringData::MakeAPCSlowPath(const APCString* shared) { auto const sd = static_cast<StringData*>( MM().smartMallocSize(sizeof(StringData) + sizeof(SharedPayload)) ); auto const data = shared->getStringData(); sd->m_data = const_cast<char*>(data->m_data); sd->m_capAndCount = data->m_capCode; // count=0, kind=data->kind sd->m_lenAndHash = data->m_lenAndHash; sd->sharedPayload()->shared = shared; sd->enlist(); shared->getHandle()->reference(); assert(sd->m_len == data->size()); assert(sd->m_count == 0); assert(sd->m_capCode == data->m_capCode); assert(sd->m_hash == data->m_hash); assert(sd->m_kind == HeaderKind::String); assert(sd->isShared()); assert(sd->checkSane()); return sd; }
APCHandle* StringData::getAPCHandle() const { if (isShared()) return sharedPayload()->shared->getHandle(); return nullptr; }