StringData* StringData::reserve(size_t cap) {
assert(!isImmutable() && !hasMultipleRefs());
assert(isFlat());
if (cap <= capacity()) return this;
cap = std::min(cap + cap / 4, size_t(MaxSize) + 1);
auto const allocRet = allocFlatForLen(cap);
auto const sd = allocRet.first;
auto const cc = allocRet.second;
auto const data = reinterpret_cast<char*>(sd + 1);
sd->m_data = data;
sd->m_hdr.init(cc, HeaderKind::String, 1);
// request-allocated StringData are always aligned at 16 bytes, thus it is
// safe to copy in 16-byte groups. This copies m_lenAndHash (8 bytes), the
// characters (m_len bytes), add the trailing zero (1 byte).
memcpy16_inline(&sd->m_lenAndHash, &m_lenAndHash,
(m_len + 8 + 1 + 15) & ~0xF);
assertx(reinterpret_cast<uintptr_t>(&m_lenAndHash) + 8 ==
reinterpret_cast<uintptr_t>(m_data));
assertx(reinterpret_cast<uintptr_t>(&m_lenAndHash) % 16 == 0);
assert(sd->hasExactlyOneRef());
assert(sd->isFlat());
assert(sd->checkSane());
return sd;
}
NEVER_INLINE
StringData* StringData::MakeSVSlowPath(APCString* shared, uint32_t len) {
auto const data = shared->getStringData();
auto const hash = data->m_hash & STRHASH_MASK;
auto const capAndHash = static_cast<uint64_t>(hash) << 32;
auto const sd = static_cast<StringData*>(
MM().smartMallocSize(sizeof(StringData) + sizeof(SharedPayload))
);
sd->m_data = const_cast<char*>(data->m_data);
sd->m_lenAndCount = len;
sd->m_capAndHash = capAndHash;
sd->sharedPayload()->shared = shared;
sd->enlist();
shared->getHandle()->reference();
assert(sd->m_len == len);
assert(sd->m_count == 0);
assert(sd->m_cap == 0); // cap == 0 means shared
assert(sd->m_hash == hash);
assert(sd->checkSane());
return sd;
}
StringData* StringData::reserve(size_t cap) {
assert(!isImmutable() && !hasMultipleRefs());
assert(isFlat());
if (cap <= capacity()) return this;
cap = std::min(cap + cap / 4, size_t(MaxSize));
auto const sd = allocFlatForLenSmall(cap);
// Request-allocated StringData are always aligned at 16 bytes, thus it is
// safe to copy in 16-byte groups.
#ifdef NO_M_DATA
// layout: [m_lenAndHash][header][...data]
sd->m_lenAndHash = m_lenAndHash;
// This copies the characters (m_len bytes), and the trailing zero (1 byte)
memcpy16_inline(sd+1, this+1, (m_len + 1 + 15) & ~0xF);
assertx(reinterpret_cast<uintptr_t>(this+1) % 16 == 0);
#else
// layout: [m_data][header][m_lenAndHash][...data]
// This copies m_lenAndHash (8 bytes), the characters (m_len bytes),
// and the trailing zero (1 byte).
memcpy16_inline(&sd->m_lenAndHash, &m_lenAndHash,
(m_len + 8 + 1 + 15) & ~0xF);
assertx(reinterpret_cast<uintptr_t>(&m_lenAndHash) + 8 ==
reinterpret_cast<uintptr_t>(m_data));
assertx(reinterpret_cast<uintptr_t>(&m_lenAndHash) % 16 == 0);
#endif
assert(sd->hasExactlyOneRef());
assert(sd->isFlat());
assert(sd->checkSane());
return sd;
}
// Create either a static or an uncounted string.
// Diffrence between static and uncounted is in the lifetime
// of the string. Static are alive for the lifetime of the process.
// Uncounted are not ref counted but will be deleted at some point.
ALWAYS_INLINE
StringData* StringData::MakeShared(StringSlice sl, bool trueStatic) {
if (UNLIKELY(sl.len > StringData::MaxSize)) {
throw_string_too_large(sl.len);
}
auto const cc = CapCode::ceil(sl.len);
auto const need = cc.decode() + kCapOverhead;
auto const sd = static_cast<StringData*>(
trueStatic ? low_malloc(need) : malloc(need)
);
auto const data = reinterpret_cast<char*>(sd + 1);
sd->m_data = data;
auto const count = trueStatic ? StaticValue : UncountedValue;
sd->m_hdr.init(cc, HeaderKind::String, count);
sd->m_len = sl.len; // m_hash is computed soon.
data[sl.len] = 0;
auto const mcret = memcpy(data, sl.ptr, sl.len);
auto const ret = reinterpret_cast<StringData*>(mcret) - 1;
// Recalculating ret from mcret avoids a spill.
ret->preCompute(); // get m_hash right
assert(ret == sd);
assert(ret->isFlat());
assert(trueStatic ? ret->isStatic() : ret->isUncounted());
assert(ret->checkSane());
return ret;
}
StringData* StringData::append(StringSlice range) {
assert(!hasMultipleRefs());
auto s = range.ptr;
auto const len = range.len;
if (len == 0) return this;
auto const newLen = size_t(m_len) + size_t(len);
if (UNLIKELY(newLen > MaxSize)) {
throw_string_too_large(newLen);
}
/*
* We may have an aliasing append. We don't allow appending with an
* interior pointer, although we may be asked to append less than
* the whole string in an aliasing situation.
*/
ALIASING_APPEND_ASSERT(s, len);
auto const requestLen = static_cast<uint32_t>(newLen);
auto const target = UNLIKELY(isShared()) ? escalate(requestLen)
: reserve(requestLen);
memcpy(target->mutableData() + m_len, s, len);
target->setSize(newLen);
assert(target->checkSane());
return target;
}
NEVER_INLINE
StringData* StringData::MakeProxySlowPath(const APCString* apcstr) {
#ifdef NO_M_DATA
always_assert(false);
not_reached();
#else
auto const sd = static_cast<StringData*>(
MM().mallocSmallSize(sizeof(StringData) + sizeof(Proxy))
);
auto const data = apcstr->getStringData();
sd->m_data = const_cast<char*>(data->m_data);
sd->m_hdr.init(data->m_hdr, 1);
sd->m_lenAndHash = data->m_lenAndHash;
sd->proxy()->apcstr = apcstr;
sd->enlist();
apcstr->reference();
assert(sd->m_len == data->size());
assert(sd->m_hdr.aux == data->m_hdr.aux);
assert(sd->m_hdr.kind == HeaderKind::String);
assert(sd->hasExactlyOneRef());
assert(sd->m_hash == data->m_hash);
assert(sd->isProxy());
assert(sd->checkSane());
return sd;
#endif
}
StringData* StringData::Make(const APCString* shared) {
// No need to check if len > MaxSize, because if it were we'd never
// have made the StringData in the APCVariant without throwing.
assert(size_t(shared->getStringData()->size()) <= size_t(MaxSize));
auto const data = shared->getStringData();
auto const len = data->size();
if (UNLIKELY(len > SmallStringReserve)) {
return MakeAPCSlowPath(shared);
}
// small-string path: make a flat copy.
static_assert(SmallStringReserve + kCapOverhead <= CapCode::Threshold, "");
static_assert(SmallStringReserve + kCapOverhead == 64, "");
auto const sd = allocFlatSmallImpl(SmallStringReserve);
sd->m_lenAndHash = data->m_lenAndHash;
auto const psrc = data->data();
auto const pdst = reinterpret_cast<char*>(sd + 1);
auto const mcret = memcpy(pdst, psrc, len + 1); // also copy the tailing 0
auto const ret = reinterpret_cast<StringData*>(mcret) - 1;
// Recalculating ret from mcret avoids a spill.
assert(ret == sd);
assert(ret->m_len == len);
assert(ret->hasExactlyOneRef());
assert(ret->m_hash == data->m_hash);
assert(ret->isFlat());
assert(ret->checkSane());
return ret;
}
HOT_FUNC
void StringData::releaseData() {
Format f = format();
if (f == IsSmall) return;
if (f == IsMalloc) {
ASSERT(checkSane());
free(m_data);
m_data = NULL;
return;
}
if (f == IsShared) {
ASSERT(checkSane());
m_big.shared->decRef();
return;
}
// Nothing to do for literals, which are rarely destructed anyway.
}
NEVER_INLINE
void StringData::releaseDataSlowPath() {
assert(isProxy());
assert(checkSane());
proxy()->apcstr->getHandle()->unreference();
delist();
MM().freeSmallSize(this, sizeof(StringData) + sizeof(Proxy));
}
StringData* StringData::Make(size_t reserveLen) {
auto const sd = allocFlatForLenSmall(reserveLen);
sd->setSize(0);
assert(sd->hasExactlyOneRef());
assert(sd->isFlat());
assert(sd->checkSane());
return sd;
}
StringData* StringData::Make(char* data, size_t len, AttachStringMode) {
if (UNLIKELY(len > StringData::MaxSize)) {
throw_string_too_large(len);
}
auto const sd = Make(StringSlice(data, len), CopyString);
free(data);
assert(sd->checkSane());
return sd;
}
NEVER_INLINE
void StringData::releaseDataSlowPath() {
assert(!isFlat());
assert(isShared());
assert(checkSane());
sharedPayload()->shared->getHandle()->unreference();
delist();
MM().freeSmallSize(this, sizeof(StringData) + sizeof(SharedPayload));
}
void StringData::escalate() {
ASSERT(isImmutable() && !isStatic() && size() > 0);
StringSlice s = slice();
char *buf = (char*)malloc(s.len + 1);
memcpy(buf, s.ptr, s.len);
buf[s.len] = 0;
m_len = s.len;
m_data = buf;
m_big.cap = s.len | IsMalloc;
// clear precomputed hashcode
m_hash = 0;
ASSERT(checkSane());
}
// State transition from Mode::Shared to Mode::Flat.
StringData* StringData::escalate(size_t cap) {
assert(isShared() && !isStatic() && cap >= m_len);
auto const sd = allocFlatForLenSmall(cap);
sd->m_lenAndHash = m_lenAndHash;
auto const data = reinterpret_cast<char*>(sd + 1);
*memcpy8(data, m_data, m_len) = 0;
assert(sd->hasExactlyOneRef());
assert(sd->isFlat());
assert(sd->checkSane());
return sd;
}
HOT_FUNC
void StringData::releaseDataSlowPath() {
assert(!isSmall());
assert(checkSane());
auto const loadedMode = mode();
if (LIKELY(loadedMode == Mode::Smart)) {
smart_free(m_data);
return;
}
if (loadedMode == Mode::Shared) {
assert(checkSane());
m_big.shared->decRef();
delist();
return;
}
assert(loadedMode == Mode::Malloc);
assert(checkSane());
free(m_data);
}
StringData* StringData::shrinkImpl(size_t len) {
assert(!isImmutable() && !hasMultipleRefs());
assert(isFlat());
assert(len <= capacity());
auto const sd = allocFlatForLenSmall(len);
sd->m_lenAndHash = len;
auto const src = static_cast<void*>(this + 1);
auto const dst = static_cast<void*>(sd + 1);
*memcpy8(dst, src, len) = 0;
assert(sd->checkSane());
return sd;
}
StringData* StringData::shrinkImpl(size_t len) {
assert(!isImmutable() && !hasMultipleRefs());
assert(isFlat());
assert(len <= capacity());
auto const sd = Make(len);
sd->m_len = len;
auto const src = slice();
auto const dst = sd->mutableData();
*memcpy8(dst, src.ptr, len) = 0;
assert(sd->checkSane());
return sd;
}
StringData* StringData::Make(StringSlice r1, StringSlice r2) {
auto const len = r1.len + r2.len;
auto const sd = allocFlatForLenSmall(len);
sd->m_lenAndHash = len; // hash=0
auto const data = reinterpret_cast<char*>(sd + 1);
memcpy(data, r1.ptr, r1.len);
memcpy(data + r1.len, r2.ptr, r2.len);
data[len] = 0;
assert(sd->hasExactlyOneRef());
assert(sd->isFlat());
assert(sd->checkSane());
return sd;
}
void StringData::initAttachDeprecated(const char* data, int len) {
if (uint32_t(len) > MaxSize) {
throw InvalidArgumentException("len>=2^30", len);
}
// Don't copy small strings here either because the caller sometimes
// assumes he can mess with data while this string is still alive,
// and we want to free it eagerly. Sketchy!
m_hash = 0;
_count = 0;
m_len = len;
m_cdata = data;
m_big.cap = len | IsMalloc;
ASSERT(checkSane());
TAINT_OBSERVER_REGISTER_MUTATED(m_taint_data, rawdata());
}
void StringData::initLiteral(const char* data, int len) {
if (uint32_t(len) > MaxSize) {
throw InvalidArgumentException("len>=2^30", len);
}
// Do not copy literals, this StringData can have a shorter lifetime than
// the literal, and the client can count on this->data() giving back
// the literal ptr with the longer lifetime. Sketchy!
m_hash = 0;
_count = 0;
m_len = len;
m_cdata = data;
m_big.cap = len | IsLiteral;
ASSERT(checkSane());
TAINT_OBSERVER_REGISTER_MUTATED(m_taint_data, rawdata());
}
StringData* StringData::Make(folly::StringPiece r1, folly::StringPiece r2) {
auto const len = r1.size() + r2.size();
auto const sd = allocFlatForLenSmall(len);
sd->m_lenAndHash = len; // hash=0
auto const data = reinterpret_cast<char*>(sd + 1);
memcpy(data, r1.data(), r1.size());
memcpy(data + r1.size(), r2.data(), r2.size());
data[len] = 0;
assert(sd->hasExactlyOneRef());
assert(sd->isFlat());
assert(sd->checkSane());
return sd;
}
StringData* StringData::Make(size_t reserveLen) {
auto const allocRet = allocFlatForLen(reserveLen);
auto const sd = allocRet.first;
auto const capCode = allocRet.second;
auto const data = reinterpret_cast<char*>(sd + 1);
data[0] = 0;
sd->m_data = data;
sd->m_capAndCount = HeaderKind::String << 24 | capCode; // count=0
sd->m_lenAndHash = 0; // len=hash=0
assert(sd->isFlat());
assert(sd->checkSane());
return sd;
}
StringData* StringData::Make(size_t reserveLen) {
auto const allocRet = allocFlatForLen(reserveLen);
auto const sd = allocRet.first;
auto const cc = allocRet.second;
auto const data = reinterpret_cast<char*>(sd + 1);
data[0] = 0;
sd->m_data = data;
sd->m_hdr.init(cc, HeaderKind::String, 1);
sd->m_lenAndHash = 0; // len=hash=0
assert(sd->hasExactlyOneRef());
assert(sd->isFlat());
assert(sd->checkSane());
return sd;
}
// State transition from Mode::Shared to Mode::Flat.
StringData* StringData::escalate(size_t cap) {
assert(isShared() && !isStatic() && cap >= m_len);
auto const sd = Make(cap);
auto const src = slice();
auto const dst = sd->mutableData();
sd->setSize(src.len);
auto const mcret = memcpy(dst, src.ptr, src.len);
auto const ret = static_cast<StringData*>(mcret) - 1;
// Recalculating ret from mcret avoids a spill.
assert(ret == sd);
assert(ret->checkSane());
return ret;
}
void StringData::release() noexcept {
assert(checkSane());
if (auto const sizeClass = m_hdr.smallSizeClass) {
assert(isFlat());
MM().freeSmallIndex(this, sizeClass,
MemoryManager::smallIndex2Size(sizeClass));
} else {
// We know that the string is request local. A `smallSizeClass' of 0 in
// its header indicates that the StringData is either not flat, or too big
// to fit in a small size class.
if (!isFlat()) return releaseDataSlowPath();
auto const size = capacity() + kCapOverhead;
assertx(size > kMaxSmallSize);
MM().freeBigSize(this, size);
}
}
StringData* StringData::Make(const StringData* s1, const StringData* s2) {
auto const len = s1->m_len + s2->m_len;
// `memcpy8()' could overrun the buffer by at most 7 bytes, so we allocate 6
// more bytes here, which (together with the trailing 0) makes it safe.
auto const sd = allocFlatForLenSmall(len + 6);
sd->m_lenAndHash = len; // hash=0
auto const data = reinterpret_cast<char*>(sd + 1);
auto const next = memcpy8(data, s1->m_data, s1->m_len);
*memcpy8(next, s2->m_data, s2->m_len) = 0;
assert(sd->getCount() == 1);
assert(sd->isFlat());
assert(sd->checkSane());
return sd;
}
StringData* StringData::Make(folly::StringPiece r1, folly::StringPiece r2,
folly::StringPiece r3) {
auto const len = r1.size() + r2.size() + r3.size();
auto const sd = allocFlatForLenSmall(len);
sd->m_lenAndHash = len; // hash=0
auto p = reinterpret_cast<char*>(sd + 1);
p = static_cast<char*>(memcpy(p, r1.data(), r1.size()));
p = static_cast<char*>(memcpy(p + r1.size(), r2.data(), r2.size()));
p = static_cast<char*>(memcpy(p + r2.size(), r3.data(), r3.size()));
p[r3.size()] = 0;
assert(sd->hasExactlyOneRef());
assert(sd->isFlat());
assert(sd->checkSane());
return sd;
}
StringData* StringData::Make(StringSlice r1, StringSlice r2,
StringSlice r3) {
auto const len = r1.len + r2.len + r3.len;
auto const sd = allocFlatForLenSmall(len);
sd->m_lenAndHash = len; // hash=0
char* p = reinterpret_cast<char*>(sd + 1);
p = static_cast<char*>(memcpy(p, r1.ptr, r1.len));
p = static_cast<char*>(memcpy(p + r1.len, r2.ptr, r2.len));
p = static_cast<char*>(memcpy(p + r2.len, r3.ptr, r3.len));
p[r3.len] = 0;
assert(sd->hasExactlyOneRef());
assert(sd->isFlat());
assert(sd->checkSane());
return sd;
}
StringData* StringData::Make(const APCString* shared) {
// No need to check if len > MaxSize, because if it were we'd never
// have made the StringData in the APCVariant without throwing.
assert(size_t(shared->getStringData()->size()) <= size_t(MaxSize));
auto const data = shared->getStringData();
auto const len = data->size();
if (UNLIKELY(len > SmallStringReserve)) {
return MakeAPCSlowPath(shared);
}
// small-string path
auto const psrc = data->data();
auto const hash = data->m_hash & STRHASH_MASK;
assert(hash != 0);
static_assert(SmallStringReserve + sizeof(StringData) + 1 <
CapCode::Threshold, "");
auto const need = sizeof(StringData) + len + 1;
auto const cap = MemoryManager::smallSizeClass(need);
auto const sd = static_cast<StringData*>(MM().mallocSmallSize(cap));
auto const pdst = reinterpret_cast<char*>(sd + 1);
auto const cc = CapCode::ceil(cap - kCapOverhead);
assert(cc.code == cap - kCapOverhead);
sd->m_data = pdst;
sd->m_hdr.init(cc, HeaderKind::String, 1);
sd->m_lenAndHash = len | int64_t{hash} << 32;
// pdst[len] = 0;
auto const mcret = memcpy(pdst, psrc, len + 1);
auto const ret = reinterpret_cast<StringData*>(mcret) - 1;
// Recalculating ret from mcret avoids a spill.
// Note: this return value thing is doing a dead lea into %rsi in
// the caller for some reason.
assert(ret == sd);
assert(ret->m_len == len);
assert(ret->hasExactlyOneRef());
assert(ret->m_hash == hash);
assert(ret->isFlat());
assert(ret->checkSane());
return ret;
}
StringData* StringData::append(StringSlice r1,
StringSlice r2,
StringSlice r3) {
assert(!hasMultipleRefs());
auto const len = r1.len + r2.len + r3.len;
if (len == 0) return this;
if (UNLIKELY(uint32_t(len) > MaxSize)) {
throw_string_too_large(len);
}
if (UNLIKELY(size_t(m_len) + size_t(len) > MaxSize)) {
throw_string_too_large(size_t(len) + size_t(m_len));
}
auto const newLen = m_len + len;
/*
* We may have an aliasing append. We don't allow appending with an
* interior pointer, although we may be asked to append less than
* the whole string in an aliasing situation.
*/
ALIASING_APPEND_ASSERT(r1.ptr, r1.len);
ALIASING_APPEND_ASSERT(r2.ptr, r2.len);
ALIASING_APPEND_ASSERT(r3.ptr, r3.len);
auto const target = UNLIKELY(isShared()) ? escalate(newLen)
: reserve(newLen);
auto const mslice = target->bufferSlice();
/*
* memcpy is safe even if it's a self append---the regions will be
* disjoint, since rN.ptr can't point past the start of our source
* pointer, and rN.len is smaller than the old length.
*/
void* p = mslice.ptr;
p = memcpy((char*)p + m_len, r1.ptr, r1.len);
p = memcpy((char*)p + r1.len, r2.ptr, r2.len);
memcpy((char*)p + r2.len, r3.ptr, r3.len);
target->setSize(newLen);
assert(target->checkSane());
return target;
}
