Memory::Memory(PageCount initial, PageCount maximum, bool& failed)
: m_size(initial.bytes())
, m_initial(initial)
, m_maximum(maximum)
, m_mode(Mode::BoundsChecking)
// FIXME: If we add signal based bounds checking then we need extra space for overflow on load.
// see: https://bugs.webkit.org/show_bug.cgi?id=162693
{
RELEASE_ASSERT(!maximum || maximum >= initial); // This should be guaranteed by our caller.
m_mappedCapacity = maximum ? maximum.bytes() : PageCount::max().bytes();
if (!m_mappedCapacity) {
// This means we specified a zero as maximum (which means we also have zero as initial size).
RELEASE_ASSERT(m_size == 0);
m_memory = nullptr;
m_mappedCapacity = 0;
failed = false;
if (verbose)
dataLogLn("Memory::Memory allocating nothing ", *this);
return;
}
// FIXME: It would be nice if we had a VM tag for wasm memory. https://bugs.webkit.org/show_bug.cgi?id=163600
void* result = Options::simulateWebAssemblyLowMemory() ? MAP_FAILED : mmap(nullptr, m_mappedCapacity, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0);
if (result == MAP_FAILED) {
// Try again with a different number.
if (verbose)
dataLogLn("Memory::Memory mmap failed once for capacity, trying again", *this);
m_mappedCapacity = m_size;
if (!m_mappedCapacity) {
m_memory = nullptr;
failed = false;
if (verbose)
dataLogLn("Memory::Memory mmap not trying again because size is zero ", *this);
return;
}
result = mmap(nullptr, m_mappedCapacity, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0);
if (result == MAP_FAILED) {
if (verbose)
dataLogLn("Memory::Memory mmap failed twice ", *this);
failed = true;
return;
}
}
ASSERT(m_size <= m_mappedCapacity);
{
bool success = !mprotect(result, static_cast<size_t>(m_size), PROT_READ | PROT_WRITE);
RELEASE_ASSERT(success);
}
m_memory = result;
failed = false;
if (verbose)
dataLogLn("Memory::Memory mmap succeeded ", *this);
}
void Plan::run()
{
if (verbose)
dataLogLn("Starting plan.");
{
ModuleParser moduleParser(m_source, m_sourceLength);
if (!moduleParser.parse()) {
if (verbose)
dataLogLn("Parsing module failed: ", moduleParser.errorMessage());
m_errorMessage = moduleParser.errorMessage();
return;
}
m_moduleInformation = WTFMove(moduleParser.moduleInformation());
}
if (verbose)
dataLogLn("Parsed module.");
if (!m_compiledFunctions.tryReserveCapacity(m_moduleInformation->functions.size())) {
StringBuilder builder;
builder.appendLiteral("Failed allocating enough space for ");
builder.appendNumber(m_moduleInformation->functions.size());
builder.appendLiteral(" compiled functions");
m_errorMessage = builder.toString();
return;
}
for (const FunctionInformation& info : m_moduleInformation->functions) {
if (verbose)
dataLogLn("Processing function starting at: ", info.start, " and ending at: ", info.end);
const uint8_t* functionStart = m_source + info.start;
size_t functionLength = info.end - info.start;
ASSERT(functionLength <= m_sourceLength);
String error = validateFunction(functionStart, functionLength, info.signature, m_moduleInformation->functions);
if (!error.isNull()) {
if (verbose) {
for (unsigned i = 0; i < functionLength; ++i)
dataLog(RawPointer(reinterpret_cast<void*>(functionStart[i])), ", ");
dataLogLn();
}
m_errorMessage = error;
return;
}
m_compiledFunctions.uncheckedAppend(parseAndCompile(*m_vm, functionStart, functionLength, m_moduleInformation->memory.get(), info.signature, m_moduleInformation->functions));
}
// Patch the call sites for each function.
for (std::unique_ptr<FunctionCompilation>& functionPtr : m_compiledFunctions) {
FunctionCompilation* function = functionPtr.get();
for (auto& call : function->unlinkedCalls)
MacroAssembler::repatchCall(call.callLocation, CodeLocationLabel(m_compiledFunctions[call.functionIndex]->code->code()));
}
m_failed = false;
}
bool ModuleParser::parseData()
{
uint32_t segmentCount;
if (!parseVarUInt32(segmentCount)
|| segmentCount == std::numeric_limits<uint32_t>::max()
|| !m_module->data.tryReserveCapacity(segmentCount))
return false;
if (verbose)
dataLogLn(" segments: ", segmentCount);
for (uint32_t segmentNumber = 0; segmentNumber < segmentCount; ++segmentNumber) {
if (verbose)
dataLogLn(" segment #", segmentNumber);
uint32_t index;
uint64_t offset;
uint8_t initOpcode;
uint32_t dataByteLength;
if (!parseVarUInt32(index)
|| index)
return false;
if (!parseInitExpr(initOpcode, offset))
return false;
if (initOpcode != OpType::I32Const)
return false;
if (verbose)
dataLogLn(" offset: ", offset);
if (!parseVarUInt32(dataByteLength)
|| dataByteLength == std::numeric_limits<uint32_t>::max())
return false;
if (verbose)
dataLogLn(" data bytes: ", dataByteLength);
Segment* segment = Segment::make(offset, dataByteLength);
if (!segment)
return false;
m_module->data.uncheckedAppend(Segment::makePtr(segment));
for (uint32_t dataByte = 0; dataByte < dataByteLength; ++dataByte) {
uint8_t byte;
if (!parseUInt8(byte))
return false;
segment->byte(dataByte) = byte;
if (verbose)
dataLogLn(" [", dataByte, "] = ", segment->byte(dataByte));
}
}
return true;
}
Memory::~Memory()
{
if (verbose)
dataLogLn("Memory::~Memory ", *this);
if (m_memory) {
if (munmap(m_memory, m_mappedCapacity))
CRASH();
}
}
void ObjectInitializationScope::verifyPropertiesAreInitialized(JSObject* object)
{
Butterfly* butterfly = object->butterfly();
Structure* structure = object->structure(m_vm);
IndexingType indexingType = structure->indexingType();
unsigned vectorLength = butterfly->vectorLength();
if (UNLIKELY(hasUndecided(indexingType)) || !hasIndexedProperties(indexingType)) {
// Nothing to verify.
} else if (LIKELY(!hasAnyArrayStorage(indexingType))) {
auto data = butterfly->contiguous().data();
for (unsigned i = 0; i < vectorLength; ++i) {
if (isScribbledValue(data[i].get())) {
dataLogLn("Found scribbled value at i = ", i);
ASSERT_NOT_REACHED();
}
}
} else {
ArrayStorage* storage = butterfly->arrayStorage();
for (unsigned i = 0; i < vectorLength; ++i) {
if (isScribbledValue(storage->m_vector[i].get())) {
dataLogLn("Found scribbled value at i = ", i);
ASSERT_NOT_REACHED();
}
}
}
auto isSafeEmptyValueForGCScanning = [] (JSValue value) {
#if USE(JSVALUE64)
return !value;
#else
return !value || !JSValue::encode(value);
#endif
};
for (int64_t i = 0; i < static_cast<int64_t>(structure->outOfLineCapacity()); i++) {
// We rely on properties past the last offset be zero for concurrent GC.
if (i + firstOutOfLineOffset > structure->lastOffset())
ASSERT(isSafeEmptyValueForGCScanning(butterfly->propertyStorage()[-i - 1].get()));
else if (isScribbledValue(butterfly->propertyStorage()[-i - 1].get())) {
dataLogLn("Found scribbled property at i = ", -i - 1);
ASSERT_NOT_REACHED();
}
}
}
bool Plan::parseAndValidateModule()
{
MonotonicTime startTime;
if (verbose || Options::reportCompileTimes())
startTime = MonotonicTime::now();
{
ModuleParser moduleParser(m_vm, m_source, m_sourceLength);
auto parseResult = moduleParser.parse();
if (!parseResult) {
m_errorMessage = parseResult.error();
return false;
}
m_moduleInformation = WTFMove(parseResult->module);
m_functionLocationInBinary = WTFMove(parseResult->functionLocationInBinary);
m_moduleSignatureIndicesToUniquedSignatureIndices = WTFMove(parseResult->moduleSignatureIndicesToUniquedSignatureIndices);
}
for (unsigned functionIndex = 0; functionIndex < m_functionLocationInBinary.size(); ++functionIndex) {
if (verbose)
dataLogLn("Processing function starting at: ", m_functionLocationInBinary[functionIndex].start, " and ending at: ", m_functionLocationInBinary[functionIndex].end);
const uint8_t* functionStart = m_source + m_functionLocationInBinary[functionIndex].start;
size_t functionLength = m_functionLocationInBinary[functionIndex].end - m_functionLocationInBinary[functionIndex].start;
ASSERT(functionLength <= m_sourceLength);
SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[functionIndex];
const Signature* signature = SignatureInformation::get(m_vm, signatureIndex);
auto validationResult = validateFunction(m_vm, functionStart, functionLength, signature, *m_moduleInformation, m_moduleSignatureIndicesToUniquedSignatureIndices);
if (!validationResult) {
if (verbose) {
for (unsigned i = 0; i < functionLength; ++i)
dataLog(RawPointer(reinterpret_cast<void*>(functionStart[i])), ", ");
dataLogLn();
}
m_errorMessage = makeString(validationResult.error(), ", in function at index ", String::number(functionIndex)); // FIXME make this an Expected.
return false;
}
}
if (verbose || Options::reportCompileTimes())
dataLogLn("Took ", (MonotonicTime::now() - startTime).microseconds(), " us to validate module");
return true;
}
const Signature* SignatureInformation::get(VM* vm, SignatureIndex index)
{
ASSERT(index != Signature::invalidIndex);
SignatureInformation* info = get(vm);
LockHolder lock(info->m_lock);
if (verbose)
dataLogLn("Got signature ", *info->m_signatures.at(index), " at index ", index);
return info->m_signatures.at(index);
}
SignatureIndex SignatureInformation::adopt(VM* vm, Signature* signature)
{
SignatureInformation* info = get(vm);
LockHolder lock(info->m_lock);
SignatureIndex nextValue = info->m_signatures.size();
auto addResult = info->m_signatureMap.add(SignatureHash { signature }, nextValue);
if (addResult.isNewEntry) {
ASSERT(nextValue == addResult.iterator->value);
if (verbose)
dataLogLn("Adopt new signature ", *signature, " with index ", addResult.iterator->value, " hash: ", signature->hash());
info->m_signatures.append(signature);
return nextValue;
}
if (verbose)
dataLogLn("Existing signature ", *signature, " with index ", addResult.iterator->value, " hash: ", signature->hash());
Signature::destroy(signature);
ASSERT(addResult.iterator->value != Signature::invalidIndex);
return addResult.iterator->value;
}
bool Memory::grow(PageCount newSize)
{
RELEASE_ASSERT(newSize > PageCount::fromBytes(m_size));
if (verbose)
dataLogLn("Memory::grow to ", newSize, " from ", *this);
if (maximum() && newSize > maximum())
return false;
uint64_t desiredSize = newSize.bytes();
if (m_memory && desiredSize <= m_mappedCapacity) {
bool success = !mprotect(static_cast<uint8_t*>(m_memory) + m_size, static_cast<size_t>(desiredSize - m_size), PROT_READ | PROT_WRITE);
RELEASE_ASSERT(success);
m_size = desiredSize;
if (verbose)
dataLogLn("Memory::grow in-place ", *this);
return true;
}
// Otherwise, let's try to make some new memory.
void* newMemory = mmap(nullptr, desiredSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
if (newMemory == MAP_FAILED)
return false;
if (m_memory) {
memcpy(newMemory, m_memory, m_size);
bool success = !munmap(m_memory, m_mappedCapacity);
RELEASE_ASSERT(success);
}
m_memory = newMemory;
m_mappedCapacity = desiredSize;
m_size = desiredSize;
if (verbose)
dataLogLn("Memory::grow ", *this);
return true;
}
bool run()
{
ASSERT(m_graph.m_form == SSA);
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
block->ssa->availabilityAtHead.clear();
block->ssa->availabilityAtTail.clear();
}
BasicBlock* root = m_graph.block(0);
root->ssa->availabilityAtHead.m_locals.fill(Availability::unavailable());
for (unsigned argument = m_graph.m_argumentFormats.size(); argument--;) {
FlushedAt flushedAt = FlushedAt(
m_graph.m_argumentFormats[argument],
virtualRegisterForArgument(argument));
root->ssa->availabilityAtHead.m_locals.argument(argument) = Availability(flushedAt);
}
// This could be made more efficient by processing blocks in reverse postorder.
LocalOSRAvailabilityCalculator calculator(m_graph);
bool changed;
do {
changed = false;
for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
calculator.beginBlock(block);
for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex)
calculator.executeNode(block->at(nodeIndex));
if (calculator.m_availability == block->ssa->availabilityAtTail)
continue;
block->ssa->availabilityAtTail = calculator.m_availability;
changed = true;
for (unsigned successorIndex = block->numSuccessors(); successorIndex--;) {
BasicBlock* successor = block->successor(successorIndex);
successor->ssa->availabilityAtHead.merge(calculator.m_availability);
successor->ssa->availabilityAtHead.pruneByLiveness(
m_graph, successor->at(0)->origin.forExit);
}
}
} while (changed);
if (validationEnabled()) {
for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
calculator.beginBlock(block);
for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
if (block->at(nodeIndex)->origin.exitOK) {
// If we're allowed to exit here, the heap must be in a state
// where exiting wouldn't crash. These particular fields are
// required for correctness because we use them during OSR exit
// to do meaningful things. It would be wrong for any of them
// to be dead.
AvailabilityMap availabilityMap = calculator.m_availability;
availabilityMap.pruneByLiveness(m_graph, block->at(nodeIndex)->origin.forExit);
for (auto heapPair : availabilityMap.m_heap) {
switch (heapPair.key.kind()) {
case ActivationScopePLoc:
case ActivationSymbolTablePLoc:
case FunctionActivationPLoc:
case FunctionExecutablePLoc:
case StructurePLoc:
if (heapPair.value.isDead()) {
dataLogLn("PromotedHeapLocation is dead, but should not be: ", heapPair.key);
availabilityMap.dump(WTF::dataFile());
CRASH();
}
break;
default:
break;
}
}
}
calculator.executeNode(block->at(nodeIndex));
}
}
}
return true;
}
// We are creating a bunch of threads that touch the main thread's stack. This will make ASAN unhappy.
// The reason this is OK is that we guarantee that the main thread doesn't continue until all threads
// that could touch its stack are done executing.
SUPPRESS_ASAN
void Plan::run()
{
if (!parseAndValidateModule())
return;
auto tryReserveCapacity = [this] (auto& vector, size_t size, const char* what) {
if (UNLIKELY(!vector.tryReserveCapacity(size))) {
StringBuilder builder;
builder.appendLiteral("Failed allocating enough space for ");
builder.appendNumber(size);
builder.append(what);
m_errorMessage = builder.toString();
return false;
}
return true;
};
if (!tryReserveCapacity(m_wasmExitStubs, m_moduleInformation->importFunctionSignatureIndices.size(), " WebAssembly to JavaScript stubs")
|| !tryReserveCapacity(m_unlinkedWasmToWasmCalls, m_functionLocationInBinary.size(), " unlinked WebAssembly to WebAssembly calls")
|| !tryReserveCapacity(m_wasmInternalFunctions, m_functionLocationInBinary.size(), " WebAssembly functions")
|| !tryReserveCapacity(m_compilationContexts, m_functionLocationInBinary.size(), " compilation contexts"))
return;
m_unlinkedWasmToWasmCalls.resize(m_functionLocationInBinary.size());
m_wasmInternalFunctions.resize(m_functionLocationInBinary.size());
m_compilationContexts.resize(m_functionLocationInBinary.size());
for (unsigned importIndex = 0; importIndex < m_moduleInformation->imports.size(); ++importIndex) {
Import* import = &m_moduleInformation->imports[importIndex];
if (import->kind != ExternalKind::Function)
continue;
unsigned importFunctionIndex = m_wasmExitStubs.size();
if (verbose)
dataLogLn("Processing import function number ", importFunctionIndex, ": ", import->module, ": ", import->field);
SignatureIndex signatureIndex = m_moduleInformation->importFunctionSignatureIndices.at(import->kindIndex);
m_wasmExitStubs.uncheckedAppend(exitStubGenerator(m_vm, m_callLinkInfos, signatureIndex, importFunctionIndex));
}
m_currentIndex = 0;
auto doWork = [this] {
while (true) {
uint32_t functionIndex;
{
auto locker = holdLock(m_lock);
if (m_currentIndex >= m_functionLocationInBinary.size())
return;
functionIndex = m_currentIndex;
++m_currentIndex;
}
const uint8_t* functionStart = m_source + m_functionLocationInBinary[functionIndex].start;
size_t functionLength = m_functionLocationInBinary[functionIndex].end - m_functionLocationInBinary[functionIndex].start;
ASSERT(functionLength <= m_sourceLength);
SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[functionIndex];
const Signature* signature = SignatureInformation::get(m_vm, signatureIndex);
unsigned functionIndexSpace = m_wasmExitStubs.size() + functionIndex;
ASSERT_UNUSED(functionIndexSpace, m_moduleInformation->signatureIndexFromFunctionIndexSpace(functionIndexSpace) == signatureIndex);
ASSERT(validateFunction(m_vm, functionStart, functionLength, signature, *m_moduleInformation, m_moduleSignatureIndicesToUniquedSignatureIndices));
m_unlinkedWasmToWasmCalls[functionIndex] = Vector<UnlinkedWasmToWasmCall>();
auto parseAndCompileResult = parseAndCompile(*m_vm, m_compilationContexts[functionIndex], functionStart, functionLength, signature, m_unlinkedWasmToWasmCalls[functionIndex], *m_moduleInformation, m_moduleSignatureIndicesToUniquedSignatureIndices);
if (UNLIKELY(!parseAndCompileResult)) {
auto locker = holdLock(m_lock);
if (!m_errorMessage) {
// Multiple compiles could fail simultaneously. We arbitrarily choose the first.
m_errorMessage = makeString(parseAndCompileResult.error(), ", in function at index ", String::number(functionIndex)); // FIXME make this an Expected.
}
m_currentIndex = m_functionLocationInBinary.size();
// We will terminate on the next execution.
continue;
}
m_wasmInternalFunctions[functionIndex] = WTFMove(*parseAndCompileResult);
}
};
MonotonicTime startTime;
if (verbose || Options::reportCompileTimes())
startTime = MonotonicTime::now();
uint32_t threadCount = Options::useConcurrentJIT() ? WTF::numberOfProcessorCores() : 1;
uint32_t numWorkerThreads = threadCount - 1;
Vector<ThreadIdentifier> threads;
threads.reserveCapacity(numWorkerThreads);
for (uint32_t i = 0; i < numWorkerThreads; i++)
threads.uncheckedAppend(createThread("jsc.wasm-b3-compilation.thread", doWork));
doWork(); // Let the main thread do some work too.
for (uint32_t i = 0; i < numWorkerThreads; i++)
waitForThreadCompletion(threads[i]);
for (uint32_t functionIndex = 0; functionIndex < m_functionLocationInBinary.size(); functionIndex++) {
{
CompilationContext& context = m_compilationContexts[functionIndex];
SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[functionIndex];
String signatureDescription = SignatureInformation::get(m_vm, signatureIndex)->toString();
{
LinkBuffer linkBuffer(*m_vm, *context.wasmEntrypointJIT, nullptr);
m_wasmInternalFunctions[functionIndex]->wasmEntrypoint.compilation =
std::make_unique<B3::Compilation>(FINALIZE_CODE(linkBuffer, ("WebAssembly function[%i] %s", functionIndex, signatureDescription.ascii().data())), WTFMove(context.wasmEntrypointByproducts));
}
{
LinkBuffer linkBuffer(*m_vm, *context.jsEntrypointJIT, nullptr);
linkBuffer.link(context.jsEntrypointToWasmEntrypointCall, FunctionPtr(m_wasmInternalFunctions[functionIndex]->wasmEntrypoint.compilation->code().executableAddress()));
m_wasmInternalFunctions[functionIndex]->jsToWasmEntrypoint.compilation =
std::make_unique<B3::Compilation>(FINALIZE_CODE(linkBuffer, ("JavaScript->WebAssembly entrypoint[%i] %s", functionIndex, signatureDescription.ascii().data())), WTFMove(context.jsEntrypointByproducts));
}
}
}
if (verbose || Options::reportCompileTimes()) {
dataLogLn("Took ", (MonotonicTime::now() - startTime).microseconds(),
" us to compile and link the module");
}
// Patch the call sites for each WebAssembly function.
for (auto& unlinked : m_unlinkedWasmToWasmCalls) {
for (auto& call : unlinked) {
void* executableAddress;
if (m_moduleInformation->isImportedFunctionFromFunctionIndexSpace(call.functionIndex)) {
// FIXME imports could have been linked in B3, instead of generating a patchpoint. This condition should be replaced by a RELEASE_ASSERT. https://bugs.webkit.org/show_bug.cgi?id=166462
executableAddress = call.target == UnlinkedWasmToWasmCall::Target::ToJs
? m_wasmExitStubs.at(call.functionIndex).wasmToJs.code().executableAddress()
: m_wasmExitStubs.at(call.functionIndex).wasmToWasm.code().executableAddress();
} else {
ASSERT(call.target != UnlinkedWasmToWasmCall::Target::ToJs);
executableAddress = m_wasmInternalFunctions.at(call.functionIndex - m_wasmExitStubs.size())->wasmEntrypoint.compilation->code().executableAddress();
}
MacroAssembler::repatchCall(call.callLocation, CodeLocationLabel(executableAddress));
}
}
m_failed = false;
}
