static void
SpecializeToMemory(CodeSegment& cs, const Metadata& metadata, HandleWasmMemoryObject memory)
{
for (const BoundsCheck& check : metadata.boundsChecks)
Assembler::UpdateBoundsCheck(check.patchAt(cs.code()), memory->buffer().byteLength());
#if defined(JS_CODEGEN_X86)
uint8_t* base = memory->buffer().dataPointerEither().unwrap();
for (const MemoryAccess& access : metadata.memoryAccesses) {
// Patch memory pointer immediate.
void* addr = access.patchMemoryPtrImmAt(cs.code());
uint32_t disp = reinterpret_cast<uint32_t>(X86Encoding::GetPointer(addr));
MOZ_ASSERT(disp <= INT32_MAX);
X86Encoding::SetPointer(addr, (void*)(base + disp));
}
#endif
}
/* static */ UniqueCodeSegment
CodeSegment::create(JSContext* cx,
const Bytes& bytecode,
const LinkData& linkData,
const Metadata& metadata,
HandleWasmMemoryObject memory)
{
MOZ_ASSERT(bytecode.length() % gc::SystemPageSize() == 0);
MOZ_ASSERT(linkData.globalDataLength % gc::SystemPageSize() == 0);
MOZ_ASSERT(linkData.functionCodeLength < bytecode.length());
auto cs = cx->make_unique<CodeSegment>();
if (!cs)
return nullptr;
cs->bytes_ = AllocateCodeSegment(cx, bytecode.length() + linkData.globalDataLength);
if (!cs->bytes_)
return nullptr;
uint8_t* codeBase = cs->base();
cs->functionCodeLength_ = linkData.functionCodeLength;
cs->codeLength_ = bytecode.length();
cs->globalDataLength_ = linkData.globalDataLength;
cs->interruptCode_ = codeBase + linkData.interruptOffset;
cs->outOfBoundsCode_ = codeBase + linkData.outOfBoundsOffset;
cs->unalignedAccessCode_ = codeBase + linkData.unalignedAccessOffset;
{
JitContext jcx(CompileRuntime::get(cx->compartment()->runtimeFromAnyThread()));
AutoFlushICache afc("CodeSegment::create");
AutoFlushICache::setRange(uintptr_t(codeBase), cs->codeLength());
memcpy(codeBase, bytecode.begin(), bytecode.length());
StaticallyLink(*cs, linkData, cx);
if (memory)
SpecializeToMemory(nullptr, *cs, metadata, memory->buffer());
}
if (!ExecutableAllocator::makeExecutable(codeBase, cs->codeLength())) {
ReportOutOfMemory(cx);
return nullptr;
}
if (!SendCodeRangesToProfiler(cx, *cs, bytecode, metadata))
return nullptr;
return cs;
}
static bool
CreateExportObject(JSContext* cx,
HandleWasmInstanceObject instanceObj,
MutableHandleWasmTableObject tableObj,
HandleWasmMemoryObject memoryObj,
const ValVector& globalImports,
const ExportVector& exports,
MutableHandleObject exportObj)
{
const Instance& instance = instanceObj->instance();
const Metadata& metadata = instance.metadata();
if (metadata.isAsmJS() && exports.length() == 1 && strlen(exports[0].fieldName()) == 0) {
RootedFunction fun(cx);
if (!instanceObj->getExportedFunction(cx, instanceObj, exports[0].funcIndex(), &fun))
return false;
exportObj.set(fun);
return true;
}
exportObj.set(JS_NewPlainObject(cx));
if (!exportObj)
return false;
for (const Export& exp : exports) {
JSAtom* atom = AtomizeUTF8Chars(cx, exp.fieldName(), strlen(exp.fieldName()));
if (!atom)
return false;
RootedId id(cx, AtomToId(atom));
RootedValue val(cx);
switch (exp.kind()) {
case DefinitionKind::Function: {
RootedFunction fun(cx);
if (!instanceObj->getExportedFunction(cx, instanceObj, exp.funcIndex(), &fun))
return false;
val = ObjectValue(*fun);
break;
}
case DefinitionKind::Table: {
if (!tableObj) {
MOZ_ASSERT(instance.tables().length() == 1);
tableObj.set(WasmTableObject::create(cx, *instance.tables()[0]));
if (!tableObj)
return false;
}
val = ObjectValue(*tableObj);
break;
}
case DefinitionKind::Memory: {
if (metadata.assumptions.newFormat)
val = ObjectValue(*memoryObj);
else
val = ObjectValue(memoryObj->buffer());
break;
}
case DefinitionKind::Global: {
if (!ExportGlobalValue(cx, metadata.globals, exp.globalIndex(), globalImports, &val))
return false;
break;
}
}
if (!JS_DefinePropertyById(cx, exportObj, id, val, JSPROP_ENUMERATE))
return false;
}
return true;
}
Instance::Instance(JSContext* cx,
Handle<WasmInstanceObject*> object,
UniqueCode code,
HandleWasmMemoryObject memory,
SharedTableVector&& tables,
Handle<FunctionVector> funcImports,
const ValVector& globalImports)
: compartment_(cx->compartment()),
object_(object),
code_(Move(code)),
memory_(memory),
tables_(Move(tables))
{
MOZ_ASSERT(funcImports.length() == metadata().funcImports.length());
MOZ_ASSERT(tables_.length() == metadata().tables.length());
tlsData_.cx = cx;
tlsData_.instance = this;
tlsData_.globalData = code_->segment().globalData();
tlsData_.memoryBase = memory ? memory->buffer().dataPointerEither().unwrap() : nullptr;
tlsData_.stackLimit = *(void**)cx->stackLimitAddressForJitCode(StackForUntrustedScript);
for (size_t i = 0; i < metadata().funcImports.length(); i++) {
HandleFunction f = funcImports[i];
const FuncImport& fi = metadata().funcImports[i];
FuncImportTls& import = funcImportTls(fi);
if (IsExportedFunction(f) && !isAsmJS() && !ExportedFunctionToInstance(f).isAsmJS()) {
Instance& calleeInstance = ExportedFunctionToInstance(f);
const Metadata& calleeMetadata = calleeInstance.metadata();
uint32_t funcIndex = ExportedFunctionToIndex(f);
const FuncExport& funcExport = calleeMetadata.lookupFuncExport(funcIndex);
const CodeRange& codeRange = calleeMetadata.codeRanges[funcExport.codeRangeIndex()];
import.tls = &calleeInstance.tlsData_;
import.code = calleeInstance.codeSegment().base() + codeRange.funcNonProfilingEntry();
import.baselineScript = nullptr;
import.obj = ExportedFunctionToInstanceObject(f);
} else {
import.tls = &tlsData_;
import.code = codeBase() + fi.interpExitCodeOffset();
import.baselineScript = nullptr;
import.obj = f;
}
}
uint8_t* globalData = code_->segment().globalData();
for (size_t i = 0; i < metadata().globals.length(); i++) {
const GlobalDesc& global = metadata().globals[i];
if (global.isConstant())
continue;
uint8_t* globalAddr = globalData + global.offset();
switch (global.kind()) {
case GlobalKind::Import: {
globalImports[global.importIndex()].writePayload(globalAddr);
break;
}
case GlobalKind::Variable: {
const InitExpr& init = global.initExpr();
switch (init.kind()) {
case InitExpr::Kind::Constant: {
init.val().writePayload(globalAddr);
break;
}
case InitExpr::Kind::GetGlobal: {
const GlobalDesc& imported = metadata().globals[init.globalIndex()];
globalImports[imported.importIndex()].writePayload(globalAddr);
break;
}
}
break;
}
case GlobalKind::Constant: {
MOZ_CRASH("skipped at the top");
}
}
}
for (size_t i = 0; i < tables_.length(); i++)
*addressOfTableBase(i) = tables_[i]->base();
}
static bool
CreateExportObject(JSContext* cx,
HandleWasmInstanceObject instanceObj,
HandleWasmMemoryObject memoryObj,
const ExportVector& exports,
MutableHandleObject exportObj)
{
const Instance& instance = instanceObj->instance();
const Metadata& metadata = instance.metadata();
const SharedTableVector& tables = instance.tables();
if (metadata.isAsmJS() && exports.length() == 1 && strlen(exports[0].fieldName()) == 0) {
exportObj.set(NewExportedFunction(cx, instanceObj, 0));
return !!exportObj;
}
exportObj.set(JS_NewPlainObject(cx));
if (!exportObj)
return false;
Rooted<ValueVector> vals(cx, ValueVector(cx));
for (size_t i = 0; i < metadata.funcExports.length(); i++) {
JSFunction* fun = NewExportedFunction(cx, instanceObj, i);
if (!fun || !vals.append(ObjectValue(*fun)))
return false;
}
RootedWasmTableObject tableObj(cx);
for (const Export& exp : exports) {
JSAtom* atom = AtomizeUTF8Chars(cx, exp.fieldName(), strlen(exp.fieldName()));
if (!atom)
return false;
RootedId id(cx, AtomToId(atom));
RootedValue val(cx);
switch (exp.kind()) {
case DefinitionKind::Function:
val = vals[exp.funcExportIndex()];
break;
case DefinitionKind::Table:
MOZ_ASSERT(tables.length() == 1);
if (!tableObj) {
tableObj = WasmTableObject::create(cx, *tables[0]);
if (!tableObj)
return false;
}
val = ObjectValue(*tableObj);
break;
case DefinitionKind::Memory:
if (metadata.assumptions.newFormat)
val = ObjectValue(*memoryObj);
else
val = ObjectValue(memoryObj->buffer());
break;
}
if (!JS_DefinePropertyById(cx, exportObj, id, val, JSPROP_ENUMERATE))
return false;
}
return true;
}
bool
Module::initSegments(JSContext* cx,
HandleWasmInstanceObject instanceObj,
Handle<FunctionVector> funcImports,
HandleWasmMemoryObject memoryObj,
const ValVector& globalImports) const
{
Instance& instance = instanceObj->instance();
const SharedTableVector& tables = instance.tables();
// Perform all error checks up front so that this function does not perform
// partial initialization if an error is reported.
for (const ElemSegment& seg : elemSegments_) {
uint32_t numElems = seg.elemCodeRangeIndices.length();
if (!numElems)
continue;
uint32_t tableLength = tables[seg.tableIndex]->length();
uint32_t offset = EvaluateInitExpr(globalImports, seg.offset);
if (offset > tableLength || tableLength - offset < numElems) {
JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, JSMSG_WASM_BAD_FIT,
"elem", "table");
return false;
}
}
if (memoryObj) {
for (const DataSegment& seg : dataSegments_) {
if (!seg.length)
continue;
uint32_t memoryLength = memoryObj->buffer().byteLength();
uint32_t offset = EvaluateInitExpr(globalImports, seg.offset);
if (offset > memoryLength || memoryLength - offset < seg.length) {
JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, JSMSG_WASM_BAD_FIT,
"data", "memory");
return false;
}
}
} else {
MOZ_ASSERT(dataSegments_.empty());
}
// Now that initialization can't fail partway through, write data/elem
// segments into memories/tables.
for (const ElemSegment& seg : elemSegments_) {
Table& table = *tables[seg.tableIndex];
uint32_t offset = EvaluateInitExpr(globalImports, seg.offset);
bool profilingEnabled = instance.code().profilingEnabled();
const CodeRangeVector& codeRanges = metadata().codeRanges;
uint8_t* codeBase = instance.codeBase();
for (uint32_t i = 0; i < seg.elemCodeRangeIndices.length(); i++) {
uint32_t funcIndex = seg.elemFuncIndices[i];
if (funcIndex < funcImports.length() && IsExportedWasmFunction(funcImports[funcIndex])) {
MOZ_ASSERT(!metadata().isAsmJS());
MOZ_ASSERT(!table.isTypedFunction());
HandleFunction f = funcImports[funcIndex];
WasmInstanceObject* exportInstanceObj = ExportedFunctionToInstanceObject(f);
const CodeRange& cr = exportInstanceObj->getExportedFunctionCodeRange(f);
Instance& exportInstance = exportInstanceObj->instance();
table.set(offset + i, exportInstance.codeBase() + cr.funcTableEntry(), exportInstance);
} else {
const CodeRange& cr = codeRanges[seg.elemCodeRangeIndices[i]];
uint32_t entryOffset = table.isTypedFunction()
? profilingEnabled
? cr.funcProfilingEntry()
: cr.funcNonProfilingEntry()
: cr.funcTableEntry();
table.set(offset + i, codeBase + entryOffset, instance);
}
}
}
if (memoryObj) {
uint8_t* memoryBase = memoryObj->buffer().dataPointerEither().unwrap(/* memcpy */);
for (const DataSegment& seg : dataSegments_) {
MOZ_ASSERT(seg.bytecodeOffset <= bytecode_->length());
MOZ_ASSERT(seg.length <= bytecode_->length() - seg.bytecodeOffset);
uint32_t offset = EvaluateInitExpr(globalImports, seg.offset);
memcpy(memoryBase + offset, bytecode_->begin() + seg.bytecodeOffset, seg.length);
}
}
return true;
}