Error JitRuntime::_add(void** dst, CodeHolder* code) noexcept {
size_t codeSize = code->getCodeSize();
if (ASMJIT_UNLIKELY(codeSize == 0)) {
*dst = nullptr;
return DebugUtils::errored(kErrorNoCodeGenerated);
}
void* p = _memMgr.alloc(codeSize, getAllocType());
if (ASMJIT_UNLIKELY(!p)) {
*dst = nullptr;
return DebugUtils::errored(kErrorNoVirtualMemory);
}
// Relocate the code and release the unused memory back to `VMemMgr`.
size_t relocSize = code->relocate(p);
if (ASMJIT_UNLIKELY(relocSize == 0)) {
*dst = nullptr;
_memMgr.release(p);
return DebugUtils::errored(kErrorInvalidState);
}
if (relocSize < codeSize)
_memMgr.shrink(p, relocSize);
flush(p, relocSize);
*dst = p;
return kErrorOk;
}
VarData* Compiler::_newVd(const VarInfo& vi, const char* name) noexcept {
VarData* vd = reinterpret_cast<VarData*>(_varAllocator.alloc(sizeof(VarData)));
if (ASMJIT_UNLIKELY(vd == nullptr))
goto _NoMemory;
vd->_name = noName;
vd->_id = OperandUtil::makeVarId(static_cast<uint32_t>(_varList.getLength()));
vd->_localId = kInvalidValue;
#if !defined(ASMJIT_DISABLE_LOGGER)
if (name != nullptr && name[0] != '\0') {
vd->_name = _stringAllocator.sdup(name);
}
#endif // !ASMJIT_DISABLE_LOGGER
vd->_type = static_cast<uint8_t>(vi.getTypeId());
vd->_class = static_cast<uint8_t>(vi.getRegClass());
vd->_flags = 0;
vd->_priority = 10;
vd->_state = kVarStateNone;
vd->_regIndex = kInvalidReg;
vd->_isStack = false;
vd->_isMemArg = false;
vd->_isCalculated = false;
vd->_saveOnUnuse = false;
vd->_modified = false;
vd->_reserved0 = 0;
vd->_alignment = static_cast<uint8_t>(Utils::iMin<uint32_t>(vi.getSize(), 64));
vd->_size = vi.getSize();
vd->_homeMask = 0;
vd->_memOffset = 0;
vd->_memCell = nullptr;
vd->rReadCount = 0;
vd->rWriteCount = 0;
vd->mReadCount = 0;
vd->mWriteCount = 0;
vd->_va = nullptr;
if (ASMJIT_UNLIKELY(_varList.append(vd) != kErrorOk))
goto _NoMemory;
return vd;
_NoMemory:
setLastError(kErrorNoHeapMemory);
return nullptr;
}
ASMJIT_FAVOR_SIZE Error CodeBuilder::addPass(CBPass* pass) noexcept {
if (ASMJIT_UNLIKELY(pass == nullptr)) {
// Since this is directly called by `addPassT()` we treat `null` argument
// as out-of-memory condition. Otherwise it would be API misuse.
return DebugUtils::errored(kErrorNoHeapMemory);
}
else if (ASMJIT_UNLIKELY(pass->_cb)) {
// Kind of weird, but okay...
if (pass->_cb == this)
return kErrorOk;
return DebugUtils::errored(kErrorInvalidState);
}
ASMJIT_PROPAGATE(_cbPasses.append(pass));
pass->_cb = this;
return kErrorOk;
}
Error CodeBuilder::comment(const char* s, size_t len) {
if (_lastError) return _lastError;
CBComment* node = newCommentNode(s, len);
if (ASMJIT_UNLIKELY(!node))
return setLastError(DebugUtils::errored(kErrorNoHeapMemory));
addNode(node);
return kErrorOk;
}
Error CodeBuilder::embedLabel(const Label& label) {
if (_lastError) return _lastError;
CBLabelData* node = newNodeT<CBLabelData>(label.getId());
if (ASMJIT_UNLIKELY(!node))
return setLastError(DebugUtils::errored(kErrorNoHeapMemory));
addNode(node);
return kErrorOk;
}
Error CodeBuilder::embed(const void* data, uint32_t size) {
if (_lastError) return _lastError;
CBData* node = newDataNode(data, size);
if (ASMJIT_UNLIKELY(!node))
return setLastError(DebugUtils::errored(kErrorNoHeapMemory));
addNode(node);
return kErrorOk;
}
Error CodeBuilder::align(uint32_t mode, uint32_t alignment) {
if (_lastError) return _lastError;
CBAlign* node = newAlignNode(mode, alignment);
if (ASMJIT_UNLIKELY(!node))
return setLastError(DebugUtils::errored(kErrorNoHeapMemory));
addNode(node);
return kErrorOk;
}
Label CodeBuilder::newNamedLabel(const char* name, size_t nameLength, uint32_t type, uint32_t parentId) {
uint32_t id = kInvalidValue;
if (!_lastError) {
CBLabel* node = newNodeT<CBLabel>(id);
if (ASMJIT_UNLIKELY(!node)) {
setLastError(DebugUtils::errored(kErrorNoHeapMemory));
}
else {
Error err = _code->newNamedLabelId(id, name, nameLength, type, parentId);
if (ASMJIT_UNLIKELY(err))
setLastError(err);
else
id = node->getId();
}
}
return Label(id);
}
Label CodeBuilder::newLabel() {
uint32_t id = kInvalidValue;
if (!_lastError) {
CBLabel* node = newNodeT<CBLabel>(id);
if (ASMJIT_UNLIKELY(!node)) {
setLastError(DebugUtils::errored(kErrorNoHeapMemory));
}
else {
Error err = registerLabelNode(node);
if (ASMJIT_UNLIKELY(err))
setLastError(err);
else
id = node->getId();
}
}
return Label(id);
}
Error CodeBuilder::bind(const Label& label) {
if (_lastError) return _lastError;
CBLabel* node;
Error err = getCBLabel(&node, label);
if (ASMJIT_UNLIKELY(err))
return setLastError(err);
addNode(node);
return kErrorOk;
}
Error PodVectorBase::_reserve(size_t n, size_t sizeOfT) noexcept {
Data* d = _d;
if (d->capacity >= n)
return kErrorOk;
size_t nBytes = sizeof(Data) + n * sizeOfT;
if (ASMJIT_UNLIKELY(nBytes < n))
return kErrorNoHeapMemory;
if (d == &_nullData) {
d = static_cast<Data*>(ASMJIT_ALLOC(nBytes));
if (ASMJIT_UNLIKELY(d == nullptr))
return kErrorNoHeapMemory;
d->length = 0;
}
else {
if (isDataStatic(this, d)) {
Data* oldD = d;
d = static_cast<Data*>(ASMJIT_ALLOC(nBytes));
if (ASMJIT_UNLIKELY(d == nullptr))
return kErrorNoHeapMemory;
size_t len = oldD->length;
d->length = len;
::memcpy(d->getData(), oldD->getData(), len * sizeOfT);
}
else {
d = static_cast<Data*>(ASMJIT_REALLOC(d, nBytes));
if (ASMJIT_UNLIKELY(d == nullptr))
return kErrorNoHeapMemory;
}
}
d->capacity = n;
_d = d;
return kErrorOk;
}
Error CodeBuilder::getCBLabel(CBLabel** pOut, uint32_t id) noexcept {
if (_lastError) return _lastError;
ASMJIT_ASSERT(_code != nullptr);
size_t index = Operand::unpackId(id);
if (ASMJIT_UNLIKELY(index >= _code->getLabelsCount()))
return DebugUtils::errored(kErrorInvalidLabel);
if (index >= _cbLabels.getLength())
ASMJIT_PROPAGATE(_cbLabels.resize(index + 1));
CBLabel* node = _cbLabels[index];
if (!node) {
node = newNodeT<CBLabel>(id);
if (ASMJIT_UNLIKELY(!node))
return DebugUtils::errored(kErrorNoHeapMemory);
_cbLabels[index] = node;
}
*pOut = node;
return kErrorOk;
}
Error CodeBuilder::embedConstPool(const Label& label, const ConstPool& pool) {
if (_lastError) return _lastError;
if (!isLabelValid(label))
return setLastError(DebugUtils::errored(kErrorInvalidLabel));
ASMJIT_PROPAGATE(align(kAlignData, static_cast<uint32_t>(pool.getAlignment())));
ASMJIT_PROPAGATE(bind(label));
CBData* node = newDataNode(nullptr, static_cast<uint32_t>(pool.getSize()));
if (ASMJIT_UNLIKELY(!node))
return setLastError(DebugUtils::errored(kErrorNoHeapMemory));
pool.fill(node->getData());
addNode(node);
return kErrorOk;
}
ASMJIT_FAVOR_SIZE Error CodeBuilder::deletePass(CBPass* pass) noexcept {
if (ASMJIT_UNLIKELY(pass == nullptr))
return DebugUtils::errored(kErrorInvalidArgument);
if (pass->_cb != nullptr) {
if (pass->_cb != this)
return DebugUtils::errored(kErrorInvalidState);
size_t index = _cbPasses.indexOf(pass);
ASMJIT_ASSERT(index != kInvalidIndex);
pass->_cb = nullptr;
_cbPasses.removeAt(index);
}
pass->~CBPass();
return kErrorOk;
}
ASMJIT_FAVOR_SIZE Error FuncDetail::init(const FuncSignature& sign) {
uint32_t ccId = sign.getCallConv();
CallConv& cc = _callConv;
uint32_t argCount = sign.getArgCount();
if (ASMJIT_UNLIKELY(argCount > kFuncArgCount))
return DebugUtils::errored(kErrorInvalidArgument);
ASMJIT_PROPAGATE(cc.init(ccId));
uint32_t gpSize = (cc.getArchType() == ArchInfo::kTypeX86) ? 4 : 8;
uint32_t deabstractDelta = TypeId::deabstractDeltaOfSize(gpSize);
const uint8_t* args = sign.getArgs();
for (uint32_t i = 0; i < argCount; i++) {
Value& arg = _args[i];
arg.initTypeId(TypeId::deabstract(args[i], deabstractDelta));
}
_argCount = static_cast<uint8_t>(argCount);
uint32_t ret = sign.getRet();
if (ret != TypeId::kVoid) {
_rets[0].initTypeId(TypeId::deabstract(ret, deabstractDelta));
_retCount = 1;
}
#if defined(ASMJIT_BUILD_X86)
if (CallConv::isX86Family(ccId))
return X86Internal::initFuncDetail(*this, sign, gpSize);
#endif // ASMJIT_BUILD_X86
#if defined(ASMJIT_BUILD_ARM)
if (CallConv::isArmFamily(ccId))
return ArmInternal::initFuncDetail(*this, sign, gpSize);
#endif // ASMJIT_BUILD_ARM
// We should never bubble here as if `cc.init()` succeeded then there has to
// be an implementation for the current architecture. However, stay safe.
return DebugUtils::errored(kErrorInvalidArgument);
}
ASMJIT_FAVOR_SIZE Error ArchUtils::typeIdToRegInfo(uint32_t archType, uint32_t& typeIdInOut, RegInfo& regInfo) noexcept {
uint32_t typeId = typeIdInOut;
// Zero the signature so it's clear in case that typeId is not invalid.
regInfo._signature = 0;
#if defined(ASMJIT_BUILD_X86)
if (ArchInfo::isX86Family(archType)) {
// Passed RegType instead of TypeId?
if (typeId <= Reg::kRegMax)
typeId = x86OpData.archRegs.regTypeToTypeId[typeId];
if (ASMJIT_UNLIKELY(!TypeId::isValid(typeId)))
return DebugUtils::errored(kErrorInvalidTypeId);
// First normalize architecture dependent types.
if (TypeId::isAbstract(typeId)) {
if (typeId == TypeId::kIntPtr)
typeId = (archType == ArchInfo::kTypeX86) ? TypeId::kI32 : TypeId::kI64;
else
typeId = (archType == ArchInfo::kTypeX86) ? TypeId::kU32 : TypeId::kU64;
}
// Type size helps to construct all kinds of registers. If the size is zero
// then the TypeId is invalid.
uint32_t size = TypeId::sizeOf(typeId);
if (ASMJIT_UNLIKELY(!size))
return DebugUtils::errored(kErrorInvalidTypeId);
if (ASMJIT_UNLIKELY(typeId == TypeId::kF80))
return DebugUtils::errored(kErrorInvalidUseOfF80);
uint32_t regType = 0;
switch (typeId) {
case TypeId::kI8:
case TypeId::kU8:
regType = X86Reg::kRegGpbLo;
break;
case TypeId::kI16:
case TypeId::kU16:
regType = X86Reg::kRegGpw;
break;
case TypeId::kI32:
case TypeId::kU32:
regType = X86Reg::kRegGpd;
break;
case TypeId::kI64:
case TypeId::kU64:
if (archType == ArchInfo::kTypeX86)
return DebugUtils::errored(kErrorInvalidUseOfGpq);
regType = X86Reg::kRegGpq;
break;
// F32 and F64 are always promoted to use vector registers.
case TypeId::kF32:
typeId = TypeId::kF32x1;
regType = X86Reg::kRegXmm;
break;
case TypeId::kF64:
typeId = TypeId::kF64x1;
regType = X86Reg::kRegXmm;
break;
// Mask registers {k}.
case TypeId::kMask8:
case TypeId::kMask16:
case TypeId::kMask32:
case TypeId::kMask64:
regType = X86Reg::kRegK;
break;
// MMX registers.
case TypeId::kMmx32:
case TypeId::kMmx64:
regType = X86Reg::kRegMm;
break;
// XMM|YMM|ZMM registers.
default:
if (size <= 16)
regType = X86Reg::kRegXmm;
else if (size == 32)
regType = X86Reg::kRegYmm;
else
regType = X86Reg::kRegZmm;
break;
}
typeIdInOut = typeId;
regInfo._signature = x86OpData.archRegs.regInfo[regType].getSignature();
return kErrorOk;
}
#endif // ASMJIT_BUILD_X86
return DebugUtils::errored(kErrorInvalidArch);
}
