extern "C" void PyString_InternInPlace(PyObject** p) noexcept {
BoxedString* s = (BoxedString*)*p;
if (s == NULL || !PyString_Check(s))
Py_FatalError("PyString_InternInPlace: strings only please!");
/* If it's a string subclass, we don't really know what putting
it in the interned dict might do. */
if (!PyString_CheckExact(s))
return;
if (PyString_CHECK_INTERNED(s))
return;
auto it = interned_strings.find(s);
if (it != interned_strings.end()) {
auto entry = *it;
Py_INCREF(entry);
Py_DECREF(*p);
*p = entry;
} else {
// TODO: do CPython's refcounting here
num_interned_strings.log();
interned_strings.insert(s);
Py_INCREF(s);
// CPython returns mortal but in our current implementation they are inmortal
s->interned_state = SSTATE_INTERNED_IMMORTAL;
}
}
uint32_t SymbolFilePDB::FindTypes(
const lldb_private::SymbolContext &sc,
const lldb_private::ConstString &name,
const lldb_private::CompilerDeclContext *parent_decl_ctx, bool append,
uint32_t max_matches,
llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
lldb_private::TypeMap &types) {
if (!append)
types.Clear();
if (!name)
return 0;
searched_symbol_files.clear();
searched_symbol_files.insert(this);
std::string name_str = name.AsCString();
// If this might be a regex, we have to return EVERY symbol and process them
// one by one, which is going
// to destroy performance on large PDB files. So try really hard not to use a
// regex match.
if (name_str.find_first_of("[]?*.-+\\") != std::string::npos)
FindTypesByRegex(name_str, max_matches, types);
else
FindTypesByName(name_str, max_matches, types);
return types.GetSize();
}
BoxedString* internStringImmortal(llvm::StringRef s) noexcept {
auto it = interned_strings.find_as(s);
if (it != interned_strings.end())
return incref(*it);
num_interned_strings.log();
BoxedString* entry = boxString(s);
// CPython returns mortal but in our current implementation they are inmortal
entry->interned_state = SSTATE_INTERNED_IMMORTAL;
interned_strings.insert((BoxedString*)entry);
Py_INCREF(entry);
return entry;
}
extern "C" void _Py_ReleaseInternedStrings() noexcept {
// printf("%ld interned strings\n", interned_strings.size());
for (const auto& p : interned_strings) {
Py_DECREF(p);
}
interned_strings.clear();
}
void addVarDeclsVisible(clang::ForStmt const *Parent,
clang::Decl const *PriorToDecl,
clang::Stmt const *PriorToStmt,
seec::seec_clang::MappedAST const &Map,
llvm::DenseSet<clang::VarDecl const *> &Set)
{
// The initialisation statement.
if (auto const Init = Parent->getInit()) {
if (PriorToStmt && Init == PriorToStmt)
return;
if (auto const DeclStmt = llvm::dyn_cast<clang::DeclStmt>(Init))
addVarDeclsVisible(DeclStmt, nullptr, nullptr, Map, Set);
}
// The condition statement.
if (PriorToStmt && Parent->getCond() == PriorToStmt)
return;
if (auto const CV = Parent->getConditionVariable())
Set.insert(CV);
// The increment statement.
if (PriorToStmt && Parent->getInc() == PriorToStmt)
return;
// Any VarDecls in the Body should have already been added.
}
ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, StackInfo stack_info, int num_slots,
int slot_size, llvm::CallingConv::ID calling_conv, LiveOutSet _live_outs,
assembler::GenericRegister return_register, TypeRecorder* type_recorder)
: next_slot_to_try(0),
stack_info(stack_info),
num_slots(num_slots),
slot_size(slot_size),
calling_conv(calling_conv),
live_outs(std::move(_live_outs)),
return_register(return_register),
type_recorder(type_recorder),
retry_in(0),
retry_backoff(1),
times_rewritten(0),
start_addr(start_addr),
slowpath_rtn_addr(slowpath_rtn_addr),
continue_addr(continue_addr) {
for (int i = 0; i < num_slots; i++) {
slots.emplace_back(this, i);
}
#if MOVING_GC
assert(ics_list.count(this) == 0);
#endif
}
void addSymbol(const CVSymbol &Symbol) {
if (Symbol.kind() == S_UDT) {
auto Iter = UdtHashes.insert(Symbol);
if (!Iter.second)
return;
}
Records.push_back(Symbol);
}
static void
addAllChildren(llvm::DenseSet<clang::Stmt const *> &Set, clang::Stmt const *S)
{
for (auto const &Child : S->children()) {
if (Child) {
Set.insert(Child);
addAllChildren(Set, Child);
}
}
}
void visitModuleFile(StringRef Filename,
serialization::ModuleKind Kind) override {
auto *File = CI.getFileManager().getFile(Filename);
assert(File && "missing file for loaded module?");
// Only rewrite each module file once.
if (!Rewritten.insert(File).second)
return;
serialization::ModuleFile *MF =
CI.getModuleManager()->getModuleManager().lookup(File);
assert(File && "missing module file for loaded module?");
// Not interested in PCH / preambles.
if (!MF->isModule())
return;
auto OS = Out.lock();
assert(OS && "loaded module file after finishing rewrite action?");
(*OS) << "#pragma clang module build ";
if (isValidIdentifier(MF->ModuleName))
(*OS) << MF->ModuleName;
else {
(*OS) << '"';
OS->write_escaped(MF->ModuleName);
(*OS) << '"';
}
(*OS) << '\n';
// Rewrite the contents of the module in a separate compiler instance.
CompilerInstance Instance(CI.getPCHContainerOperations(),
&CI.getPreprocessor().getPCMCache());
Instance.setInvocation(
std::make_shared<CompilerInvocation>(CI.getInvocation()));
Instance.createDiagnostics(
new ForwardingDiagnosticConsumer(CI.getDiagnosticClient()),
/*ShouldOwnClient=*/true);
Instance.getFrontendOpts().DisableFree = false;
Instance.getFrontendOpts().Inputs.clear();
Instance.getFrontendOpts().Inputs.emplace_back(
Filename, InputKind(InputKind::Unknown, InputKind::Precompiled));
Instance.getFrontendOpts().ModuleFiles.clear();
Instance.getFrontendOpts().ModuleMapFiles.clear();
// Don't recursively rewrite imports. We handle them all at the top level.
Instance.getPreprocessorOutputOpts().RewriteImports = false;
llvm::CrashRecoveryContext().RunSafelyOnThread([&]() {
RewriteIncludesAction Action;
Action.OutputStream = OS;
Instance.ExecuteAction(Action);
});
(*OS) << "#pragma clang module endbuild /*" << MF->ModuleName << "*/\n";
}
void Fix(CompoundStmt* CS) {
if (!CS->size())
return;
typedef llvm::SmallVector<Stmt*, 32> Statements;
Statements Stmts;
Stmts.append(CS->body_begin(), CS->body_end());
for (Statements::iterator I = Stmts.begin(); I != Stmts.end(); ++I) {
if (!TraverseStmt(*I) && !m_HandledDecls.count(m_FoundDRE->getDecl())) {
Sema::DeclGroupPtrTy VDPtrTy
= m_Sema->ConvertDeclToDeclGroup(m_FoundDRE->getDecl());
StmtResult DS = m_Sema->ActOnDeclStmt(VDPtrTy,
m_FoundDRE->getLocStart(),
m_FoundDRE->getLocEnd());
assert(!DS.isInvalid() && "Invalid DeclStmt.");
I = Stmts.insert(I, DS.take());
m_HandledDecls.insert(m_FoundDRE->getDecl());
}
}
CS->setStmts(m_Sema->getASTContext(), Stmts.data(), Stmts.size());
}
void foundDecl(ValueDecl *D, DeclVisibilityKind Reason) override {
// If the declaration has an override, name lookup will also have found
// the overridden method. Skip this declaration, because we prefer the
// overridden method.
if (D->getOverriddenDecl())
return;
// Initializers cannot be found by dynamic lookup.
if (isa<ConstructorDecl>(D))
return;
// Check if we already reported a decl with the same signature.
if (auto *FD = dyn_cast<FuncDecl>(D)) {
assert(FD->getImplicitSelfDecl() && "should not find free functions");
(void)FD;
// Get the type without the first uncurry level with 'self'.
CanType T = D->getType()
->castTo<AnyFunctionType>()
->getResult()
->getCanonicalType();
auto Signature = std::make_pair(D->getName(), T);
if (!FunctionsReported.insert(Signature).second)
return;
} else if (isa<SubscriptDecl>(D)) {
auto Signature = D->getType()->getCanonicalType();
if (!SubscriptsReported.insert(Signature).second)
return;
} else if (isa<VarDecl>(D)) {
auto Signature =
std::make_pair(D->getName(), D->getType()->getCanonicalType());
if (!PropertiesReported.insert(Signature).second)
return;
} else {
llvm_unreachable("unhandled decl kind");
}
if (isDeclVisibleInLookupMode(D, LS, CurrDC, TypeResolver))
ChainedConsumer.foundDecl(D, DeclVisibilityKind::DynamicLookup);
}
bool
ConsumedResultToEpilogueRetainMatcher::
isTransitiveSuccessorsRetainFree(llvm::DenseSet<SILBasicBlock *> BBs) {
// For every block with retain, we need to check the transitive
// closure of its successors are retain-free.
for (auto &I : EpilogueRetainInsts) {
auto *CBB = I->getParent();
for (auto &Succ : CBB->getSuccessors()) {
if (BBs.find(Succ) != BBs.end())
continue;
return false;
}
}
for (auto CBB : BBs) {
for (auto &Succ : CBB->getSuccessors()) {
if (BBs.find(Succ) != BBs.end())
continue;
return false;
}
}
return true;
}
void addVarDeclsVisible(clang::DeclStmt const *Parent,
clang::Decl const *PriorToDecl,
clang::Stmt const *PriorToStmt,
seec::seec_clang::MappedAST const &Map,
llvm::DenseSet<clang::VarDecl const *> &Set)
{
if (Parent->isSingleDecl()) {
auto const Decl = Parent->getSingleDecl();
if (auto const VarDecl = llvm::dyn_cast<clang::VarDecl>(Decl))
Set.insert(VarDecl);
}
else {
for (auto const Decl : Parent->getDeclGroup()) {
if (auto const VarDecl = llvm::dyn_cast<clang::VarDecl>(Decl))
Set.insert(VarDecl);
if (PriorToDecl && Decl == PriorToDecl)
return;
}
}
}
void addVarDeclsVisible(clang::WhileStmt const *Parent,
clang::Decl const *PriorToDecl,
clang::Stmt const *PriorToStmt,
seec::seec_clang::MappedAST const &Map,
llvm::DenseSet<clang::VarDecl const *> &Set)
{
if (PriorToStmt && Parent->getCond() == PriorToStmt)
return;
if (auto const CV = Parent->getConditionVariable())
Set.insert(CV);
// Any VarDecls in the Body should have already been added.
}
bool ConsumedResultToEpilogueRetainMatcher::isTransitiveSuccessorsRetainFree(
const llvm::DenseSet<SILBasicBlock *> &BBs) {
// For every block with retain, we need to check the transitive
// closure of its successors are retain-free.
for (auto &I : EpilogueRetainInsts) {
for (auto &Succ : I->getParent()->getSuccessors()) {
if (BBs.count(Succ))
continue;
return false;
}
}
// FIXME: We are iterating over a DenseSet. That can lead to non-determinism
// and is in general pretty inefficient since we are iterating over a hash
// table.
for (auto CBB : BBs) {
for (auto &Succ : CBB->getSuccessors()) {
if (BBs.count(Succ))
continue;
return false;
}
}
return true;
}
/// Check whether all operands are loop invariant.
static bool hasLoopInvariantOperands(SILInstruction *I, SILLoop *L,
llvm::DenseSet<SILInstruction *> &Inv) {
auto Opds = I->getAllOperands();
return std::all_of(Opds.begin(), Opds.end(), [=](Operand &Op) {
ValueBase *Def = Op.get();
// Operand is outside the loop or marked invariant.
if (auto *Inst = Def->getDefiningInstruction())
return !L->contains(Inst->getParent()) || Inv.count(Inst);
if (auto *Arg = dyn_cast<SILArgument>(Def))
return !L->contains(Arg->getParent());
return false;
});
}
namespace pyston {
using namespace pyston::assembler;
#define MAX_RETRY_BACKOFF 1024
// TODO not right place for this...
int64_t ICInvalidator::version() {
return cur_version;
}
void ICInvalidator::addDependent(ICSlotInfo* entry_info) {
dependents.insert(entry_info);
}
void ICInvalidator::invalidateAll() {
cur_version++;
for (ICSlotInfo* slot : dependents) {
slot->clear();
}
dependents.clear();
}
void ICSlotInfo::clear() {
ic->clear(this);
}
ICSlotRewrite::ICSlotRewrite(ICInfo* ic, const char* debug_name)
: ic(ic), debug_name(debug_name), buf((uint8_t*)malloc(ic->getSlotSize())), assembler(buf, ic->getSlotSize()) {
assembler.nop();
if (VERBOSITY() >= 4)
printf("starting %s icentry\n", debug_name);
}
ICSlotRewrite::~ICSlotRewrite() {
free(buf);
}
void ICSlotRewrite::abort() {
ic->retry_backoff = std::min(MAX_RETRY_BACKOFF, 2 * ic->retry_backoff);
ic->retry_in = ic->retry_backoff;
}
ICSlotInfo* ICSlotRewrite::prepareEntry() {
this->ic_entry = ic->pickEntryForRewrite(debug_name);
return this->ic_entry;
}
uint8_t* ICSlotRewrite::getSlotStart() {
assert(ic_entry != NULL);
return (uint8_t*)ic->start_addr + ic_entry->idx * ic->getSlotSize();
}
// Map of gc pointers -> number of ics that point tot hem.
static llvm::DenseMap<void*, int> ic_gc_references;
void ICSlotRewrite::commit(CommitHook* hook, std::vector<void*> gc_references) {
bool still_valid = true;
for (int i = 0; i < dependencies.size(); i++) {
int orig_version = dependencies[i].second;
ICInvalidator* invalidator = dependencies[i].first;
if (orig_version != invalidator->version()) {
still_valid = false;
break;
}
}
if (!still_valid) {
if (VERBOSITY() >= 3)
printf("not committing %s icentry since a dependency got updated before commit\n", debug_name);
return;
}
uint8_t* slot_start = getSlotStart();
uint8_t* continue_point = (uint8_t*)ic->continue_addr;
bool do_commit = hook->finishAssembly(continue_point - slot_start);
if (!do_commit)
return;
assert(!assembler.hasFailed());
for (int i = 0; i < dependencies.size(); i++) {
ICInvalidator* invalidator = dependencies[i].first;
invalidator->addDependent(ic_entry);
}
ic->next_slot_to_try++;
// if (VERBOSITY()) printf("Commiting to %p-%p\n", start, start + ic->slot_size);
memcpy(slot_start, buf, ic->getSlotSize());
for (auto p : ic_entry->gc_references) {
int& i = ic_gc_references[p];
if (i == 1)
ic_gc_references.erase(p);
else
--i;
}
ic_entry->gc_references = std::move(gc_references);
for (auto p : ic_entry->gc_references)
ic_gc_references[p]++;
ic->times_rewritten++;
if (ic->times_rewritten == IC_MEGAMORPHIC_THRESHOLD) {
static StatCounter megamorphic_ics("megamorphic_ics");
megamorphic_ics.log();
}
llvm::sys::Memory::InvalidateInstructionCache(slot_start, ic->getSlotSize());
}
void ICSlotRewrite::gc_visit(GCVisitor* visitor) {
for (auto& dependency : dependencies) {
visitor->visitPotentialRedundant(dependency.first);
}
}
void ICSlotRewrite::addDependenceOn(ICInvalidator& invalidator) {
dependencies.push_back(std::make_pair(&invalidator, invalidator.version()));
}
int ICSlotRewrite::getSlotSize() {
return ic->getSlotSize();
}
int ICSlotRewrite::getScratchRspOffset() {
assert(ic->stack_info.scratch_size);
return ic->stack_info.scratch_rsp_offset;
}
int ICSlotRewrite::getScratchSize() {
return ic->stack_info.scratch_size;
}
TypeRecorder* ICSlotRewrite::getTypeRecorder() {
return ic->type_recorder;
}
assembler::GenericRegister ICSlotRewrite::returnRegister() {
return ic->return_register;
}
std::unique_ptr<ICSlotRewrite> ICInfo::startRewrite(const char* debug_name) {
return std::unique_ptr<ICSlotRewrite>(new ICSlotRewrite(this, debug_name));
}
ICSlotInfo* ICInfo::pickEntryForRewrite(const char* debug_name) {
int num_slots = getNumSlots();
for (int _i = 0; _i < num_slots; _i++) {
int i = (_i + next_slot_to_try) % num_slots;
ICSlotInfo& sinfo = slots[i];
assert(sinfo.num_inside >= 0);
if (sinfo.num_inside)
continue;
if (VERBOSITY() >= 4) {
printf("picking %s icentry to in-use slot %d at %p\n", debug_name, i, start_addr);
}
next_slot_to_try = i;
return &sinfo;
}
if (VERBOSITY() >= 4)
printf("not committing %s icentry since there are no available slots\n", debug_name);
return NULL;
}
// Keep track of all ICInfo(s) that we create because they contain pointers to Pyston heap objects
// that we have written into the generated code and we may need to scan those.
static llvm::DenseSet<ICInfo*> ics_list;
static llvm::DenseMap<void*, ICInfo*> ics_by_return_addr;
void registerGCTrackedICInfo(ICInfo* ic) {
#if MOVING_GC
assert(ics_list.count(ic) == 0);
ics_list.insert(ic);
#endif
}
void deregisterGCTrackedICInfo(ICInfo* ic) {
#if MOVING_GC
assert(ics_list.count(ic) == 1);
ics_list.erase(ic);
#endif
}
ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, StackInfo stack_info, int num_slots,
int slot_size, llvm::CallingConv::ID calling_conv, LiveOutSet _live_outs,
assembler::GenericRegister return_register, TypeRecorder* type_recorder)
: next_slot_to_try(0),
stack_info(stack_info),
num_slots(num_slots),
slot_size(slot_size),
calling_conv(calling_conv),
live_outs(std::move(_live_outs)),
return_register(return_register),
type_recorder(type_recorder),
retry_in(0),
retry_backoff(1),
times_rewritten(0),
start_addr(start_addr),
slowpath_rtn_addr(slowpath_rtn_addr),
continue_addr(continue_addr) {
for (int i = 0; i < num_slots; i++) {
slots.emplace_back(this, i);
}
#if MOVING_GC
assert(ics_list.count(this) == 0);
#endif
}
ICInfo::~ICInfo() {
#if MOVING_GC
assert(ics_list.count(this) == 0);
#endif
}
std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t* slowpath_start_addr,
uint8_t* continue_addr, uint8_t* slowpath_rtn_addr,
const ICSetupInfo* ic, StackInfo stack_info, LiveOutSet live_outs) {
assert(slowpath_start_addr - start_addr >= ic->num_slots * ic->slot_size);
assert(slowpath_rtn_addr > slowpath_start_addr);
assert(slowpath_rtn_addr <= start_addr + ic->totalSize());
assembler::GenericRegister return_register;
assert(ic->getCallingConvention() == llvm::CallingConv::C
|| ic->getCallingConvention() == llvm::CallingConv::PreserveAll);
if (ic->hasReturnValue()) {
static const int DWARF_RAX = 0;
// It's possible that the return value doesn't get used, in which case
// we can avoid copying back into RAX at the end
live_outs.clear(DWARF_RAX);
// TODO we only need to do this if 0 was in live_outs, since if it wasn't, that indicates
// the return value won't be used and we can optimize based on that.
return_register = assembler::RAX;
}
// we can let the user just slide down the nop section, but instead
// emit jumps to the end.
// Not sure if this is worth it or not?
for (int i = 0; i < ic->num_slots; i++) {
uint8_t* start = start_addr + i * ic->slot_size;
// std::unique_ptr<MCWriter> writer(createMCWriter(start, ic->slot_size * (ic->num_slots - i), 0));
// writer->emitNop();
// writer->emitGuardFalse();
Assembler writer(start, ic->slot_size);
writer.nop();
// writer.trap();
// writer.jmp(JumpDestination::fromStart(ic->slot_size * (ic->num_slots - i)));
writer.jmp(JumpDestination::fromStart(slowpath_start_addr - start));
}
ICInfo* icinfo = new ICInfo(start_addr, slowpath_rtn_addr, continue_addr, stack_info, ic->num_slots, ic->slot_size,
ic->getCallingConvention(), std::move(live_outs), return_register, ic->type_recorder);
assert(!ics_by_return_addr.count(slowpath_rtn_addr));
ics_by_return_addr[slowpath_rtn_addr] = icinfo;
registerGCTrackedICInfo(icinfo);
return std::unique_ptr<ICInfo>(icinfo);
}
void deregisterCompiledPatchpoint(ICInfo* ic) {
assert(ics_by_return_addr.count(ic->slowpath_rtn_addr));
ics_by_return_addr.erase(ic->slowpath_rtn_addr);
deregisterGCTrackedICInfo(ic);
}
ICInfo* getICInfo(void* rtn_addr) {
// TODO: load this from the CF instead of tracking it separately
auto&& it = ics_by_return_addr.find(rtn_addr);
if (it == ics_by_return_addr.end())
return NULL;
return it->second;
}
void ICInfo::clear(ICSlotInfo* icentry) {
assert(icentry);
uint8_t* start = (uint8_t*)start_addr + icentry->idx * getSlotSize();
if (VERBOSITY() >= 4)
printf("clearing patchpoint %p, slot at %p\n", start_addr, start);
Assembler writer(start, getSlotSize());
writer.nop();
writer.jmp(JumpDestination::fromStart(getSlotSize()));
assert(writer.bytesWritten() <= IC_INVALDITION_HEADER_SIZE);
// std::unique_ptr<MCWriter> writer(createMCWriter(start, getSlotSize(), 0));
// writer->emitNop();
// writer->emitGuardFalse();
// writer->endWithSlowpath();
llvm::sys::Memory::InvalidateInstructionCache(start, getSlotSize());
}
bool ICInfo::shouldAttempt() {
if (retry_in) {
retry_in--;
return false;
}
// Note(kmod): in some pathological deeply-recursive cases, it's important that we set the
// retry counter even if we attempt it again. We could probably handle this by setting
// the backoff to 0 on commit, and then setting the retry to the backoff here.
return !isMegamorphic();
}
bool ICInfo::isMegamorphic() {
return times_rewritten >= IC_MEGAMORPHIC_THRESHOLD;
}
void ICInfo::visitGCReferences(gc::GCVisitor* v) {
for (auto&& p : ic_gc_references) {
v->visitNonRelocatable(p.first);
}
#if MOVING_GC
for (const auto& p : ics_list) {
for (auto& slot : p->slots) {
v->visitNonRelocatableRange(&slot.gc_references[0], &slot.gc_references[slot.gc_references.size()]);
}
}
#endif
}
}
ICInfo::~ICInfo() {
#if MOVING_GC
assert(ics_list.count(this) == 0);
#endif
}
void deregisterGCTrackedICInfo(ICInfo* ic) {
#if MOVING_GC
assert(ics_list.count(ic) == 1);
ics_list.erase(ic);
#endif
}
void registerGCTrackedICInfo(ICInfo* ic) {
#if MOVING_GC
assert(ics_list.count(ic) == 0);
ics_list.insert(ic);
#endif
}
void foundDecl(ValueDecl *D, DeclVisibilityKind Reason) override {
// If the declaration has an override, name lookup will also have found
// the overridden method. Skip this declaration, because we prefer the
// overridden method.
if (D->getOverriddenDecl())
return;
// If the declaration is not @objc, it cannot be called dynamically.
if (!D->isObjC())
return;
// Ensure that the declaration has a type.
if (!D->hasInterfaceType()) {
if (!TypeResolver) return;
TypeResolver->resolveDeclSignature(D);
if (!D->hasInterfaceType()) return;
}
switch (D->getKind()) {
#define DECL(ID, SUPER) \
case DeclKind::ID:
#define VALUE_DECL(ID, SUPER)
#include "swift/AST/DeclNodes.def"
llvm_unreachable("not a ValueDecl!");
// Types cannot be found by dynamic lookup.
case DeclKind::GenericTypeParam:
case DeclKind::AssociatedType:
case DeclKind::TypeAlias:
case DeclKind::Enum:
case DeclKind::Class:
case DeclKind::Struct:
case DeclKind::Protocol:
return;
// Initializers cannot be found by dynamic lookup.
case DeclKind::Constructor:
case DeclKind::Destructor:
return;
// These cases are probably impossible here but can also just
// be safely ignored.
case DeclKind::EnumElement:
case DeclKind::Param:
case DeclKind::Module:
return;
// For other kinds of values, check if we already reported a decl
// with the same signature.
case DeclKind::Accessor:
case DeclKind::Func: {
auto FD = cast<FuncDecl>(D);
assert(FD->getImplicitSelfDecl() && "should not find free functions");
(void)FD;
if (FD->isInvalid())
break;
// Get the type without the first uncurry level with 'self'.
CanType T = D->getInterfaceType()
->castTo<AnyFunctionType>()
->getResult()
->getCanonicalType();
auto Signature = std::make_pair(D->getBaseName(), T);
if (!FunctionsReported.insert(Signature).second)
return;
break;
}
case DeclKind::Subscript: {
auto Signature = D->getInterfaceType()->getCanonicalType();
if (!SubscriptsReported.insert(Signature).second)
return;
break;
}
case DeclKind::Var: {
auto *VD = cast<VarDecl>(D);
auto Signature =
std::make_pair(VD->getName(),
VD->getInterfaceType()->getCanonicalType());
if (!PropertiesReported.insert(Signature).second)
return;
break;
}
}
if (isDeclVisibleInLookupMode(D, LS, CurrDC, TypeResolver))
ChainedConsumer.foundDecl(D, DeclVisibilityKind::DynamicLookup);
}
bool isInterceptedFunction(uintptr_t Address) const {
return InterceptorAddresses.count(Address);
}
void ClosureSpecializer::gatherCallSites(
SILFunction *Caller,
llvm::SmallVectorImpl<ClosureInfo*> &ClosureCandidates,
llvm::DenseSet<FullApplySite> &MultipleClosureAI) {
// A set of apply inst that we have associated with a closure. We use this to
// make sure that we do not handle call sites with multiple closure arguments.
llvm::DenseSet<FullApplySite> VisitedAI;
// For each basic block BB in Caller...
for (auto &BB : *Caller) {
// For each instruction II in BB...
for (auto &II : BB) {
// If II is not a closure that we support specializing, skip it...
if (!isSupportedClosure(&II))
continue;
ClosureInfo *CInfo = nullptr;
// Go through all uses of our closure.
for (auto *Use : II.getUses()) {
// If this use is not an apply inst or an apply inst with
// substitutions, there is nothing interesting for us to do, so
// continue...
auto AI = FullApplySite::isa(Use->getUser());
if (!AI || AI.hasSubstitutions())
continue;
// Check if we have already associated this apply inst with a closure to
// be specialized. We do not handle applies that take in multiple
// closures at this time.
if (!VisitedAI.insert(AI).second) {
MultipleClosureAI.insert(AI);
continue;
}
// If AI does not have a function_ref definition as its callee, we can
// not do anything here... so continue...
SILFunction *ApplyCallee = AI.getReferencedFunction();
if (!ApplyCallee || ApplyCallee->isExternalDeclaration())
continue;
// Ok, we know that we can perform the optimization but not whether or
// not the optimization is profitable. Find the index of the argument
// corresponding to our partial apply.
Optional<unsigned> ClosureIndex;
for (unsigned i = 0, e = AI.getNumArguments(); i != e; ++i) {
if (AI.getArgument(i) != SILValue(&II))
continue;
ClosureIndex = i;
DEBUG(llvm::dbgs() << " Found callsite with closure argument at "
<< i << ": " << *AI.getInstruction());
break;
}
// If we did not find an index, there is nothing further to do,
// continue.
if (!ClosureIndex.hasValue())
continue;
// Make sure that the Closure is invoked in the Apply's callee. We only
// want to perform closure specialization if we know that we will be
// able to change a partial_apply into an apply.
//
// TODO: Maybe just call the function directly instead of moving the
// partial apply?
SILValue Arg = ApplyCallee->getArgument(ClosureIndex.getValue());
if (std::none_of(Arg->use_begin(), Arg->use_end(),
[&Arg](Operand *Op) -> bool {
auto UserAI = FullApplySite::isa(Op->getUser());
return UserAI && UserAI.getCallee() == Arg;
})) {
continue;
}
auto NumIndirectResults =
AI.getSubstCalleeType()->getNumIndirectResults();
assert(ClosureIndex.getValue() >= NumIndirectResults);
auto ClosureParamIndex = ClosureIndex.getValue() - NumIndirectResults;
auto ParamInfo = AI.getSubstCalleeType()->getParameters();
SILParameterInfo ClosureParamInfo = ParamInfo[ClosureParamIndex];
// Get all non-failure exit BBs in the Apply Callee if our partial apply
// is guaranteed. If we do not understand one of the exit BBs, bail.
//
// We need this to make sure that we insert a release in the appropriate
// locations to balance the +1 from the creation of the partial apply.
llvm::TinyPtrVector<SILBasicBlock *> NonFailureExitBBs;
if (ClosureParamInfo.isGuaranteed() &&
!findAllNonFailureExitBBs(ApplyCallee, NonFailureExitBBs)) {
continue;
}
// Compute the final release points of the closure. We will insert
// release of the captured arguments here.
if (!CInfo) {
CInfo = new ClosureInfo(&II);
ValueLifetimeAnalysis VLA(CInfo->Closure);
VLA.computeFrontier(CInfo->LifetimeFrontier,
ValueLifetimeAnalysis::AllowToModifyCFG);
}
// Now we know that CSDesc is profitable to specialize. Add it to our
// call site list.
CInfo->CallSites.push_back(
CallSiteDescriptor(CInfo, AI, ClosureIndex.getValue(),
ClosureParamInfo, std::move(NonFailureExitBBs)));
}
if (CInfo)
ClosureCandidates.push_back(CInfo);
}
}
}
namespace pyston {
static llvm::DenseSet<BoxedString*> interned_strings;
static StatCounter num_interned_strings("num_interned_string");
extern "C" PyObject* PyString_InternFromString(const char* s) noexcept {
RELEASE_ASSERT(s, "");
return internStringImmortal(s);
}
BoxedString* internStringImmortal(llvm::StringRef s) noexcept {
auto it = interned_strings.find_as(s);
if (it != interned_strings.end())
return incref(*it);
num_interned_strings.log();
BoxedString* entry = boxString(s);
// CPython returns mortal but in our current implementation they are inmortal
entry->interned_state = SSTATE_INTERNED_IMMORTAL;
interned_strings.insert((BoxedString*)entry);
Py_INCREF(entry);
return entry;
}
extern "C" void PyString_InternInPlace(PyObject** p) noexcept {
BoxedString* s = (BoxedString*)*p;
if (s == NULL || !PyString_Check(s))
Py_FatalError("PyString_InternInPlace: strings only please!");
/* If it's a string subclass, we don't really know what putting
it in the interned dict might do. */
if (!PyString_CheckExact(s))
return;
if (PyString_CHECK_INTERNED(s))
return;
auto it = interned_strings.find(s);
if (it != interned_strings.end()) {
auto entry = *it;
Py_INCREF(entry);
Py_DECREF(*p);
*p = entry;
} else {
// TODO: do CPython's refcounting here
num_interned_strings.log();
interned_strings.insert(s);
Py_INCREF(s);
// CPython returns mortal but in our current implementation they are inmortal
s->interned_state = SSTATE_INTERNED_IMMORTAL;
}
}
extern "C" void _Py_ReleaseInternedStrings() noexcept {
// printf("%ld interned strings\n", interned_strings.size());
for (const auto& p : interned_strings) {
Py_DECREF(p);
}
interned_strings.clear();
}
}
/// TODO: We should consult the cached LoweredLocalCaptures the SIL
/// TypeConverter calculates, but that would require plumbing SILModule&
/// through every SILDeclRef constructor. Since this is only used to determine
/// "natural uncurry level", and "uncurry level" is a concept we'd like to
/// phase out, it's not worth it.
static bool hasLoweredLocalCaptures(AnyFunctionRef AFR,
llvm::DenseSet<AnyFunctionRef> &visited) {
if (!AFR.getCaptureInfo().hasLocalCaptures())
return false;
// Scan for local, non-function captures.
bool functionCapturesToRecursivelyCheck = false;
auto addFunctionCapture = [&](AnyFunctionRef capture) {
if (visited.find(capture) == visited.end())
functionCapturesToRecursivelyCheck = true;
};
for (auto &capture : AFR.getCaptureInfo().getCaptures()) {
if (!capture.getDecl()->getDeclContext()->isLocalContext())
continue;
// We transitively capture a local function's captures.
if (auto func = dyn_cast<AbstractFunctionDecl>(capture.getDecl())) {
addFunctionCapture(func);
continue;
}
// We may either directly capture properties, or capture through their
// accessors.
if (auto var = dyn_cast<VarDecl>(capture.getDecl())) {
switch (var->getStorageKind()) {
case VarDecl::StoredWithTrivialAccessors:
llvm_unreachable("stored local variable with trivial accessors?");
case VarDecl::InheritedWithObservers:
llvm_unreachable("inherited local variable?");
case VarDecl::StoredWithObservers:
case VarDecl::Addressed:
case VarDecl::AddressedWithTrivialAccessors:
case VarDecl::AddressedWithObservers:
case VarDecl::ComputedWithMutableAddress:
// Directly capture storage if we're supposed to.
if (capture.isDirect())
return true;
// Otherwise, transitively capture the accessors.
SWIFT_FALLTHROUGH;
case VarDecl::Computed:
addFunctionCapture(var->getGetter());
if (auto setter = var->getSetter())
addFunctionCapture(setter);
continue;
case VarDecl::Stored:
return true;
}
}
// Anything else is directly captured.
return true;
}
// Recursively consider function captures, since we didn't have any direct
// captures.
auto captureHasLocalCaptures = [&](AnyFunctionRef capture) -> bool {
if (visited.insert(capture).second)
return hasLoweredLocalCaptures(capture, visited);
return false;
};
if (functionCapturesToRecursivelyCheck) {
for (auto &capture : AFR.getCaptureInfo().getCaptures()) {
if (!capture.getDecl()->getDeclContext()->isLocalContext())
continue;
if (auto func = dyn_cast<AbstractFunctionDecl>(capture.getDecl())) {
if (captureHasLocalCaptures(func))
return true;
continue;
}
if (auto var = dyn_cast<VarDecl>(capture.getDecl())) {
switch (var->getStorageKind()) {
case VarDecl::StoredWithTrivialAccessors:
llvm_unreachable("stored local variable with trivial accessors?");
case VarDecl::InheritedWithObservers:
llvm_unreachable("inherited local variable?");
case VarDecl::StoredWithObservers:
case VarDecl::Addressed:
case VarDecl::AddressedWithTrivialAccessors:
case VarDecl::AddressedWithObservers:
case VarDecl::ComputedWithMutableAddress:
assert(!capture.isDirect() && "should have short circuited out");
// Otherwise, transitively capture the accessors.
SWIFT_FALLTHROUGH;
case VarDecl::Computed:
if (captureHasLocalCaptures(var->getGetter()))
return true;
if (auto setter = var->getSetter())
if (captureHasLocalCaptures(setter))
return true;
continue;
case VarDecl::Stored:
llvm_unreachable("should have short circuited out");
}
}
llvm_unreachable("should have short circuited out");
}
}
return false;
}
/// \brief Check if a Decl is referenced by non-system code.
///
bool isReferenced(::clang::Decl const *D) const {
return DeclsReferenced.count(D);
}