void CommandTerminal::parseParams() {
if(g_serialString.length() > 0) {
clearParams();
// Command strings will look like:
// x:3|y:0|r:255|g:255|b:128
// x:4|y:0|r:255|g:128|b:128
// x:5|y:0|r:234|g:188|b:164
// x:6|y:0|r:234|g:188|b:164
// First split by paramDelim
const int kMaxCommands = 16;
int pipeIndexex[kMaxCommands] = {};
int pipeIndex = -1;
for(int x = 0; x < 16; x++) {
pipeIndex = g_serialString.indexOf(paramDelim);
if (pipeIndex > -1) {
// get text left of index
String cmdSet = g_serialString.substring(0, pipeIndex);
appendParam(cmdSet, x);
// Remove cmdSet from g_serialString
String n = g_serialString.substring(pipeIndex+1);
g_serialString = n;
} else {
// Last command
appendParam(g_serialString, x);
g_serialString = "";
break;
}
}
}
}
char *
test_freeParamArray()
{
struct ParamArray *pa = newParamArray();
struct Param *p1 = setUpTestParam();
struct Param *p2 = setUpTestParam();
appendParam(pa, p1);
appendParam(pa, p2);
freeParamArray(pa);
mu_assert("freeParamArray() should free param array pointer witnout \
segfaulting", 1);
return NULL;
}
void FuncEmitter::setBuiltinFunc(const ClassInfo::MethodInfo* info,
BuiltinFunction bif, BuiltinFunction nif,
Offset base_) {
assert(info);
m_info = info;
Attr attrs_ = AttrBuiltin;
if (info->attribute & (ClassInfo::RefVariableArguments |
ClassInfo::MixedVariableArguments)) {
attrs_ |= AttrVariadicByRef;
}
if (info->attribute & ClassInfo::IsReference) {
attrs_ |= AttrReference;
}
if (info->attribute & ClassInfo::NoInjection) {
attrs_ |= AttrNoInjection;
}
if (info->attribute & ClassInfo::NoFCallBuiltin) {
attrs_ |= AttrNoFCallBuiltin;
}
if (info->attribute & ClassInfo::ParamCoerceModeNull) {
attrs_ |= AttrParamCoerceModeNull;
} else if (info->attribute & ClassInfo::ParamCoerceModeFalse) {
attrs_ |= AttrParamCoerceModeFalse;
}
if (pce()) {
if (info->attribute & ClassInfo::IsStatic) {
attrs_ |= AttrStatic;
}
if (info->attribute & ClassInfo::IsFinal) {
attrs_ |= AttrFinal;
}
if (info->attribute & ClassInfo::IsAbstract) {
attrs_ |= AttrAbstract;
}
if (info->attribute & ClassInfo::IsPrivate) {
attrs_ |= AttrPrivate;
} else if (info->attribute & ClassInfo::IsProtected) {
attrs_ |= AttrProtected;
} else {
attrs_ |= AttrPublic;
}
} else if (info->attribute & ClassInfo::AllowOverride) {
attrs_ |= AttrAllowOverride;
}
returnType = info->returnType;
docComment = makeStaticString(info->docComment);
setLocation(0, 0);
setBuiltinFunc(bif, nif, attrs_, base_);
for (unsigned i = 0; i < info->parameters.size(); ++i) {
// For builtin only, we use a dummy ParamInfo
FuncEmitter::ParamInfo pi;
const auto& parameter = info->parameters[i];
pi.byRef = parameter->attribute & ClassInfo::IsReference;
pi.builtinType = parameter->argType;
appendParam(makeStaticString(parameter->name), pi);
}
}
void FuncEmitter::setBuiltinFunc(const ClassInfo::MethodInfo* info,
BuiltinFunction bif, BuiltinFunction nif,
Offset base) {
assert(info);
assert(bif);
m_info = info;
m_builtinFuncPtr = bif;
m_nativeFuncPtr = nif;
m_base = base;
m_top = true;
m_docComment = StringData::GetStaticString(info->docComment);
m_line1 = 0;
m_line2 = 0;
m_attrs = AttrBuiltin | AttrSkipFrame;
// TODO: Task #1137917: See if we can avoid marking most builtins with
// "MayUseVV" and still make things work
m_attrs = m_attrs | AttrMayUseVV;
if (info->attribute & (ClassInfo::RefVariableArguments |
ClassInfo::MixedVariableArguments)) {
m_attrs = m_attrs | AttrVariadicByRef;
}
if (info->attribute & ClassInfo::IsReference) {
m_attrs = m_attrs | AttrReference;
}
if (info->attribute & ClassInfo::NoInjection) {
m_attrs = m_attrs | AttrNoInjection;
}
if (pce()) {
if (info->attribute & ClassInfo::IsStatic) {
m_attrs = m_attrs | AttrStatic;
}
if (info->attribute & ClassInfo::IsFinal) {
m_attrs = m_attrs | AttrFinal;
}
if (info->attribute & ClassInfo::IsAbstract) {
m_attrs = m_attrs | AttrAbstract;
}
if (info->attribute & ClassInfo::IsPrivate) {
m_attrs = m_attrs | AttrPrivate;
} else if (info->attribute & ClassInfo::IsProtected) {
m_attrs = m_attrs | AttrProtected;
} else {
m_attrs = m_attrs | AttrPublic;
}
} else if (info->attribute & ClassInfo::AllowOverride) {
m_attrs = m_attrs | AttrAllowOverride;
}
m_returnType = info->returnType;
for (unsigned i = 0; i < info->parameters.size(); ++i) {
// For builtin only, we use a dummy ParamInfo
FuncEmitter::ParamInfo pi;
pi.setRef((bool)(info->parameters[i]->attribute & ClassInfo::IsReference));
pi.setBuiltinType(info->parameters[i]->argType);
appendParam(StringData::GetStaticString(info->parameters[i]->name), pi);
}
}
// Add a character string with a specified length
void
CatApiRequest::addConstParam (const char * value, const Lng32 length)
{
// Allocate space for parameter and api structure
char *param = new char [length + 1]; // 1 for null byte
strncpy(param, value, length);
param[length] = '\0';
CatApiParam *newParam = new CatApiParam( param );
// append to end of list
appendParam(newParam);
}
char *
test_getParam()
{
struct ParamArray *pa = newParamArray();
struct Param *p1 = setUpTestParam();
struct Param *p2 = setUpTestParam();
struct Param *ret = NULL;
appendParam(pa, p1);
appendParam(pa, p2);
ret = getParam(pa, 2);
mu_assert("getParam() should return NULL when passed index of entry \
known to NOT be in the array", !ret);
ret = getParam(pa, 0);
mu_assert("getParam() should not return NULL when passed index of \
entry known to be in array", ret);
ret = getParam(pa, 1);
return NULL;
}
// Add a character string that has single-quotes that have been doubled in
// preparation for a prepare query and so the length to be used
// in the request length of the string before the single-quotes were doubled.
void
CatApiRequest::addConstParam (const Lng32 lengthForMsg, const char * value)
{
size_t realLen = strlen(value);
// Allocate space for parameter and api structure
char *param = new char [realLen + 1]; // 1 for null byte
strncpy(param, value, realLen);
param[realLen] = '\0';
CatApiParam *newParam = new CatApiParam( param, lengthForMsg );
// append param to end of list of params
appendParam(newParam);
}
void FuncEmitter::setBuiltinFunc(const ClassInfo::MethodInfo* info,
BuiltinFunction funcPtr, Offset base) {
ASSERT(info);
ASSERT(funcPtr);
m_info = info;
m_builtinFuncPtr = funcPtr;
m_base = base;
m_top = true;
m_docComment = StringData::GetStaticString(info->docComment);
m_line1 = 0;
m_line2 = 0;
m_attrs = AttrNone;
// TODO: Task #1137917: See if we can avoid marking most builtins with
// "MayUseVV" and still make things work
m_attrs = (Attr)(m_attrs | AttrMayUseVV);
if (info->attribute & (ClassInfo::RefVariableArguments |
ClassInfo::MixedVariableArguments)) {
m_attrs = Attr(m_attrs | AttrVariadicByRef);
}
if (info->attribute & ClassInfo::IsReference) {
m_attrs = (Attr)(m_attrs | AttrReference);
}
if (info->attribute & ClassInfo::NoInjection) {
m_attrs = (Attr)(m_attrs | AttrNoInjection);
}
if (pce()) {
if (info->attribute & ClassInfo::IsStatic) {
m_attrs = (Attr)(m_attrs | AttrStatic);
}
if (info->attribute & ClassInfo::IsFinal) {
m_attrs = (Attr)(m_attrs | AttrFinal);
}
if (info->attribute & ClassInfo::IsAbstract) {
m_attrs = (Attr)(m_attrs | AttrAbstract);
}
if (info->attribute & ClassInfo::IsPrivate) {
m_attrs = (Attr)(m_attrs | AttrPrivate);
} else if (info->attribute & ClassInfo::IsProtected) {
m_attrs = (Attr)(m_attrs | AttrProtected);
} else {
m_attrs = (Attr)(m_attrs | AttrPublic);
}
}
for (unsigned i = 0; i < info->parameters.size(); ++i) {
// For builtin only, we use a dummy ParamInfo
FuncEmitter::ParamInfo pi;
pi.setRef((bool)(info->parameters[i]->attribute & ClassInfo::IsReference));
appendParam(StringData::GetStaticString(info->parameters[i]->name), pi);
}
}
/*
* Warning:
* The results of test_appendParam are dependent on the correct functioning
* of newParamArray() and growParamArray()-- if the tests below fail, please
* insure the correct functioning of these functions before debugging this code.
* TODO: test structural integrity of added params under new setup (was
* confirmed under old, operation should be independent of what values param
* struct has, but should be confirmed at some point)
*/
char *
test_appendParam()
{
struct ParamArray *pa = newParamArray();
struct Param *p1 = setUpTestParam();
int i;
appendParam(pa, p1);
mu_assert("appendParam() should increment nElements appropriately",
pa->nElements == 1);
mu_assert("appendParam() should not affect param array len when \
number of elements added is less than old len",
pa->len == PA_DEFAULT_SZ);
mu_assert("first element in param array should not be null",
pa->data[0]);
appendParam(pa, p1);
mu_assert("appendParam() should increment nElements appropriately",
pa->nElements == 2);
mu_assert("appendParam() should not affect param array len when \
number of elements added is less than old len",
pa->len == PA_DEFAULT_SZ);
mu_assert("second element in param array should not be null",
pa->data[1]);
for (i=0; i < 24; i++) {
appendParam(pa, p1);
}
mu_assert("appendParam() should call growParamArray() to double size \
of array when nElements exceeds len",
pa->len == 2 * PA_DEFAULT_SZ);
mu_assert("appendParam() should successfully append param past old \
len of the array", pa->data[25]);
return NULL;
}
void Net<Dtype>::Init(const NetParameter& in_param){
CHECK(Dragon::get_root_solver() || root_net)
<< "Root net need to be set for all non-root solvers.";
phase = in_param.state().phase();
NetParameter filtered_param, param;
// filter for unqualified LayerParameters(e.g Test DataLayer)
filterNet(in_param, &filtered_param);
insertSplits(filtered_param, ¶m);
name = param.name();
LOG_IF(INFO, Dragon::get_root_solver())
<< "Initialize net from parameters: ";/*<< endl << param.DebugString();*/
map<string, int> blob_name_to_idx;
set<string> available_blobs;
CHECK_EQ(param.input_size(), param.input_shape_size())<< "input blob_shape must specify a blob.";
memory_used = 0;
// check and stuff virtual input blobs firstly [Viewing Mode Only]
for (int input_id=0; input_id < param.input_size(); input_id++){
const int layer_id = -1;
// net_input.push_back(.....virtual blob.....)
appendTop(param, layer_id, input_id, &available_blobs, &blob_name_to_idx);
}
// stuff real blobs for each layer then [Traning/Testing/Viewing Mode]
bottom_vecs.resize(param.layer_size());
bottom_id_vecs.resize(param.layer_size());
bottoms_need_backward.resize(param.layer_size());
top_vecs.resize(param.layer_size());
top_id_vecs.resize(param.layer_size());
param_id_vecs.resize(param.layer_size());
for (int layer_id = 0; layer_id < param.layer_size(); layer_id++){
bool share_from_root = !Dragon::get_root_solver()
&& root_net->layers[layer_id]->shareInParallel();
// copy net phase to layer if not set
if (!param.layer(layer_id).has_phase())
param.mutable_layer(layer_id)->set_phase(phase);
const LayerParameter& layer_param = param.layer(layer_id);
if (share_from_root){
LOG(INFO) << "Share Layer: " << layer_param.name() << " from the root net.";
// share layer by pointer
layers.push_back(root_net->layers[layer_id]);
layers[layer_id]->setShared(true);
}
else{
// use layer factory to create a pointer
// layer type is referred by layer_param->type()
// see more in layer_factory.hpp
layers.push_back(LayerFactory<Dtype>::createLayer(layer_param));
}
layer_names.push_back(layer_param.name());
LOG_IF(INFO, Dragon::get_root_solver()) << "Create Layer: " << layer_param.name();
bool need_bp = false;
// stuff bottom blobs
for (int bottom_id = 0; bottom_id < layer_param.bottom_size(); bottom_id++){
const int blob_id = appendBottom(param, layer_id, bottom_id, &available_blobs, &blob_name_to_idx);
// check whether a bottom need back propogation
need_bp |= blobs_need_backward[blob_id];
}
// stuff top blobs
for (int top_id = 0; top_id < layer_param.top_size(); top_id++)
appendTop(param, layer_id, top_id, &available_blobs, &blob_name_to_idx);
// auto top blobs
// NOT_IMPLEMENTED;
Layer<Dtype>* layer = layers[layer_id].get();
// setup for layer
if (share_from_root){
const vector<Blob<Dtype>*> base_top = root_net->top_vecs[layer_id];
const vector<Blob<Dtype>*> this_top = this->top_vecs[layer_id];
// reshape solely after root_net finishing
for (int top_id = 0; top_id < base_top.size(); top_id++){
this_top[top_id]->reshapeLike(*base_top[top_id]);
}
}
else layer->setup(bottom_vecs[layer_id], top_vecs[layer_id]);
LOG_IF(INFO, Dragon::get_root_solver()) << "Setup Layer: " << layer_param.name();
for (int top_id = 0; top_id < top_vecs[layer_id].size(); top_id++){
// extend size to max number of blobs if necessary
if (blobs_loss_weight.size() <= top_id_vecs[layer_id][top_id])
blobs_loss_weight.resize(top_id_vecs[layer_id][top_id] + 1, Dtype(0));
// store global loss weights from each layer each blob
blobs_loss_weight[top_id_vecs[layer_id][top_id]] = layer->getLoss(top_id);
LOG_IF(INFO, Dragon::get_root_solver())
<< "Top shape: " << top_vecs[layer_id][top_id]->shape_string();
if (layer->getLoss(top_id)) LOG_IF(INFO, Dragon::get_root_solver())
<< " with loss weight " << layer->getLoss(top_id);
// sum up for training parameter statistic
memory_used += top_vecs[layer_id][top_id]->count();
}
LOG_IF(INFO, Dragon::get_root_solver())
<< "Memory required for Data: " << memory_used*sizeof(Dtype);
const int param_size = layer_param.param_size();
// blobs_size will be set after layer->setup()
const int param_blobs_size = layer->getBlobs().size();
CHECK_LE(param_size, param_blobs_size)<< "Too many params specify for layer.";
// use if do not specify hyperparameter
// lr_mult=decay_mult=1.0
ParamSpec default_hyperparameter;
for (int param_id = 0; param_id < param_blobs_size; param_id++){
const ParamSpec* hyperparameter = param_id < param_size ?
&layer_param.param(param_id) : &default_hyperparameter;
const bool param_need_bp = hyperparameter->lr_mult() != 0;
// check whether a param blob need back propogation [default=true]
need_bp |= param_need_bp;
layer->setParamNeedBp(param_id, param_need_bp);
}
// stuff param blobs
for (int param_id = 0; param_id < param_blobs_size; param_id++)
appendParam(param, layer_id, param_id);
// update param blobs if share others
shareWeights();
layer_need_backward.push_back(need_bp);
// after checking all bottom blobs and param blobs
if (need_bp)
for (int top_id = 0; top_id < top_id_vecs[layer_id].size(); top_id++)
blobs_need_backward[top_id_vecs[layer_id][top_id]] = true;
} // end layer_id
set<string> blobs_under_loss, blobs_skip_bp;
for (int layer_id = layers.size()-1; layer_id >= 0; layer_id--){
bool layer_contributes_loss = false;
bool layer_skip_bp = true;
Layer<Dtype>* layer = layers[layer_id].get();
for (int top_id = 0; top_id < top_vecs[layer_id].size(); top_id++){
const string& blob_name = blobs_name[top_id_vecs[layer_id][top_id]];
if (layer->getLoss(top_id) || blobs_under_loss.count(blob_name))
layer_contributes_loss = true;
if (!blobs_skip_bp.count(blob_name)) layer_skip_bp = false;
// find any top blobs if affected by loss and do not force to skip bp
if (layer_contributes_loss&&!layer_skip_bp) break;
}
// optimization trick: set lr_mult but is not affected by loss
if (layer_need_backward[layer_id] && layer_skip_bp){
// cancel layer
layer_need_backward[layer_id] = false;
// cancel bottom
for (int bottom_id = 0; bottom_id < bottom_vecs[layer_id].size(); bottom_id++){
bottoms_need_backward[layer_id][bottom_id] = false;
}
}
// cancel directly if layer is not affected by loss
if (!layer_contributes_loss) layer_need_backward[layer_id] = false;
// debug info
if (Dragon::get_root_solver()){
if (layer_need_backward[layer_id])
LOG(INFO) << "Layer: " << layer_names[layer_id] << " need back-propogation.";
else LOG(INFO) << "Layer: " << layer_names[layer_id] << " does not need back-propogation.";
}
// if one top blob affected by loss
// all bottom blobs will be affected
// regard it as "loss back-affected"
for (int bottom_id = 0; bottom_id < bottom_vecs[layer_id].size(); bottom_id++){
const string& blob_name = blobs_name[bottom_id_vecs[layer_id][bottom_id]];
if (layer_contributes_loss) blobs_under_loss.insert(blob_name);
else bottoms_need_backward[layer_id][bottom_id] = false;
// use for optimization trick : skip all bottom blobs
if (!bottoms_need_backward[layer_id][bottom_id]) blobs_skip_bp.insert(blob_name);
}
} // end layer id
if (param.force_backward()){
for (int layer_id = 0; layer_id < layers.size(); layer_id++){
layer_need_backward[layer_id] = true;
for (int bottom_id = 0; bottom_id < bottom_vecs[layer_id].size(); bottom_id++){
// set for bottoms
bottoms_need_backward[layer_id][bottom_id] =
bottoms_need_backward[layer_id][bottom_id]||layers[layer_id]->allowForceBackward(bottom_id);
// set for blobs
blobs_need_backward[bottom_id_vecs[layer_id][bottom_id]] =
blobs_need_backward[bottom_id_vecs[layer_id][bottom_id]]||bottoms_need_backward[layer_id][bottom_id];
}
// set for params
for (int param_id = 0; param_id < layers[layer_id]->getBlobs().size(); param_id++){
layers[layer_id]->setParamNeedBp(param_id, true);
}
}
}
// move un-used(declare top but not use as bottom) blobs into output blobs
// usually contain loss blobs
for (set<string>::iterator i = available_blobs.begin(); i != available_blobs.end(); i++){
LOG_IF(INFO, Dragon::get_root_solver())
<< "Network produces output: " << *i;
net_output_blobs.push_back(blobs[blob_name_to_idx[*i]].get());
net_output_blob_indices.push_back(blob_name_to_idx[*i]);
}
// store blob_name -> blob_ids
blobs_name_idx = blob_name_to_idx;
// store layer_name -> layer_id
for (size_t layer_id = 0; layer_id < layer_names.size(); layer_id++)
layers_name_idx[layer_names[layer_id]] = layer_id;
debug_info = param.debug_info();
LOG_IF(INFO, Dragon::get_root_solver()) << "Network Initializion done.";
}
Func* FuncEmitter::create(Unit& unit, PreClass* preClass /* = NULL */) const {
bool isGenerated = isdigit(name->data()[0]) || needsStripInOut(name);
Attr attrs = this->attrs;
if (preClass && preClass->attrs() & AttrInterface) {
attrs |= AttrAbstract;
}
if (!RuntimeOption::RepoAuthoritative) {
if (RuntimeOption::EvalJitEnableRenameFunction) {
attrs |= AttrInterceptable;
} else {
attrs = Attr(attrs & ~AttrInterceptable);
}
}
if (attrs & AttrPersistent && !preClass) {
if ((RuntimeOption::EvalJitEnableRenameFunction ||
attrs & AttrInterceptable ||
(!RuntimeOption::RepoAuthoritative && SystemLib::s_inited))) {
if (attrs & AttrBuiltin) {
SystemLib::s_anyNonPersistentBuiltins = true;
}
attrs = Attr(attrs & ~AttrPersistent);
}
} else {
assertx(preClass || !(attrs & AttrBuiltin));
}
if (!RuntimeOption::RepoAuthoritative) {
// In non-RepoAuthoritative mode, any function could get a VarEnv because
// of evalPHPDebugger.
attrs |= AttrMayUseVV;
} else if ((attrs & AttrInterceptable) &&
!name->empty() &&
!Func::isSpecial(name) &&
!isClosureBody) {
// intercepted functions need to pass all args through
// to the interceptee
attrs |= AttrMayUseVV;
}
if (isVariadic()) {
attrs |= AttrVariadicParam;
if (isVariadicByRef()) {
attrs |= AttrVariadicByRef;
}
}
assertx(!m_pce == !preClass);
auto f = m_ue.newFunc(this, unit, name, attrs, params.size());
f->m_isPreFunc = !!preClass;
bool const needsExtendedSharedData =
isNative ||
line2 - line1 >= Func::kSmallDeltaLimit ||
past - base >= Func::kSmallDeltaLimit ||
m_numClsRefSlots > 3;
f->m_shared.reset(
needsExtendedSharedData
? new Func::ExtendedSharedData(preClass, base, past, line1, line2,
top, !containsCalls, docComment)
: new Func::SharedData(preClass, base, past,
line1, line2, top, !containsCalls, docComment)
);
f->init(params.size());
if (auto const ex = f->extShared()) {
ex->m_hasExtendedSharedData = true;
ex->m_arFuncPtr = nullptr;
ex->m_nativeFuncPtr = nullptr;
ex->m_line2 = line2;
ex->m_past = past;
ex->m_returnByValue = false;
ex->m_isMemoizeWrapper = false;
ex->m_isMemoizeWrapperLSB = false;
ex->m_actualNumClsRefSlots = m_numClsRefSlots;
}
std::vector<Func::ParamInfo> fParams;
for (unsigned i = 0; i < params.size(); ++i) {
Func::ParamInfo pi = params[i];
if (pi.isVariadic()) {
pi.builtinType = RuntimeOption::EvalHackArrDVArrs
? KindOfVec : KindOfArray;
}
f->appendParam(params[i].byRef, pi, fParams);
}
auto const originalFullName =
(!originalFilename ||
!RuntimeOption::RepoAuthoritative ||
FileUtil::isAbsolutePath(originalFilename->slice())) ?
originalFilename :
makeStaticString(RuntimeOption::SourceRoot +
originalFilename->toCppString());
f->shared()->m_localNames.create(m_localNames);
f->shared()->m_numLocals = m_numLocals;
f->shared()->m_numIterators = m_numIterators;
f->m_maxStackCells = maxStackCells;
f->shared()->m_staticVars = staticVars;
f->shared()->m_ehtab = ehtab;
f->shared()->m_fpitab = fpitab;
f->shared()->m_isClosureBody = isClosureBody;
f->shared()->m_isAsync = isAsync;
f->shared()->m_isGenerator = isGenerator;
f->shared()->m_isPairGenerator = isPairGenerator;
f->shared()->m_userAttributes = userAttributes;
f->shared()->m_retTypeConstraint = retTypeConstraint;
f->shared()->m_retUserType = retUserType;
f->shared()->m_originalFilename = originalFullName;
f->shared()->m_isGenerated = isGenerated;
f->shared()->m_repoReturnType = repoReturnType;
f->shared()->m_repoAwaitedReturnType = repoAwaitedReturnType;
f->shared()->m_isMemoizeWrapper = isMemoizeWrapper;
f->shared()->m_isMemoizeWrapperLSB = isMemoizeWrapperLSB;
f->shared()->m_numClsRefSlots = m_numClsRefSlots;
if (isNative) {
auto const ex = f->extShared();
ex->m_hniReturnType = hniReturnType;
auto const info = getNativeInfo();
Attr dummy = AttrNone;
auto nativeAttributes = parseNativeAttributes(dummy);
Native::getFunctionPointers(
info,
nativeAttributes,
ex->m_arFuncPtr,
ex->m_nativeFuncPtr
);
ex->m_takesNumArgs = !!(nativeAttributes & Native::AttrTakesNumArgs);
if (ex->m_nativeFuncPtr) {
if (info.sig.ret == Native::NativeSig::Type::MixedTV) {
ex->m_returnByValue = true;
}
int extra =
(nativeAttributes & Native::AttrTakesNumArgs ? 1 : 0) +
(isMethod() ? 1 : 0);
assertx(info.sig.args.size() == params.size() + extra);
for (auto i = params.size(); i--; ) {
switch (info.sig.args[extra + i]) {
case Native::NativeSig::Type::ObjectArg:
case Native::NativeSig::Type::StringArg:
case Native::NativeSig::Type::ArrayArg:
case Native::NativeSig::Type::ResourceArg:
case Native::NativeSig::Type::OutputArg:
case Native::NativeSig::Type::MixedTV:
fParams[i].nativeArg = true;
break;
default:
break;
}
}
}
}
f->finishedEmittingParams(fParams);
return f;
}
Func* FuncEmitter::create(Unit& unit, PreClass* preClass /* = NULL */) const {
bool isGenerated = isdigit(name->data()[0]) ||
ParserBase::IsClosureName(name->toCppString());
Attr attrs = this->attrs;
if (preClass && preClass->attrs() & AttrInterface) {
attrs |= AttrAbstract;
}
if (attrs & AttrPersistent &&
((RuntimeOption::EvalJitEnableRenameFunction && !isGenerated) ||
(!RuntimeOption::RepoAuthoritative && SystemLib::s_inited) ||
attrs & AttrInterceptable)) {
if (attrs & AttrBuiltin) {
SystemLib::s_anyNonPersistentBuiltins = true;
}
attrs = Attr(attrs & ~AttrPersistent);
}
if (!RuntimeOption::RepoAuthoritative) {
// In non-RepoAuthoritative mode, any function could get a VarEnv because
// of evalPHPDebugger.
attrs |= AttrMayUseVV;
} else if (RuntimeOption::EvalJitEnableRenameFunction &&
!name->empty() &&
!Func::isSpecial(name) &&
!isClosureBody) {
// intercepted functions need to pass all args through
// to the interceptee
attrs |= AttrMayUseVV;
}
if (isVariadic()) { attrs |= AttrVariadicParam; }
if (!containsCalls) { attrs |= AttrPhpLeafFn; }
assert(!m_pce == !preClass);
auto f = m_ue.newFunc(this, unit, name, attrs, params.size());
f->m_isPreFunc = !!preClass;
bool const needsExtendedSharedData =
m_info ||
m_builtinFuncPtr ||
m_nativeFuncPtr ||
(attrs & AttrNative) ||
line2 - line1 >= Func::kSmallDeltaLimit ||
past - base >= Func::kSmallDeltaLimit;
f->m_shared.reset(
needsExtendedSharedData
? new Func::ExtendedSharedData(preClass, base, past, line1, line2,
top, docComment)
: new Func::SharedData(preClass, base, past,
line1, line2, top, docComment)
);
f->init(params.size());
if (auto const ex = f->extShared()) {
ex->m_hasExtendedSharedData = true;
ex->m_builtinFuncPtr = m_builtinFuncPtr;
ex->m_nativeFuncPtr = m_nativeFuncPtr;
ex->m_info = m_info;
ex->m_line2 = line2;
ex->m_past = past;
ex->m_returnByValue = false;
}
std::vector<Func::ParamInfo> fParams;
for (unsigned i = 0; i < params.size(); ++i) {
Func::ParamInfo pi = params[i];
if (pi.isVariadic()) {
pi.builtinType = KindOfArray;
}
f->appendParam(params[i].byRef, pi, fParams);
}
f->shared()->m_returnType = returnType;
f->shared()->m_localNames.create(m_localNames);
f->shared()->m_numLocals = m_numLocals;
f->shared()->m_numIterators = m_numIterators;
f->m_maxStackCells = maxStackCells;
f->shared()->m_staticVars = staticVars;
f->shared()->m_ehtab = toFixed(ehtab);
f->shared()->m_fpitab = fpitab;
f->shared()->m_isClosureBody = isClosureBody;
f->shared()->m_isAsync = isAsync;
f->shared()->m_isGenerator = isGenerator;
f->shared()->m_isPairGenerator = isPairGenerator;
f->shared()->m_userAttributes = userAttributes;
f->shared()->m_retTypeConstraint = retTypeConstraint;
f->shared()->m_retUserType = retUserType;
f->shared()->m_originalFilename = originalFilename;
f->shared()->m_isGenerated = isGenerated;
if (attrs & AttrNative) {
auto const ex = f->extShared();
auto const& info = Native::GetBuiltinFunction(
name,
m_pce ? m_pce->name() : nullptr,
f->isStatic()
);
Attr dummy = AttrNone;
auto nativeAttributes = parseNativeAttributes(dummy);
Native::getFunctionPointers(
info,
nativeAttributes,
ex->m_builtinFuncPtr,
ex->m_nativeFuncPtr
);
if (ex->m_nativeFuncPtr &&
!(nativeAttributes & Native::AttrZendCompat)) {
if (info.sig.ret == Native::NativeSig::Type::MixedTV) {
ex->m_returnByValue = true;
}
int extra =
(attrs & AttrNumArgs ? 1 : 0) +
(isMethod() ? 1 : 0);
assert(info.sig.args.size() == params.size() + extra);
for (auto i = params.size(); i--; ) {
switch (info.sig.args[extra + i]) {
case Native::NativeSig::Type::ObjectArg:
case Native::NativeSig::Type::StringArg:
case Native::NativeSig::Type::ArrayArg:
case Native::NativeSig::Type::ResourceArg:
case Native::NativeSig::Type::OutputArg:
case Native::NativeSig::Type::MixedTV:
fParams[i].nativeArg = true;
break;
default:
break;
}
}
}
}
f->finishedEmittingParams(fParams);
return f;
}
Func* FuncEmitter::create(Unit& unit, PreClass* preClass /* = NULL */) const {
bool isGenerated = isdigit(name->data()[0]) ||
ParserBase::IsClosureName(name->toCppString());
Attr attrs = this->attrs;
if (preClass && preClass->attrs() & AttrInterface) {
attrs |= AttrAbstract;
}
if (attrs & AttrPersistent &&
((RuntimeOption::EvalJitEnableRenameFunction && !isGenerated) ||
(!RuntimeOption::RepoAuthoritative && SystemLib::s_inited) ||
attrs & AttrInterceptable)) {
if (attrs & AttrBuiltin) {
SystemLib::s_anyNonPersistentBuiltins = true;
}
attrs = Attr(attrs & ~AttrPersistent);
}
if (!RuntimeOption::RepoAuthoritative) {
// In non-RepoAuthoritative mode, any function could get a VarEnv because
// of evalPHPDebugger.
attrs |= AttrMayUseVV;
} else if (RuntimeOption::EvalJitEnableRenameFunction &&
!name->empty() &&
!Func::isSpecial(name) &&
!isClosureBody) {
// intercepted functions need to pass all args through
// to the interceptee
attrs |= AttrMayUseVV;
}
if (isVariadic()) { attrs |= AttrVariadicParam; }
if (!containsCalls) { attrs |= AttrPhpLeafFn; }
assert(!m_pce == !preClass);
auto f = m_ue.newFunc(this, unit, name, attrs, params.size());
f->m_isPreFunc = !!preClass;
bool const needsExtendedSharedData =
m_info ||
m_builtinFuncPtr ||
m_nativeFuncPtr ||
(attrs & AttrNative) ||
line2 - line1 >= Func::kSmallDeltaLimit ||
past - base >= Func::kSmallDeltaLimit;
f->m_shared.reset(
needsExtendedSharedData
? new Func::ExtendedSharedData(preClass, base, past, line1, line2,
top, docComment)
: new Func::SharedData(preClass, base, past,
line1, line2, top, docComment)
);
f->init(params.size());
if (auto const ex = f->extShared()) {
ex->m_hasExtendedSharedData = true;
ex->m_builtinFuncPtr = m_builtinFuncPtr;
ex->m_nativeFuncPtr = m_nativeFuncPtr;
ex->m_info = m_info;
ex->m_line2 = line2;
ex->m_past = past;
}
std::vector<Func::ParamInfo> fParams;
bool usesDoubles = false, variadic = false;
for (unsigned i = 0; i < params.size(); ++i) {
Func::ParamInfo pi = params[i];
if (pi.builtinType == KindOfDouble) usesDoubles = true;
if (pi.isVariadic()) variadic = true;
f->appendParam(params[i].byRef, pi, fParams);
}
f->shared()->m_returnType = returnType;
f->shared()->m_localNames.create(m_localNames);
f->shared()->m_numLocals = m_numLocals;
f->shared()->m_numIterators = m_numIterators;
f->m_maxStackCells = maxStackCells;
f->shared()->m_staticVars = staticVars;
f->shared()->m_ehtab = toFixed(ehtab);
f->shared()->m_fpitab = fpitab;
f->shared()->m_isClosureBody = isClosureBody;
f->shared()->m_isAsync = isAsync;
f->shared()->m_isGenerator = isGenerator;
f->shared()->m_isPairGenerator = isPairGenerator;
f->shared()->m_userAttributes = userAttributes;
f->shared()->m_retTypeConstraint = retTypeConstraint;
f->shared()->m_retUserType = retUserType;
f->shared()->m_originalFilename = originalFilename;
f->shared()->m_isGenerated = isGenerated;
f->finishedEmittingParams(fParams);
if (attrs & AttrNative) {
auto const ex = f->extShared();
auto const& info = Native::GetBuiltinFunction(
name,
m_pce ? m_pce->name() : nullptr,
f->isStatic()
);
auto const nif = info.ptr;
if (nif) {
Attr dummy = AttrNone;
int nativeAttrs = parseNativeAttributes(dummy);
if (nativeAttrs & Native::AttrZendCompat) {
ex->m_nativeFuncPtr = nif;
ex->m_builtinFuncPtr = zend_wrap_func;
} else {
if (parseNativeAttributes(dummy) & Native::AttrActRec) {
ex->m_builtinFuncPtr = nif;
ex->m_nativeFuncPtr = nullptr;
} else {
ex->m_nativeFuncPtr = nif;
ex->m_builtinFuncPtr =
Native::getWrapper(m_pce, usesDoubles, variadic);
}
}
} else {
ex->m_builtinFuncPtr = Native::unimplementedWrapper;
}
}
return f;
}