static tnfctl_errcode_t
percmd(expr_t *expr_p, cmd_kind_t kind, fcn_t *fcn_p, boolean_t isnew,
void *calldata_p)
{
process_args_t *args_p = (process_args_t *)calldata_p;
tnfctl_handle_t *hndl = args_p->hndl;
tnfctl_probe_t *probe_p = args_p->probe_p;
tnfctl_errcode_t err = TNFCTL_ERR_NONE;
char *attrs;
attrs = list_getattrs(probe_p);
if (expr_match(expr_p, attrs)) {
#if defined(DEBUG) || defined(lint)
if (g_verbose) {
char *cmdstr[] = {
"enable", "disable",
"connect", "clear",
"trace", "untrace"};
(void) fprintf(stderr, ": %s command: %s ",
(isnew) ? "new" : "old", cmdstr[kind]);
expr_print(stderr, expr_p);
}
#endif
switch (kind) {
case CMD_ENABLE:
err = tnfctl_probe_enable(hndl, probe_p, NULL);
break;
case CMD_DISABLE:
err = tnfctl_probe_disable(hndl, probe_p, NULL);
break;
case CMD_TRACE:
err = tnfctl_probe_trace(hndl, probe_p, NULL);
break;
case CMD_UNTRACE:
err = tnfctl_probe_untrace(hndl, probe_p, NULL);
break;
case CMD_CONNECT:
err = tnfctl_probe_connect(hndl, probe_p, NULL,
fcn_p->entry_name_p);
break;
case CMD_CLEAR:
err = tnfctl_probe_disconnect_all(hndl, probe_p, NULL);
break;
}
#if defined(DEBUG) || defined(lint)
if (g_verbose)
(void) fprintf(stderr, "\n");
#endif
}
if (attrs)
free(attrs);
return (err);
}
boolean_t
set_match(set_t * set_p, const char *name, const char *keys)
{
if (!set_p)
return (B_FALSE);
return (expr_match(set_p->exprlist_p, name, keys));
} /* end set_match */
int
tool_process_seq(const char *id, const char *comment, const char *seq)
{
int match, i, c, s;
i = expr_match(&mi, id);
c = expr_match(&mc, comment);
s = expr_match(&ms, seq);
if ((mflags & MATCH_AND)) {
match = i && c && s;
}
else {
match = i || c || s;
}
if (mflags & MATCH_INV) {
match = !match;
}
if (match) {
fasta_write(stdout, id, comment, seq, main_width);
}
return FASTA_OK;
}
void CodeGen_X86::visit(const Select *op) {
// LLVM doesn't correctly use pblendvb for u8 vectors that aren't
// width 16, so we peephole optimize them to intrinsics.
struct Pattern {
string intrin;
Expr pattern;
};
static Pattern patterns[] = {
{"pblendvb_ult_i8x16", select(wild_u8x_ < wild_u8x_, wild_i8x_, wild_i8x_)},
{"pblendvb_ult_i8x16", select(wild_u8x_ < wild_u8x_, wild_u8x_, wild_u8x_)},
{"pblendvb_slt_i8x16", select(wild_i8x_ < wild_i8x_, wild_i8x_, wild_i8x_)},
{"pblendvb_slt_i8x16", select(wild_i8x_ < wild_i8x_, wild_u8x_, wild_u8x_)},
{"pblendvb_ule_i8x16", select(wild_u8x_ <= wild_u8x_, wild_i8x_, wild_i8x_)},
{"pblendvb_ule_i8x16", select(wild_u8x_ <= wild_u8x_, wild_u8x_, wild_u8x_)},
{"pblendvb_sle_i8x16", select(wild_i8x_ <= wild_i8x_, wild_i8x_, wild_i8x_)},
{"pblendvb_sle_i8x16", select(wild_i8x_ <= wild_i8x_, wild_u8x_, wild_u8x_)},
{"pblendvb_ne_i8x16", select(wild_u8x_ != wild_u8x_, wild_i8x_, wild_i8x_)},
{"pblendvb_ne_i8x16", select(wild_u8x_ != wild_u8x_, wild_u8x_, wild_u8x_)},
{"pblendvb_ne_i8x16", select(wild_i8x_ != wild_i8x_, wild_i8x_, wild_i8x_)},
{"pblendvb_ne_i8x16", select(wild_i8x_ != wild_i8x_, wild_i8x_, wild_i8x_)},
{"pblendvb_eq_i8x16", select(wild_u8x_ == wild_u8x_, wild_i8x_, wild_i8x_)},
{"pblendvb_eq_i8x16", select(wild_u8x_ == wild_u8x_, wild_u8x_, wild_u8x_)},
{"pblendvb_eq_i8x16", select(wild_i8x_ == wild_i8x_, wild_i8x_, wild_i8x_)},
{"pblendvb_eq_i8x16", select(wild_i8x_ == wild_i8x_, wild_i8x_, wild_i8x_)},
{"pblendvb_i8x16", select(wild_u1x_, wild_i8x_, wild_i8x_)},
{"pblendvb_i8x16", select(wild_u1x_, wild_u8x_, wild_u8x_)}
};
if (target.has_feature(Target::SSE41) &&
op->condition.type().is_vector() &&
op->type.bits() == 8 &&
op->type.lanes() != 16) {
vector<Expr> matches;
for (size_t i = 0; i < sizeof(patterns)/sizeof(patterns[0]); i++) {
if (expr_match(patterns[i].pattern, op, matches)) {
value = call_intrin(op->type, 16, patterns[i].intrin, matches);
return;
}
}
}
CodeGen_Posix::visit(op);
}
void CodeGen_X86::visit(const Cast *op) {
if (!op->type.is_vector()) {
// We only have peephole optimizations for vectors in here.
CodeGen_Posix::visit(op);
return;
}
vector<Expr> matches;
struct Pattern {
Target::Feature feature;
bool wide_op;
Type type;
int min_lanes;
string intrin;
Expr pattern;
};
static Pattern patterns[] = {
{Target::AVX2, true, Int(8, 32), 0, "llvm.x86.avx2.padds.b",
i8_sat(wild_i16x_ + wild_i16x_)},
{Target::FeatureEnd, true, Int(8, 16), 0, "llvm.x86.sse2.padds.b",
i8_sat(wild_i16x_ + wild_i16x_)},
{Target::AVX2, true, Int(8, 32), 0, "llvm.x86.avx2.psubs.b",
i8_sat(wild_i16x_ - wild_i16x_)},
{Target::FeatureEnd, true, Int(8, 16), 0, "llvm.x86.sse2.psubs.b",
i8_sat(wild_i16x_ - wild_i16x_)},
#if LLVM_VERSION < 80
// Older LLVM versions support this as an intrinsic
{Target::AVX2, true, UInt(8, 32), 0, "llvm.x86.avx2.paddus.b",
u8_sat(wild_u16x_ + wild_u16x_)},
{Target::FeatureEnd, true, UInt(8, 16), 0, "llvm.x86.sse2.paddus.b",
u8_sat(wild_u16x_ + wild_u16x_)},
{Target::AVX2, true, UInt(8, 32), 0, "llvm.x86.avx2.psubus.b",
u8(max(wild_i16x_ - wild_i16x_, 0))},
{Target::FeatureEnd, true, UInt(8, 16), 0, "llvm.x86.sse2.psubus.b",
u8(max(wild_i16x_ - wild_i16x_, 0))},
#else
// LLVM 8.0+ require using helpers from x86.ll
{Target::AVX2, true, UInt(8, 32), 0, "paddusbx32",
u8_sat(wild_u16x_ + wild_u16x_)},
{Target::FeatureEnd, true, UInt(8, 16), 0, "paddusbx16",
u8_sat(wild_u16x_ + wild_u16x_)},
{Target::AVX2, true, UInt(8, 32), 0, "psubusbx32",
u8(max(wild_i16x_ - wild_i16x_, 0))},
{Target::FeatureEnd, true, UInt(8, 16), 0, "psubusbx16",
u8(max(wild_i16x_ - wild_i16x_, 0))},
#endif
{Target::AVX2, true, Int(16, 16), 0, "llvm.x86.avx2.padds.w",
i16_sat(wild_i32x_ + wild_i32x_)},
{Target::FeatureEnd, true, Int(16, 8), 0, "llvm.x86.sse2.padds.w",
i16_sat(wild_i32x_ + wild_i32x_)},
{Target::AVX2, true, Int(16, 16), 0, "llvm.x86.avx2.psubs.w",
i16_sat(wild_i32x_ - wild_i32x_)},
{Target::FeatureEnd, true, Int(16, 8), 0, "llvm.x86.sse2.psubs.w",
i16_sat(wild_i32x_ - wild_i32x_)},
#if LLVM_VERSION < 80
// Older LLVM versions support this as an intrinsic
{Target::AVX2, true, UInt(16, 16), 0, "llvm.x86.avx2.paddus.w",
u16_sat(wild_u32x_ + wild_u32x_)},
{Target::FeatureEnd, true, UInt(16, 8), 0, "llvm.x86.sse2.paddus.w",
u16_sat(wild_u32x_ + wild_u32x_)},
{Target::AVX2, true, UInt(16, 16), 0, "llvm.x86.avx2.psubus.w",
u16(max(wild_i32x_ - wild_i32x_, 0))},
{Target::FeatureEnd, true, UInt(16, 8), 0, "llvm.x86.sse2.psubus.w",
u16(max(wild_i32x_ - wild_i32x_, 0))},
#else
// LLVM 8.0+ require using helpers from x86.ll
{Target::AVX2, true, UInt(16, 16), 0, "padduswx16",
u16_sat(wild_u32x_ + wild_u32x_)},
{Target::FeatureEnd, true, UInt(16, 8), 0, "padduswx8",
u16_sat(wild_u32x_ + wild_u32x_)},
{Target::AVX2, true, UInt(16, 16), 0, "psubuswx16",
u16(max(wild_i32x_ - wild_i32x_, 0))},
{Target::FeatureEnd, true, UInt(16, 8), 0, "psubuswx8",
u16(max(wild_i32x_ - wild_i32x_, 0))},
#endif
// Only use the avx2 version if we have > 8 lanes
{Target::AVX2, true, Int(16, 16), 9, "llvm.x86.avx2.pmulh.w",
i16((wild_i32x_ * wild_i32x_) / 65536)},
{Target::AVX2, true, UInt(16, 16), 9, "llvm.x86.avx2.pmulhu.w",
u16((wild_u32x_ * wild_u32x_) / 65536)},
{Target::FeatureEnd, true, Int(16, 8), 0, "llvm.x86.sse2.pmulh.w",
i16((wild_i32x_ * wild_i32x_) / 65536)},
{Target::FeatureEnd, true, UInt(16, 8), 0, "llvm.x86.sse2.pmulhu.w",
u16((wild_u32x_ * wild_u32x_) / 65536)},
// LLVM 6.0+ require using helpers from x86.ll
{Target::AVX2, true, UInt(8, 32), 0, "pavgbx32",
u8(((wild_u16x_ + wild_u16x_) + 1) / 2)},
{Target::FeatureEnd, true, UInt(8, 16), 0, "pavgbx16",
u8(((wild_u16x_ + wild_u16x_) + 1) / 2)},
{Target::AVX2, true, UInt(16, 16), 0, "pavgwx16",
u16(((wild_u32x_ + wild_u32x_) + 1) / 2)},
{Target::FeatureEnd, true, UInt(16, 8), 0, "pavgwx8",
u16(((wild_u32x_ + wild_u32x_) + 1) / 2)},
{Target::AVX2, false, Int(16, 16), 0, "packssdwx16",
i16_sat(wild_i32x_)},
{Target::FeatureEnd, false, Int(16, 8), 0, "packssdwx8",
i16_sat(wild_i32x_)},
{Target::AVX2, false, Int(8, 32), 0, "packsswbx32",
i8_sat(wild_i16x_)},
{Target::FeatureEnd, false, Int(8, 16), 0, "packsswbx16",
i8_sat(wild_i16x_)},
{Target::AVX2, false, UInt(8, 32), 0, "packuswbx32",
u8_sat(wild_i16x_)},
{Target::FeatureEnd, false, UInt(8, 16), 0, "packuswbx16",
u8_sat(wild_i16x_)},
{Target::AVX2, false, UInt(16, 16), 0, "packusdwx16",
u16_sat(wild_i32x_)},
{Target::SSE41, false, UInt(16, 8), 0, "packusdwx8",
u16_sat(wild_i32x_)}
};
for (size_t i = 0; i < sizeof(patterns)/sizeof(patterns[0]); i++) {
const Pattern &pattern = patterns[i];
if (!target.has_feature(pattern.feature)) {
continue;
}
if (op->type.lanes() < pattern.min_lanes) {
continue;
}
if (expr_match(pattern.pattern, op, matches)) {
bool match = true;
if (pattern.wide_op) {
// Try to narrow the matches to the target type.
for (size_t i = 0; i < matches.size(); i++) {
matches[i] = lossless_cast(op->type, matches[i]);
if (!matches[i].defined()) match = false;
}
}
if (match) {
value = call_intrin(op->type, pattern.type.lanes(), pattern.intrin, matches);
return;
}
}
}
// Workaround for https://llvm.org/bugs/show_bug.cgi?id=24512
// LLVM uses a numerically unstable method for vector
// uint32->float conversion before AVX.
if (op->value.type().element_of() == UInt(32) &&
op->type.is_float() &&
op->type.is_vector() &&
!target.has_feature(Target::AVX)) {
Type signed_type = Int(32, op->type.lanes());
// Convert the top 31 bits to float using the signed version
Expr top_bits = cast(signed_type, op->value / 2);
top_bits = cast(op->type, top_bits);
// Convert the bottom bit
Expr bottom_bit = cast(signed_type, op->value % 2);
bottom_bit = cast(op->type, bottom_bit);
// Recombine as floats
codegen(top_bits + top_bits + bottom_bit);
return;
}
CodeGen_Posix::visit(op);
}
void CodeGen_X86::visit(const Cast *op) {
if (!op->type.is_vector()) {
// We only have peephole optimizations for vectors in here.
CodeGen_Posix::visit(op);
return;
}
vector<Expr> matches;
struct Pattern {
bool needs_sse_41;
bool wide_op;
Type type;
string intrin;
Expr pattern;
};
static Pattern patterns[] = {
{false, true, Int(8, 16), "llvm.x86.sse2.padds.b",
_i8(clamp(wild_i16x_ + wild_i16x_, -128, 127))},
{false, true, Int(8, 16), "llvm.x86.sse2.psubs.b",
_i8(clamp(wild_i16x_ - wild_i16x_, -128, 127))},
{false, true, UInt(8, 16), "llvm.x86.sse2.paddus.b",
_u8(min(wild_u16x_ + wild_u16x_, 255))},
{false, true, UInt(8, 16), "llvm.x86.sse2.psubus.b",
_u8(max(wild_i16x_ - wild_i16x_, 0))},
{false, true, Int(16, 8), "llvm.x86.sse2.padds.w",
_i16(clamp(wild_i32x_ + wild_i32x_, -32768, 32767))},
{false, true, Int(16, 8), "llvm.x86.sse2.psubs.w",
_i16(clamp(wild_i32x_ - wild_i32x_, -32768, 32767))},
{false, true, UInt(16, 8), "llvm.x86.sse2.paddus.w",
_u16(min(wild_u32x_ + wild_u32x_, 65535))},
{false, true, UInt(16, 8), "llvm.x86.sse2.psubus.w",
_u16(max(wild_i32x_ - wild_i32x_, 0))},
{false, true, Int(16, 8), "llvm.x86.sse2.pmulh.w",
_i16((wild_i32x_ * wild_i32x_) / 65536)},
{false, true, UInt(16, 8), "llvm.x86.sse2.pmulhu.w",
_u16((wild_u32x_ * wild_u32x_) / 65536)},
{false, true, UInt(8, 16), "llvm.x86.sse2.pavg.b",
_u8(((wild_u16x_ + wild_u16x_) + 1) / 2)},
{false, true, UInt(16, 8), "llvm.x86.sse2.pavg.w",
_u16(((wild_u32x_ + wild_u32x_) + 1) / 2)},
{false, false, Int(16, 8), "packssdwx8",
_i16(clamp(wild_i32x_, -32768, 32767))},
{false, false, Int(8, 16), "packsswbx16",
_i8(clamp(wild_i16x_, -128, 127))},
{false, false, UInt(8, 16), "packuswbx16",
_u8(clamp(wild_i16x_, 0, 255))},
{true, false, UInt(16, 8), "packusdwx8",
_u16(clamp(wild_i32x_, 0, 65535))}
};
for (size_t i = 0; i < sizeof(patterns)/sizeof(patterns[0]); i++) {
const Pattern &pattern = patterns[i];
if (!target.has_feature(Target::SSE41) && pattern.needs_sse_41) {
continue;
}
if (expr_match(pattern.pattern, op, matches)) {
bool match = true;
if (pattern.wide_op) {
// Try to narrow the matches to the target type.
for (size_t i = 0; i < matches.size(); i++) {
matches[i] = lossless_cast(op->type, matches[i]);
if (!matches[i].defined()) match = false;
}
}
if (match) {
value = call_intrin(op->type, pattern.type.lanes(), pattern.intrin, matches);
return;
}
}
}
#if LLVM_VERSION >= 38
// Workaround for https://llvm.org/bugs/show_bug.cgi?id=24512
// LLVM uses a numerically unstable method for vector
// uint32->float conversion before AVX.
if (op->value.type().element_of() == UInt(32) &&
op->type.is_float() &&
op->type.is_vector() &&
!target.has_feature(Target::AVX)) {
Type signed_type = Int(32, op->type.lanes());
// Convert the top 31 bits to float using the signed version
Expr top_bits = cast(signed_type, op->value / 2);
top_bits = cast(op->type, top_bits);
// Convert the bottom bit
Expr bottom_bit = cast(signed_type, op->value % 2);
bottom_bit = cast(op->type, bottom_bit);
// Recombine as floats
codegen(top_bits + top_bits + bottom_bit);
return;
}
#endif
CodeGen_Posix::visit(op);
}
ast_result_t pass_expr(ast_t** astp, pass_opt_t* options)
{
typecheck_t* t = &options->check;
ast_t* ast = *astp;
bool r = true;
switch(ast_id(ast))
{
case TK_NOMINAL: r = expr_nominal(options, astp); break;
case TK_FVAR:
case TK_FLET:
case TK_PARAM: r = expr_field(options, ast); break;
case TK_NEW:
case TK_BE:
case TK_FUN: r = expr_fun(options, ast); break;
case TK_SEQ: r = expr_seq(ast); break;
case TK_VAR:
case TK_LET: r = expr_local(t, ast); break;
case TK_BREAK: r = expr_break(t, ast); break;
case TK_CONTINUE: r = expr_continue(t, ast); break;
case TK_RETURN: r = expr_return(options, ast); break;
case TK_IS:
case TK_ISNT: r = expr_identity(options, ast); break;
case TK_ASSIGN: r = expr_assign(options, ast); break;
case TK_CONSUME: r = expr_consume(t, ast); break;
case TK_RECOVER: r = expr_recover(ast); break;
case TK_DOT: r = expr_dot(options, astp); break;
case TK_TILDE: r = expr_tilde(options, astp); break;
case TK_QUALIFY: r = expr_qualify(options, astp); break;
case TK_CALL: r = expr_call(options, astp); break;
case TK_IF: r = expr_if(options, ast); break;
case TK_WHILE: r = expr_while(options, ast); break;
case TK_REPEAT: r = expr_repeat(options, ast); break;
case TK_TRY_NO_CHECK:
case TK_TRY: r = expr_try(options, ast); break;
case TK_MATCH: r = expr_match(options, ast); break;
case TK_CASES: r = expr_cases(ast); break;
case TK_CASE: r = expr_case(options, ast); break;
case TK_TUPLE: r = expr_tuple(ast); break;
case TK_ARRAY: r = expr_array(options, astp); break;
case TK_REFERENCE: r = expr_reference(options, astp); break;
case TK_THIS: r = expr_this(options, ast); break;
case TK_TRUE:
case TK_FALSE: r = expr_literal(options, ast, "Bool"); break;
case TK_ERROR: r = expr_error(ast); break;
case TK_COMPILER_INTRINSIC:
r = expr_compiler_intrinsic(t, ast); break;
case TK_POSITIONALARGS:
case TK_NAMEDARGS:
case TK_NAMEDARG:
case TK_UPDATEARG: ast_inheritflags(ast); break;
case TK_AMP: r = expr_addressof(options, ast); break;
case TK_DONTCARE: r = expr_dontcare(ast); break;
case TK_INT:
// Integer literals can be integers or floats
make_literal_type(ast);
break;
case TK_FLOAT:
make_literal_type(ast);
break;
case TK_STRING:
if(ast_id(ast_parent(ast)) == TK_PACKAGE)
return AST_OK;
r = expr_literal(options, ast, "String");
break;
case TK_FFICALL:
return expr_ffi(options, ast);
default: {}
}
if(!r)
{
assert(get_error_count() > 0);
return AST_ERROR;
}
// Can't use ast here, it might have changed
symtab_t* symtab = ast_get_symtab(*astp);
if(symtab != NULL && !symtab_check_all_defined(symtab))
return AST_ERROR;
return AST_OK;
}
