/** * Validate a function. */ void gallivm_verify_function(struct gallivm_state *gallivm, LLVMValueRef func) { /* Verify the LLVM IR. If invalid, dump and abort */ #ifdef DEBUG if (LLVMVerifyFunction(func, LLVMPrintMessageAction)) { lp_debug_dump_value(func); assert(0); return; } #endif gallivm_optimize_function(gallivm, func); if (gallivm_debug & GALLIVM_DEBUG_IR) { /* Print the LLVM IR to stderr */ lp_debug_dump_value(func); debug_printf("\n"); } }
LLVMValueRef lp_build_intrinsic(LLVMBuilderRef builder, const char *name, LLVMTypeRef ret_type, LLVMValueRef *args, unsigned num_args, unsigned attr_mask) { LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder))); LLVMValueRef function, call; bool set_callsite_attrs = HAVE_LLVM >= 0x0400 && !(attr_mask & LP_FUNC_ATTR_LEGACY); function = LLVMGetNamedFunction(module, name); if(!function) { LLVMTypeRef arg_types[LP_MAX_FUNC_ARGS]; unsigned i; assert(num_args <= LP_MAX_FUNC_ARGS); for(i = 0; i < num_args; ++i) { assert(args[i]); arg_types[i] = LLVMTypeOf(args[i]); } function = lp_declare_intrinsic(module, name, ret_type, arg_types, num_args); /* * If llvm removes an intrinsic we use, we'll hit this abort (rather * than a call to address zero in the jited code). */ if (LLVMGetIntrinsicID(function) == 0) { _debug_printf("llvm (version 0x%x) found no intrinsic for %s, going to crash...\n", HAVE_LLVM, name); abort(); } if (!set_callsite_attrs) lp_add_func_attributes(function, attr_mask); if (gallivm_debug & GALLIVM_DEBUG_IR) { lp_debug_dump_value(function); } } call = LLVMBuildCall(builder, function, args, num_args, ""); if (set_callsite_attrs) lp_add_func_attributes(call, attr_mask); return call; }
/** * Validate and optimze a function. */ static void gallivm_optimize_function(struct gallivm_state *gallivm, LLVMValueRef func) { if (0) { debug_printf("optimizing %s...\n", LLVMGetValueName(func)); } assert(gallivm->passmgr); /* Apply optimizations to LLVM IR */ LLVMRunFunctionPassManager(gallivm->passmgr, func); if (0) { if (gallivm_debug & GALLIVM_DEBUG_IR) { /* Print the LLVM IR to stderr */ lp_debug_dump_value(func); debug_printf("\n"); } } }
void lp_build_tgsi_aos(struct gallivm_state *gallivm, const struct tgsi_token *tokens, struct lp_type type, const unsigned char swizzles[4], LLVMValueRef consts_ptr, const LLVMValueRef *inputs, LLVMValueRef *outputs, struct lp_build_sampler_aos *sampler, const struct tgsi_shader_info *info) { struct lp_build_tgsi_aos_context bld; struct tgsi_parse_context parse; uint num_immediates = 0; unsigned chan; int pc = 0; /* Setup build context */ memset(&bld, 0, sizeof bld); lp_build_context_init(&bld.bld_base.base, gallivm, type); lp_build_context_init(&bld.bld_base.uint_bld, gallivm, lp_uint_type(type)); lp_build_context_init(&bld.bld_base.int_bld, gallivm, lp_int_type(type)); lp_build_context_init(&bld.int_bld, gallivm, lp_int_type(type)); for (chan = 0; chan < 4; ++chan) { bld.swizzles[chan] = swizzles[chan]; bld.inv_swizzles[swizzles[chan]] = chan; } bld.inputs = inputs; bld.outputs = outputs; bld.consts_ptr = consts_ptr; bld.sampler = sampler; bld.indirect_files = info->indirect_files; bld.bld_base.emit_swizzle = swizzle_aos; bld.bld_base.info = info; bld.bld_base.emit_fetch_funcs[TGSI_FILE_CONSTANT] = emit_fetch_constant; bld.bld_base.emit_fetch_funcs[TGSI_FILE_IMMEDIATE] = emit_fetch_immediate; bld.bld_base.emit_fetch_funcs[TGSI_FILE_INPUT] = emit_fetch_input; bld.bld_base.emit_fetch_funcs[TGSI_FILE_TEMPORARY] = emit_fetch_temporary; /* Set opcode actions */ lp_set_default_actions_cpu(&bld.bld_base); if (!lp_bld_tgsi_list_init(&bld.bld_base)) { return; } tgsi_parse_init(&parse, tokens); while (!tgsi_parse_end_of_tokens(&parse)) { tgsi_parse_token(&parse); switch(parse.FullToken.Token.Type) { case TGSI_TOKEN_TYPE_DECLARATION: /* Inputs already interpolated */ lp_emit_declaration_aos(&bld, &parse.FullToken.FullDeclaration); break; case TGSI_TOKEN_TYPE_INSTRUCTION: /* save expanded instruction */ lp_bld_tgsi_add_instruction(&bld.bld_base, &parse.FullToken.FullInstruction); break; case TGSI_TOKEN_TYPE_IMMEDIATE: /* simply copy the immediate values into the next immediates[] slot */ { const uint size = parse.FullToken.FullImmediate.Immediate.NrTokens - 1; float imm[4]; assert(size <= 4); assert(num_immediates < LP_MAX_TGSI_IMMEDIATES); for (chan = 0; chan < 4; ++chan) { imm[chan] = 0.0f; } for (chan = 0; chan < size; ++chan) { unsigned swizzle = bld.swizzles[chan]; imm[swizzle] = parse.FullToken.FullImmediate.u[chan].Float; } bld.immediates[num_immediates] = lp_build_const_aos(gallivm, type, imm[0], imm[1], imm[2], imm[3], NULL); num_immediates++; } break; case TGSI_TOKEN_TYPE_PROPERTY: break; default: assert(0); } } while (pc != -1) { struct tgsi_full_instruction *instr = bld.bld_base.instructions + pc; const struct tgsi_opcode_info *opcode_info = tgsi_get_opcode_info(instr->Instruction.Opcode); if (!lp_emit_instruction_aos(&bld, instr, opcode_info, &pc)) _debug_printf("warning: failed to translate tgsi opcode %s to LLVM\n", opcode_info->mnemonic); } if (0) { LLVMBasicBlockRef block = LLVMGetInsertBlock(gallivm->builder); LLVMValueRef function = LLVMGetBasicBlockParent(block); debug_printf("11111111111111111111111111111 \n"); tgsi_dump(tokens, 0); lp_debug_dump_value(function); debug_printf("2222222222222222222222222222 \n"); } tgsi_parse_free(&parse); FREE(bld.bld_base.instructions); if (0) { LLVMModuleRef module = LLVMGetGlobalParent( LLVMGetBasicBlockParent(LLVMGetInsertBlock(gallivm->builder))); LLVMDumpModule(module); } }
/** * Generate the runtime callable function for the whole fragment pipeline. * Note that the function which we generate operates on a block of 16 * pixels at at time. The block contains 2x2 quads. Each quad contains * 2x2 pixels. */ static void generate_fragment(struct llvmpipe_context *lp, struct lp_fragment_shader *shader, struct lp_fragment_shader_variant *variant, unsigned partial_mask) { struct llvmpipe_screen *screen = llvmpipe_screen(lp->pipe.screen); const struct lp_fragment_shader_variant_key *key = &variant->key; char func_name[256]; struct lp_type fs_type; struct lp_type blend_type; LLVMTypeRef fs_elem_type; LLVMTypeRef fs_int_vec_type; LLVMTypeRef blend_vec_type; LLVMTypeRef arg_types[11]; LLVMTypeRef func_type; LLVMValueRef context_ptr; LLVMValueRef x; LLVMValueRef y; LLVMValueRef a0_ptr; LLVMValueRef dadx_ptr; LLVMValueRef dady_ptr; LLVMValueRef color_ptr_ptr; LLVMValueRef depth_ptr; LLVMValueRef mask_input; LLVMValueRef counter = NULL; LLVMBasicBlockRef block; LLVMBuilderRef builder; struct lp_build_sampler_soa *sampler; struct lp_build_interp_soa_context interp; LLVMValueRef fs_mask[LP_MAX_VECTOR_LENGTH]; LLVMValueRef fs_out_color[PIPE_MAX_COLOR_BUFS][NUM_CHANNELS][LP_MAX_VECTOR_LENGTH]; LLVMValueRef blend_mask; LLVMValueRef function; LLVMValueRef facing; unsigned num_fs; unsigned i; unsigned chan; unsigned cbuf; /* TODO: actually pick these based on the fs and color buffer * characteristics. */ memset(&fs_type, 0, sizeof fs_type); fs_type.floating = TRUE; /* floating point values */ fs_type.sign = TRUE; /* values are signed */ fs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ fs_type.width = 32; /* 32-bit float */ fs_type.length = 4; /* 4 elements per vector */ num_fs = 4; /* number of quads per block */ memset(&blend_type, 0, sizeof blend_type); blend_type.floating = FALSE; /* values are integers */ blend_type.sign = FALSE; /* values are unsigned */ blend_type.norm = TRUE; /* values are in [0,1] or [-1,1] */ blend_type.width = 8; /* 8-bit ubyte values */ blend_type.length = 16; /* 16 elements per vector */ /* * Generate the function prototype. Any change here must be reflected in * lp_jit.h's lp_jit_frag_func function pointer type, and vice-versa. */ fs_elem_type = lp_build_elem_type(fs_type); fs_int_vec_type = lp_build_int_vec_type(fs_type); blend_vec_type = lp_build_vec_type(blend_type); util_snprintf(func_name, sizeof(func_name), "fs%u_variant%u_%s", shader->no, variant->no, partial_mask ? "partial" : "whole"); arg_types[0] = screen->context_ptr_type; /* context */ arg_types[1] = LLVMInt32Type(); /* x */ arg_types[2] = LLVMInt32Type(); /* y */ arg_types[3] = LLVMFloatType(); /* facing */ arg_types[4] = LLVMPointerType(fs_elem_type, 0); /* a0 */ arg_types[5] = LLVMPointerType(fs_elem_type, 0); /* dadx */ arg_types[6] = LLVMPointerType(fs_elem_type, 0); /* dady */ arg_types[7] = LLVMPointerType(LLVMPointerType(blend_vec_type, 0), 0); /* color */ arg_types[8] = LLVMPointerType(fs_int_vec_type, 0); /* depth */ arg_types[9] = LLVMInt32Type(); /* mask_input */ arg_types[10] = LLVMPointerType(LLVMInt32Type(), 0);/* counter */ func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0); function = LLVMAddFunction(screen->module, func_name, func_type); LLVMSetFunctionCallConv(function, LLVMCCallConv); variant->function[partial_mask] = function; /* XXX: need to propagate noalias down into color param now we are * passing a pointer-to-pointer? */ for(i = 0; i < Elements(arg_types); ++i) if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) LLVMAddAttribute(LLVMGetParam(function, i), LLVMNoAliasAttribute); context_ptr = LLVMGetParam(function, 0); x = LLVMGetParam(function, 1); y = LLVMGetParam(function, 2); facing = LLVMGetParam(function, 3); a0_ptr = LLVMGetParam(function, 4); dadx_ptr = LLVMGetParam(function, 5); dady_ptr = LLVMGetParam(function, 6); color_ptr_ptr = LLVMGetParam(function, 7); depth_ptr = LLVMGetParam(function, 8); mask_input = LLVMGetParam(function, 9); lp_build_name(context_ptr, "context"); lp_build_name(x, "x"); lp_build_name(y, "y"); lp_build_name(a0_ptr, "a0"); lp_build_name(dadx_ptr, "dadx"); lp_build_name(dady_ptr, "dady"); lp_build_name(color_ptr_ptr, "color_ptr_ptr"); lp_build_name(depth_ptr, "depth"); lp_build_name(mask_input, "mask_input"); if (key->occlusion_count) { counter = LLVMGetParam(function, 10); lp_build_name(counter, "counter"); } /* * Function body */ block = LLVMAppendBasicBlock(function, "entry"); builder = LLVMCreateBuilder(); LLVMPositionBuilderAtEnd(builder, block); /* * The shader input interpolation info is not explicitely baked in the * shader key, but everything it derives from (TGSI, and flatshade) is * already included in the shader key. */ lp_build_interp_soa_init(&interp, lp->num_inputs, lp->inputs, builder, fs_type, a0_ptr, dadx_ptr, dady_ptr, x, y); /* code generated texture sampling */ sampler = lp_llvm_sampler_soa_create(key->sampler, context_ptr); /* loop over quads in the block */ for(i = 0; i < num_fs; ++i) { LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), i, 0); LLVMValueRef out_color[PIPE_MAX_COLOR_BUFS][NUM_CHANNELS]; LLVMValueRef depth_ptr_i; if(i != 0) lp_build_interp_soa_update(&interp, i); depth_ptr_i = LLVMBuildGEP(builder, depth_ptr, &index, 1, ""); generate_fs(lp, shader, key, builder, fs_type, context_ptr, i, &interp, sampler, &fs_mask[i], /* output */ out_color, depth_ptr_i, facing, partial_mask, mask_input, counter); for(cbuf = 0; cbuf < key->nr_cbufs; cbuf++) for(chan = 0; chan < NUM_CHANNELS; ++chan) fs_out_color[cbuf][chan][i] = out_color[cbuf][chan]; } sampler->destroy(sampler); /* Loop over color outputs / color buffers to do blending. */ for(cbuf = 0; cbuf < key->nr_cbufs; cbuf++) { LLVMValueRef color_ptr; LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), cbuf, 0); LLVMValueRef blend_in_color[NUM_CHANNELS]; unsigned rt; /* * Convert the fs's output color and mask to fit to the blending type. */ for(chan = 0; chan < NUM_CHANNELS; ++chan) { lp_build_conv(builder, fs_type, blend_type, fs_out_color[cbuf][chan], num_fs, &blend_in_color[chan], 1); lp_build_name(blend_in_color[chan], "color%d.%c", cbuf, "rgba"[chan]); } if (partial_mask || !variant->opaque) { lp_build_conv_mask(builder, fs_type, blend_type, fs_mask, num_fs, &blend_mask, 1); } else { blend_mask = lp_build_const_int_vec(blend_type, ~0); } color_ptr = LLVMBuildLoad(builder, LLVMBuildGEP(builder, color_ptr_ptr, &index, 1, ""), ""); lp_build_name(color_ptr, "color_ptr%d", cbuf); /* which blend/colormask state to use */ rt = key->blend.independent_blend_enable ? cbuf : 0; /* * Blending. */ generate_blend(&key->blend, rt, builder, blend_type, context_ptr, blend_mask, blend_in_color, color_ptr); } #ifdef PIPE_ARCH_X86 /* Avoid corrupting the FPU stack on 32bit OSes. */ lp_build_intrinsic(builder, "llvm.x86.mmx.emms", LLVMVoidType(), NULL, 0); #endif LLVMBuildRetVoid(builder); LLVMDisposeBuilder(builder); /* Verify the LLVM IR. If invalid, dump and abort */ #ifdef DEBUG if(LLVMVerifyFunction(function, LLVMPrintMessageAction)) { if (1) lp_debug_dump_value(function); abort(); } #endif /* Apply optimizations to LLVM IR */ LLVMRunFunctionPassManager(screen->pass, function); if (gallivm_debug & GALLIVM_DEBUG_IR) { /* Print the LLVM IR to stderr */ lp_debug_dump_value(function); debug_printf("\n"); } /* * Translate the LLVM IR into machine code. */ { void *f = LLVMGetPointerToGlobal(screen->engine, function); variant->jit_function[partial_mask] = (lp_jit_frag_func)pointer_to_func(f); if (gallivm_debug & GALLIVM_DEBUG_ASM) { lp_disassemble(f); } lp_func_delete_body(function); } }