18#include "ExprConfig.h"
22#ifdef SEEXPR_ENABLE_LLVM
23#include <llvm/Config/llvm-config.h>
24#include <llvm/Support/Compiler.h>
36#ifdef SEEXPR_ENABLE_LLVM
45 class LLVMEvaluationContext {
47 typedef void (*FunctionPtr)(T *,
char **, uint32_t);
48 typedef void (*FunctionPtrMultiple)(
char **, uint32_t, uint32_t, uint32_t);
49 FunctionPtr functionPtr;
50 FunctionPtrMultiple functionPtrMultiple;
54 LLVMEvaluationContext(
const LLVMEvaluationContext &) =
delete;
55 LLVMEvaluationContext &operator=(
const LLVMEvaluationContext &) =
delete;
56 ~LLVMEvaluationContext() {
delete[] resultData; }
57 LLVMEvaluationContext() : functionPtr(nullptr), resultData(nullptr) {}
58 void init(
void *fp,
void *fpLoop,
int dim) {
60 functionPtr =
reinterpret_cast<FunctionPtr
>(fp);
61 functionPtrMultiple =
reinterpret_cast<FunctionPtrMultiple
>(fpLoop);
62 resultData =
new T[dim];
65 if (resultData)
delete[] resultData;
66 functionPtr =
nullptr;
69 const T *operator()(VarBlock *varBlock) {
70 assert(functionPtr && resultData);
71 functionPtr(resultData, varBlock ? varBlock->data() :
nullptr, varBlock ? varBlock->indirectIndex : 0);
74 void operator()(VarBlock *varBlock,
size_t outputVarBlockOffset,
size_t rangeStart,
size_t rangeEnd) {
75 assert(functionPtr && resultData);
76 functionPtrMultiple(varBlock ? varBlock->data() :
nullptr, outputVarBlockOffset, rangeStart, rangeEnd);
79 std::unique_ptr<LLVMEvaluationContext<double>> _llvmEvalFP;
80 std::unique_ptr<LLVMEvaluationContext<char *>> _llvmEvalStr;
82 std::unique_ptr<llvm::LLVMContext> _llvmContext;
83 std::unique_ptr<llvm::ExecutionEngine> TheExecutionEngine;
88 const char *
evalStr(VarBlock *varBlock) {
return *(*_llvmEvalStr)(varBlock); }
89 const double *
evalFP(VarBlock *varBlock) {
return (*_llvmEvalFP)(varBlock); }
91 void evalMultiple(VarBlock *varBlock, uint32_t outputVarBlockOffset, uint32_t rangeStart, uint32_t rangeEnd) {
92 return (*_llvmEvalFP)(varBlock, outputVarBlockOffset, rangeStart, rangeEnd);
99 bool prepLLVM(ExprNode *parseTree, ExprType desiredReturnType) {
100 using namespace llvm;
101 InitializeNativeTarget();
102 InitializeNativeTargetAsmPrinter();
103 InitializeNativeTargetAsmParser();
105 std::string uniqueName = getUniqueName();
108 _llvmContext.reset(
new LLVMContext());
110 std::unique_ptr<Module> TheModule(
new Module(uniqueName +
"_module", *_llvmContext));
113 Type *i8PtrTy = Type::getInt8PtrTy(*_llvmContext);
114 PointerType *i8PtrPtrTy = PointerType::getUnqual(i8PtrTy);
115 PointerType *i8PtrPtrPtrTy = PointerType::getUnqual(i8PtrPtrTy);
116 Type *i32Ty = Type::getInt32Ty(*_llvmContext);
117 Type *i32PtrTy = Type::getInt32PtrTy(*_llvmContext);
118 Type *i64Ty = Type::getInt64Ty(*_llvmContext);
119 Type *doublePtrTy = Type::getDoublePtrTy(*_llvmContext);
120 PointerType *doublePtrPtrTy = PointerType::getUnqual(doublePtrTy);
121 Type *voidTy = Type::getVoidTy(*_llvmContext);
124 Function *SeExpr2LLVMEvalCustomFunctionFunc =
nullptr;
125 Function *SeExpr2LLVMEvalFPVarRefFunc =
nullptr;
126 Function *SeExpr2LLVMEvalStrVarRefFunc =
nullptr;
127 Function *SeExpr2LLVMEvalstrlenFunc =
nullptr;
128 Function *SeExpr2LLVMEvalmallocFunc =
nullptr;
129 Function *SeExpr2LLVMEvalfreeFunc =
nullptr;
130 Function *SeExpr2LLVMEvalmemsetFunc =
nullptr;
131 Function *SeExpr2LLVMEvalstrcatFunc =
nullptr;
134 FunctionType *FT = FunctionType::get(voidTy, {i32PtrTy, doublePtrTy, i8PtrPtrTy, i8PtrPtrTy, i64Ty},
false);
135 SeExpr2LLVMEvalCustomFunctionFunc = Function::Create(FT, GlobalValue::ExternalLinkage,
"SeExpr2LLVMEvalCustomFunction", TheModule.get());
138 FunctionType *FT = FunctionType::get(voidTy, {i8PtrTy, doublePtrTy},
false);
139 SeExpr2LLVMEvalFPVarRefFunc = Function::Create(FT, GlobalValue::ExternalLinkage,
"SeExpr2LLVMEvalFPVarRef", TheModule.get());
142 FunctionType *FT = FunctionType::get(voidTy, {i8PtrTy, i8PtrPtrTy},
false);
143 SeExpr2LLVMEvalStrVarRefFunc = Function::Create(FT, GlobalValue::ExternalLinkage,
"SeExpr2LLVMEvalStrVarRef", TheModule.get());
146 FunctionType *FT = FunctionType::get(i32Ty, { i8PtrTy },
false);
147 SeExpr2LLVMEvalstrlenFunc = Function::Create(FT, Function::ExternalLinkage,
"strlen", TheModule.get());
150 FunctionType *FT = FunctionType::get(i8PtrTy, { i32Ty },
false);
151 SeExpr2LLVMEvalmallocFunc = Function::Create(FT, Function::ExternalLinkage,
"malloc", TheModule.get());
154 FunctionType *FT = FunctionType::get(voidTy, { i8PtrTy },
false);
155 SeExpr2LLVMEvalfreeFunc = Function::Create(FT, Function::ExternalLinkage,
"free", TheModule.get());
158 FunctionType *FT = FunctionType::get(voidTy, { i8PtrTy, i32Ty, i32Ty },
false);
159 SeExpr2LLVMEvalmemsetFunc = Function::Create(FT, Function::ExternalLinkage,
"memset", TheModule.get());
162 FunctionType *FT = FunctionType::get(i8PtrTy, { i8PtrTy, i8PtrTy },
false);
163 SeExpr2LLVMEvalstrcatFunc = Function::Create(FT, Function::ExternalLinkage,
"strcat", TheModule.get());
168 bool desireFP = desiredReturnType.isFP();
170 desireFP ? doublePtrTy : i8PtrPtrTy,
174 FunctionType *FT = FunctionType::get(voidTy, ParamTys,
false);
175 Function *F = Function::Create(FT, Function::ExternalLinkage, uniqueName +
"_func", TheModule.get());
176#if LLVM_VERSION_MAJOR > 4
177 F->addAttribute(llvm::AttributeList::FunctionIndex, llvm::Attribute::AlwaysInline);
179 F->addAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::AlwaysInline);
183 const char *names[] = {
"outputPointer",
"dataBlock",
"indirectIndex"};
185 for (
auto &arg : F->args()) arg.setName(names[idx++]);
188 unsigned int dimDesired = (unsigned)desiredReturnType.dim();
189 unsigned int dimGenerated = parseTree->type().dim();
191 BasicBlock *BB = BasicBlock::Create(*_llvmContext,
"entry", F);
192 IRBuilder<> Builder(BB);
195 Value *lastVal = parseTree->codegen(Builder);
198 Value *firstArg = &*F->arg_begin();
200 if (dimGenerated > 1) {
201 Value *newLastVal = promoteToDim(lastVal, dimDesired, Builder);
202 assert(newLastVal->getType()->getVectorNumElements() >= dimDesired);
203 for (
unsigned i = 0; i < dimDesired; ++i) {
204 Value *idx = ConstantInt::get(Type::getInt64Ty(*_llvmContext), i);
205 Value *val = Builder.CreateExtractElement(newLastVal, idx);
206 Value *ptr = Builder.CreateInBoundsGEP(firstArg, idx);
207 Builder.CreateStore(val, ptr);
209 }
else if (dimGenerated == 1) {
210 for (
unsigned i = 0; i < dimDesired; ++i) {
211 Value *ptr = Builder.CreateConstInBoundsGEP1_32(
nullptr, firstArg, i);
212 Builder.CreateStore(lastVal, ptr);
215 assert(
false &&
"error. dim of FP is less than 1.");
218 Builder.CreateStore(lastVal, firstArg);
221 Builder.CreateRetVoid();
225 FunctionType *FTLOOP = FunctionType::get(voidTy, {i8PtrTy, i32Ty, i32Ty, i32Ty},
false);
226 Function *FLOOP = Function::Create(FTLOOP, Function::ExternalLinkage, uniqueName +
"_loopfunc", TheModule.get());
229 const char *names[] = {
"dataBlock",
"outputVarBlockOffset",
"rangeStart",
"rangeEnd"};
231 for (
auto &arg : FLOOP->args()) {
232 arg.setName(names[idx++]);
237 Value *dimValue = ConstantInt::get(i32Ty, dimDesired);
238 Value *oneValue = ConstantInt::get(i32Ty, 1);
241 BasicBlock *entryBlock = BasicBlock::Create(*_llvmContext,
"entry", FLOOP);
242 BasicBlock *loopCmpBlock = BasicBlock::Create(*_llvmContext,
"loopCmp", FLOOP);
243 BasicBlock *loopRepeatBlock = BasicBlock::Create(*_llvmContext,
"loopRepeat", FLOOP);
244 BasicBlock *loopIncBlock = BasicBlock::Create(*_llvmContext,
"loopInc", FLOOP);
245 BasicBlock *loopEndBlock = BasicBlock::Create(*_llvmContext,
"loopEnd", FLOOP);
246 IRBuilder<> Builder(entryBlock);
247 Builder.SetInsertPoint(entryBlock);
250 Function::arg_iterator argIterator = FLOOP->arg_begin();
251 Value *varBlockCharPtrPtrArg = &*argIterator; ++argIterator;
252 Value *outputVarBlockOffsetArg = &*argIterator; ++argIterator;
253 Value *rangeStartArg = &*argIterator; ++argIterator;
254 Value *rangeEndArg = &*argIterator; ++argIterator;
257 Value *rangeStartVar = Builder.CreateAlloca(Type::getInt32Ty(*_llvmContext), oneValue,
"rangeStartVar");
258 Value *rangeEndVar = Builder.CreateAlloca(Type::getInt32Ty(*_llvmContext), oneValue,
"rangeEndVar");
259 Value *indexVar = Builder.CreateAlloca(Type::getInt32Ty(*_llvmContext), oneValue,
"indexVar");
260 Value *outputVarBlockOffsetVar = Builder.CreateAlloca(Type::getInt32Ty(*_llvmContext), oneValue,
"outputVarBlockOffsetVar");
261 Value *varBlockDoublePtrPtrVar = Builder.CreateAlloca(doublePtrPtrTy, oneValue,
"varBlockDoublePtrPtrVar");
262 Value *varBlockTPtrPtrVar = Builder.CreateAlloca(desireFP ==
true ? doublePtrPtrTy : i8PtrPtrPtrTy, oneValue,
"varBlockTPtrPtrVar");
265 Builder.CreateStore(Builder.CreatePointerCast(varBlockCharPtrPtrArg, doublePtrPtrTy,
"varBlockAsDoublePtrPtr"), varBlockDoublePtrPtrVar);
266 Builder.CreateStore(Builder.CreatePointerCast(varBlockCharPtrPtrArg, desireFP ? doublePtrPtrTy : i8PtrPtrPtrTy,
"varBlockAsTPtrPtr"), varBlockTPtrPtrVar);
267 Builder.CreateStore(rangeStartArg, rangeStartVar);
268 Builder.CreateStore(rangeEndArg, rangeEndVar);
269 Builder.CreateStore(outputVarBlockOffsetArg, outputVarBlockOffsetVar);
272 Value *outputBasePtrPtr = Builder.CreateGEP(
nullptr, Builder.CreateLoad(varBlockTPtrPtrVar), outputVarBlockOffsetArg,
"outputBasePtrPtr");
273 Value *outputBasePtr = Builder.CreateLoad(outputBasePtrPtr,
"outputBasePtr");
274 Builder.CreateStore(Builder.CreateLoad(rangeStartVar), indexVar);
276 Builder.CreateBr(loopCmpBlock);
277 Builder.SetInsertPoint(loopCmpBlock);
278 Value *cond = Builder.CreateICmpULT(Builder.CreateLoad(indexVar), Builder.CreateLoad(rangeEndVar));
279 Builder.CreateCondBr(cond, loopRepeatBlock, loopEndBlock);
281 Builder.SetInsertPoint(loopRepeatBlock);
282 Value *myOutputPtr = Builder.CreateGEP(
nullptr, outputBasePtr, Builder.CreateMul(dimValue, Builder.CreateLoad(indexVar)));
283 Builder.CreateCall(F, {myOutputPtr, Builder.CreateLoad(varBlockDoublePtrPtrVar), Builder.CreateLoad(indexVar)});
285 Builder.CreateBr(loopIncBlock);
287 Builder.SetInsertPoint(loopIncBlock);
288 Builder.CreateStore(Builder.CreateAdd(Builder.CreateLoad(indexVar), oneValue), indexVar);
289 Builder.CreateBr(loopCmpBlock);
291 Builder.SetInsertPoint(loopEndBlock);
292 Builder.CreateRetVoid();
297 std::cerr <<
"Pre verified LLVM byte code " << std::endl;
298 TheModule->print(llvm::errs(),
nullptr);
307 Module *altModule = TheModule.get();
309 TheExecutionEngine.reset(EngineBuilder(std::move(TheModule))
310 .setErrorStr(&ErrStr)
312 .setOptLevel(CodeGenOpt::Aggressive)
315 altModule->setDataLayout(TheExecutionEngine->getDataLayout());
321 TheExecutionEngine->addGlobalMapping(SeExpr2LLVMEvalstrlenFunc, (
void *)strlen);
322 TheExecutionEngine->addGlobalMapping(SeExpr2LLVMEvalstrcatFunc, (
void *)strcat);
323 TheExecutionEngine->addGlobalMapping(SeExpr2LLVMEvalmemsetFunc, (
void *)memset);
324 TheExecutionEngine->addGlobalMapping(SeExpr2LLVMEvalmallocFunc, (
void *)malloc);
325 TheExecutionEngine->addGlobalMapping(SeExpr2LLVMEvalfreeFunc, (
void *)free);
328 std::string errorStr;
329 llvm::raw_string_ostream raw(errorStr);
330 if (llvm::verifyModule(*altModule, &raw)) {
331 parseTree->addError(raw.str());
336 llvm::PassManagerBuilder builder;
337 std::unique_ptr<llvm::legacy::PassManager> pm(
new llvm::legacy::PassManager);
338 std::unique_ptr<llvm::legacy::FunctionPassManager> fpm(
new llvm::legacy::FunctionPassManager(altModule));
339 builder.OptLevel = 3;
340#if (LLVM_VERSION_MAJOR >= 4)
341 builder.Inliner = llvm::createAlwaysInlinerLegacyPass();
343 builder.Inliner = llvm::createAlwaysInlinerPass();
345 builder.populateModulePassManager(*pm);
347 builder.populateFunctionPassManager(*fpm);
354 if (!TheExecutionEngine) {
355 fprintf(stderr,
"Could not create ExecutionEngine: %s\n", ErrStr.c_str());
359 TheExecutionEngine->finalizeObject();
360 void *fp = TheExecutionEngine->getPointerToFunction(F);
361 void *fpLoop = TheExecutionEngine->getPointerToFunction(FLOOP);
363 _llvmEvalFP.reset(
new LLVMEvaluationContext<double>);
364 _llvmEvalFP->init(fp, fpLoop, dimDesired);
366 _llvmEvalStr.reset(
new LLVMEvaluationContext<char *>);
367 _llvmEvalStr->init(fp, fpLoop, dimDesired);
372 std::cerr <<
"Pre verified LLVM byte code " << std::endl;
373 altModule->print(llvm::errs(),
nullptr);
380 std::string getUniqueName()
const {
381 std::ostringstream o;
382 o << std::setbase(16) << (uint64_t)(
this);
383 return (
"_" + o.str());
390 void unsupported() {
throw std::runtime_error(
"LLVM is not enabled in build"); }
void SeExpr2LLVMEvalFPVarRef(SeExpr2::ExprVarRef *seVR, double *result)
void SeExpr2LLVMEvalCustomFunction(int *opDataArg, double *fpArg, char **strArg, void **funcdata, const SeExpr2::ExprFuncNode *node)
void SeExpr2LLVMEvalStrVarRef(SeExpr2::ExprVarRef *seVR, double *result)
Node that calls a function.
abstract class for implementing variable references
static bool debugging
Whether to debug expressions.
const char * evalStr(VarBlock *varBlock)
void evalMultiple(VarBlock *varBlock, int outputVarBlockOffset, size_t rangeStart, size_t rangeEnd)
bool prepLLVM(ExprNode *parseTree, ExprType desiredReturnType)
const double * evalFP(VarBlock *varBlock)
A thread local evaluation context. Just allocate and fill in with data.