$(TARGET:%=$(ObjDir)/%GenMCCodeEmitter.inc.tmp): \
$(ObjDir)/%GenMCCodeEmitter.inc.tmp: %.td $(ObjDir)/.dir $(LLVM_TBLGEN)
$(Echo) "Building $(<F) MC code emitter with tblgen"
- $(Verb) $(LLVMTableGen) -gen-emitter -o $(call SYSPATH, $@) $<
+ $(Verb) $(LLVMTableGen) -gen-emitter -mc-emitter -o $(call SYSPATH, $@) $<
$(TARGET:%=$(ObjDir)/%GenMCPseudoLowering.inc.tmp): \
$(ObjDir)/%GenMCPseudoLowering.inc.tmp: %.td $(ObjDir)/.dir $(LLVM_TBLGEN)
$(Echo) "Building $(<F) MC Pseudo instruction expander with tblgen"
$(Verb) $(LLVMTableGen) -gen-pseudo-lowering -o $(call SYSPATH, $@) $<
+$(TARGET:%=$(ObjDir)/%GenCodeEmitter.inc.tmp): \
+$(ObjDir)/%GenCodeEmitter.inc.tmp: %.td $(ObjDir)/.dir $(LLVM_TBLGEN)
+ $(Echo) "Building $(<F) code emitter with tblgen"
+ $(Verb) $(LLVMTableGen) -gen-emitter -o $(call SYSPATH, $@) $<
+
$(TARGET:%=$(ObjDir)/%GenDAGISel.inc.tmp): \
$(ObjDir)/%GenDAGISel.inc.tmp : %.td $(ObjDir)/.dir $(LLVM_TBLGEN)
$(Echo) "Building $(<F) DAG instruction selector implementation with tblgen"
LEVEL := ../../..
LIBRARYNAME := llvm_executionengine
-UsedComponents := executionengine mcjit interpreter native
+UsedComponents := executionengine jit interpreter native
UsedOcamlInterfaces := llvm llvm_target
include ../Makefile.ocaml
/* Force the LLVM interpreter and JIT to be linked in. */
void llvm_initialize(void) {
LLVMLinkInInterpreter();
- LLVMLinkInMCJIT();
+ LLVMLinkInJIT();
}
/* unit -> bool */
**Output**: C++ code, implementing the target's CodeEmitter
class by overriding the virtual functions as ``<Target>CodeEmitter::function()``.
-**Usage**: Used to include directly at the end of ``<Target>MCCodeEmitter.cpp``.
+**Usage**: Used to include directly at the end of ``<Target>CodeEmitter.cpp``, and
+with option `-mc-emitter` to be included in ``<Target>MCCodeEmitter.cpp``.
RegisterInfo
------------
#include "BrainF.h"
#include "llvm/Bitcode/ReaderWriter.h"
-#include "llvm/ExecutionEngine/ExecutionEngine.h"
#include "llvm/ExecutionEngine/GenericValue.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/CommandLine.h"
BitWriter
Core
ExecutionEngine
+ JIT
MC
Support
nativecodegen
// Build engine with JIT
llvm::EngineBuilder factory(module);
factory.setEngineKind(llvm::EngineKind::JIT);
+ factory.setAllocateGVsWithCode(false);
factory.setTargetOptions(Opts);
factory.setMCJITMemoryManager(MemMgr);
+ factory.setUseMCJIT(true);
llvm::ExecutionEngine *executionEngine = factory.create();
{
Core
ExecutionEngine
Interpreter
+ JIT
MC
Support
nativecodegen
#include "llvm/IR/Verifier.h"
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/Interpreter.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Instructions.h"
Core
ExecutionEngine
Interpreter
+ JIT
MC
Support
nativecodegen
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/Interpreter.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
// Import result of execution:
outs() << "Result: " << gv.IntVal << "\n";
+ EE->freeMachineCodeForFunction(FooF);
delete EE;
llvm_shutdown();
return 0;
Core
ExecutionEngine
InstCombine
+ JIT
MC
ScalarOpts
Support
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
Core
ExecutionEngine
InstCombine
+ JIT
MC
ScalarOpts
Support
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
Core
ExecutionEngine
InstCombine
+ JIT
MC
ScalarOpts
Support
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
Core
ExecutionEngine
InstCombine
+ JIT
MC
ScalarOpts
Support
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
std::string ErrStr;
ExecutionEngine *NewEngine = EngineBuilder(M)
.setErrorStr(&ErrStr)
+ .setUseMCJIT(true)
.setMCJITMemoryManager(new HelpingMemoryManager(this))
.create();
if (!NewEngine) {
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/ExecutionEngine/MCJIT.h"
#include "llvm/ExecutionEngine/ObjectCache.h"
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
cl::desc("Dump IR from modules to stderr on shutdown"),
cl::init(false));
+ cl::opt<bool> UseMCJIT(
+ "use-mcjit", cl::desc("Use the MCJIT execution engine"),
+ cl::init(true));
+
cl::opt<bool> EnableLazyCompilation(
"enable-lazy-compilation", cl::desc("Enable lazy compilation when using the MCJIT engine"),
cl::init(true));
virtual void dump();
};
+//===----------------------------------------------------------------------===//
+// Helper class for JIT execution engine
+//===----------------------------------------------------------------------===//
+
+class JITHelper : public BaseHelper {
+public:
+ JITHelper(LLVMContext &Context) {
+ // Make the module, which holds all the code.
+ if (!InputIR.empty()) {
+ TheModule = parseInputIR(InputIR, Context);
+ } else {
+ TheModule = new Module("my cool jit", Context);
+ }
+
+ // Create the JIT. This takes ownership of the module.
+ std::string ErrStr;
+ TheExecutionEngine = EngineBuilder(TheModule).setErrorStr(&ErrStr).create();
+ if (!TheExecutionEngine) {
+ fprintf(stderr, "Could not create ExecutionEngine: %s\n", ErrStr.c_str());
+ exit(1);
+ }
+
+ TheFPM = new FunctionPassManager(TheModule);
+
+ // Set up the optimizer pipeline. Start with registering info about how the
+ // target lays out data structures.
+ TheFPM->add(new DataLayout(*TheExecutionEngine->getDataLayout()));
+ // Provide basic AliasAnalysis support for GVN.
+ TheFPM->add(createBasicAliasAnalysisPass());
+ // Promote allocas to registers.
+ TheFPM->add(createPromoteMemoryToRegisterPass());
+ // Do simple "peephole" optimizations and bit-twiddling optzns.
+ TheFPM->add(createInstructionCombiningPass());
+ // Reassociate expressions.
+ TheFPM->add(createReassociatePass());
+ // Eliminate Common SubExpressions.
+ TheFPM->add(createGVNPass());
+ // Simplify the control flow graph (deleting unreachable blocks, etc).
+ TheFPM->add(createCFGSimplificationPass());
+
+ TheFPM->doInitialization();
+ }
+
+ virtual ~JITHelper() {
+ if (TheFPM)
+ delete TheFPM;
+ if (TheExecutionEngine)
+ delete TheExecutionEngine;
+ }
+
+ virtual Function *getFunction(const std::string FnName) {
+ assert(TheModule);
+ return TheModule->getFunction(FnName);
+ }
+
+ virtual Module *getModuleForNewFunction() {
+ assert(TheModule);
+ return TheModule;
+ }
+
+ virtual void *getPointerToFunction(Function* F) {
+ assert(TheExecutionEngine);
+ return TheExecutionEngine->getPointerToFunction(F);
+ }
+
+ virtual void *getPointerToNamedFunction(const std::string &Name) {
+ return TheExecutionEngine->getPointerToNamedFunction(Name);
+ }
+
+ virtual void runFPM(Function &F) {
+ assert(TheFPM);
+ TheFPM->run(F);
+ }
+
+ virtual void closeCurrentModule() {
+ // This should never be called for JIT
+ assert(false);
+ }
+
+ virtual void dump() {
+ assert(TheModule);
+ TheModule->dump();
+ }
+
+private:
+ Module *TheModule;
+ ExecutionEngine *TheExecutionEngine;
+ FunctionPassManager *TheFPM;
+};
+
//===----------------------------------------------------------------------===//
// MCJIT helper class
//===----------------------------------------------------------------------===//
std::string ErrStr;
ExecutionEngine *EE = EngineBuilder(M)
.setErrorStr(&ErrStr)
+ .setUseMCJIT(true)
.setMCJITMemoryManager(new HelpingMemoryManager(this))
.create();
if (!EE) {
Value *OperandV = Operand->Codegen();
if (OperandV == 0) return 0;
Function *F;
- F = TheHelper->getFunction(
- MakeLegalFunctionName(std::string("unary") + Opcode));
+ if (UseMCJIT)
+ F = TheHelper->getFunction(MakeLegalFunctionName(std::string("unary")+Opcode));
+ else
+ F = TheHelper->getFunction(std::string("unary")+Opcode);
if (F == 0)
return ErrorV("Unknown unary operator");
// If it wasn't a builtin binary operator, it must be a user defined one. Emit
// a call to it.
Function *F;
- F = TheHelper->getFunction(MakeLegalFunctionName(std::string("binary")+Op));
+ if (UseMCJIT)
+ F = TheHelper->getFunction(MakeLegalFunctionName(std::string("binary")+Op));
+ else
+ F = TheHelper->getFunction(std::string("binary")+Op);
assert(F && "binary operator not found!");
Value *Ops[] = { L, R };
Doubles, false);
std::string FnName;
- FnName = MakeLegalFunctionName(Name);
+ if (UseMCJIT)
+ FnName = MakeLegalFunctionName(Name);
+ else
+ FnName = Name;
Module* M = TheHelper->getModuleForNewFunction();
Function *F = Function::Create(FT, Function::ExternalLinkage, FnName, M);
// Validate the generated code, checking for consistency.
verifyFunction(*TheFunction);
+ // Optimize the function.
+ if (!UseMCJIT)
+ TheHelper->runFPM(*TheFunction);
+
return TheFunction;
}
static void HandleDefinition() {
if (FunctionAST *F = ParseDefinition()) {
- if (EnableLazyCompilation)
+ if (UseMCJIT && EnableLazyCompilation)
TheHelper->closeCurrentModule();
Function *LF = F->Codegen();
if (LF && VerboseOutput) {
int main(int argc, char **argv) {
InitializeNativeTarget();
- InitializeNativeTargetAsmPrinter();
- InitializeNativeTargetAsmParser();
+ if (UseMCJIT) {
+ InitializeNativeTargetAsmPrinter();
+ InitializeNativeTargetAsmParser();
+ }
LLVMContext &Context = getGlobalContext();
cl::ParseCommandLineOptions(argc, argv,
BinopPrecedence['*'] = 40; // highest.
// Make the Helper, which holds all the code.
- TheHelper = new MCJITHelper(Context);
+ if (UseMCJIT)
+ TheHelper = new MCJITHelper(Context);
+ else
+ TheHelper = new JITHelper(Context);
// Prime the first token.
if (!SuppressPrompts)
std::string ErrStr;
ExecutionEngine *NewEngine = EngineBuilder(OpenModule)
.setErrorStr(&ErrStr)
+ .setUseMCJIT(true)
.setMCJITMemoryManager(new HelpingMemoryManager(this))
.create();
if (!NewEngine) {
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
std::string ErrStr;
ExecutionEngine *NewEngine = EngineBuilder(M)
.setErrorStr(&ErrStr)
+ .setUseMCJIT(true)
.setMCJITMemoryManager(new HelpingMemoryManager(this))
.create();
if (!NewEngine) {
Core
ExecutionEngine
Interpreter
+ JIT
Support
nativecodegen
)
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/Interpreter.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Instructions.h"
* @{
*/
+void LLVMLinkInJIT(void);
void LLVMLinkInMCJIT(void);
void LLVMLinkInInterpreter(void);
--- /dev/null
+//===-- llvm/CodeGen/JITCodeEmitter.h - Code emission ----------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines an abstract interface that is used by the machine code
+// emission framework to output the code. This allows machine code emission to
+// be separated from concerns such as resolution of call targets, and where the
+// machine code will be written (memory or disk, f.e.).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_JITCODEEMITTER_H
+#define LLVM_CODEGEN_JITCODEEMITTER_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/MachineCodeEmitter.h"
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/MathExtras.h"
+#include <string>
+
+namespace llvm {
+
+class MachineBasicBlock;
+class MachineConstantPool;
+class MachineJumpTableInfo;
+class MachineFunction;
+class MachineModuleInfo;
+class MachineRelocation;
+class Value;
+class GlobalValue;
+class Function;
+
+/// JITCodeEmitter - This class defines two sorts of methods: those for
+/// emitting the actual bytes of machine code, and those for emitting auxiliary
+/// structures, such as jump tables, relocations, etc.
+///
+/// Emission of machine code is complicated by the fact that we don't (in
+/// general) know the size of the machine code that we're about to emit before
+/// we emit it. As such, we preallocate a certain amount of memory, and set the
+/// BufferBegin/BufferEnd pointers to the start and end of the buffer. As we
+/// emit machine instructions, we advance the CurBufferPtr to indicate the
+/// location of the next byte to emit. In the case of a buffer overflow (we
+/// need to emit more machine code than we have allocated space for), the
+/// CurBufferPtr will saturate to BufferEnd and ignore stores. Once the entire
+/// function has been emitted, the overflow condition is checked, and if it has
+/// occurred, more memory is allocated, and we reemit the code into it.
+///
+class JITCodeEmitter : public MachineCodeEmitter {
+ void anchor() override;
+public:
+ virtual ~JITCodeEmitter() {}
+
+ /// startFunction - This callback is invoked when the specified function is
+ /// about to be code generated. This initializes the BufferBegin/End/Ptr
+ /// fields.
+ ///
+ void startFunction(MachineFunction &F) override = 0;
+
+ /// finishFunction - This callback is invoked when the specified function has
+ /// finished code generation. If a buffer overflow has occurred, this method
+ /// returns true (the callee is required to try again), otherwise it returns
+ /// false.
+ ///
+ bool finishFunction(MachineFunction &F) override = 0;
+
+ /// allocIndirectGV - Allocates and fills storage for an indirect
+ /// GlobalValue, and returns the address.
+ virtual void *allocIndirectGV(const GlobalValue *GV,
+ const uint8_t *Buffer, size_t Size,
+ unsigned Alignment) = 0;
+
+ /// emitByte - This callback is invoked when a byte needs to be written to the
+ /// output stream.
+ ///
+ void emitByte(uint8_t B) {
+ if (CurBufferPtr != BufferEnd)
+ *CurBufferPtr++ = B;
+ }
+
+ /// emitWordLE - This callback is invoked when a 32-bit word needs to be
+ /// written to the output stream in little-endian format.
+ ///
+ void emitWordLE(uint32_t W) {
+ if (4 <= BufferEnd-CurBufferPtr) {
+ *CurBufferPtr++ = (uint8_t)(W >> 0);
+ *CurBufferPtr++ = (uint8_t)(W >> 8);
+ *CurBufferPtr++ = (uint8_t)(W >> 16);
+ *CurBufferPtr++ = (uint8_t)(W >> 24);
+ } else {
+ CurBufferPtr = BufferEnd;
+ }
+ }
+
+ /// emitWordBE - This callback is invoked when a 32-bit word needs to be
+ /// written to the output stream in big-endian format.
+ ///
+ void emitWordBE(uint32_t W) {
+ if (4 <= BufferEnd-CurBufferPtr) {
+ *CurBufferPtr++ = (uint8_t)(W >> 24);
+ *CurBufferPtr++ = (uint8_t)(W >> 16);
+ *CurBufferPtr++ = (uint8_t)(W >> 8);
+ *CurBufferPtr++ = (uint8_t)(W >> 0);
+ } else {
+ CurBufferPtr = BufferEnd;
+ }
+ }
+
+ /// emitDWordLE - This callback is invoked when a 64-bit word needs to be
+ /// written to the output stream in little-endian format.
+ ///
+ void emitDWordLE(uint64_t W) {
+ if (8 <= BufferEnd-CurBufferPtr) {
+ *CurBufferPtr++ = (uint8_t)(W >> 0);
+ *CurBufferPtr++ = (uint8_t)(W >> 8);
+ *CurBufferPtr++ = (uint8_t)(W >> 16);
+ *CurBufferPtr++ = (uint8_t)(W >> 24);
+ *CurBufferPtr++ = (uint8_t)(W >> 32);
+ *CurBufferPtr++ = (uint8_t)(W >> 40);
+ *CurBufferPtr++ = (uint8_t)(W >> 48);
+ *CurBufferPtr++ = (uint8_t)(W >> 56);
+ } else {
+ CurBufferPtr = BufferEnd;
+ }
+ }
+
+ /// emitDWordBE - This callback is invoked when a 64-bit word needs to be
+ /// written to the output stream in big-endian format.
+ ///
+ void emitDWordBE(uint64_t W) {
+ if (8 <= BufferEnd-CurBufferPtr) {
+ *CurBufferPtr++ = (uint8_t)(W >> 56);
+ *CurBufferPtr++ = (uint8_t)(W >> 48);
+ *CurBufferPtr++ = (uint8_t)(W >> 40);
+ *CurBufferPtr++ = (uint8_t)(W >> 32);
+ *CurBufferPtr++ = (uint8_t)(W >> 24);
+ *CurBufferPtr++ = (uint8_t)(W >> 16);
+ *CurBufferPtr++ = (uint8_t)(W >> 8);
+ *CurBufferPtr++ = (uint8_t)(W >> 0);
+ } else {
+ CurBufferPtr = BufferEnd;
+ }
+ }
+
+ /// emitAlignment - Move the CurBufferPtr pointer up to the specified
+ /// alignment (saturated to BufferEnd of course).
+ void emitAlignment(unsigned Alignment) {
+ if (Alignment == 0) Alignment = 1;
+ uint8_t *NewPtr = (uint8_t*)RoundUpToAlignment((uintptr_t)CurBufferPtr,
+ Alignment);
+ CurBufferPtr = std::min(NewPtr, BufferEnd);
+ }
+
+ /// emitAlignmentWithFill - Similar to emitAlignment, except that the
+ /// extra bytes are filled with the provided byte.
+ void emitAlignmentWithFill(unsigned Alignment, uint8_t Fill) {
+ if (Alignment == 0) Alignment = 1;
+ uint8_t *NewPtr = (uint8_t*)RoundUpToAlignment((uintptr_t)CurBufferPtr,
+ Alignment);
+ // Fail if we don't have room.
+ if (NewPtr > BufferEnd) {
+ CurBufferPtr = BufferEnd;
+ return;
+ }
+ while (CurBufferPtr < NewPtr) {
+ *CurBufferPtr++ = Fill;
+ }
+ }
+
+ /// emitULEB128Bytes - This callback is invoked when a ULEB128 needs to be
+ /// written to the output stream.
+ void emitULEB128Bytes(uint64_t Value, unsigned PadTo = 0) {
+ do {
+ uint8_t Byte = Value & 0x7f;
+ Value >>= 7;
+ if (Value || PadTo != 0) Byte |= 0x80;
+ emitByte(Byte);
+ } while (Value);
+
+ if (PadTo) {
+ do {
+ uint8_t Byte = (PadTo > 1) ? 0x80 : 0x0;
+ emitByte(Byte);
+ } while (--PadTo);
+ }
+ }
+
+ /// emitSLEB128Bytes - This callback is invoked when a SLEB128 needs to be
+ /// written to the output stream.
+ void emitSLEB128Bytes(int64_t Value) {
+ int32_t Sign = Value >> (8 * sizeof(Value) - 1);
+ bool IsMore;
+
+ do {
+ uint8_t Byte = Value & 0x7f;
+ Value >>= 7;
+ IsMore = Value != Sign || ((Byte ^ Sign) & 0x40) != 0;
+ if (IsMore) Byte |= 0x80;
+ emitByte(Byte);
+ } while (IsMore);
+ }
+
+ /// emitString - This callback is invoked when a String needs to be
+ /// written to the output stream.
+ void emitString(const std::string &String) {
+ for (size_t i = 0, N = String.size(); i < N; ++i) {
+ uint8_t C = String[i];
+ emitByte(C);
+ }
+ emitByte(0);
+ }
+
+ /// emitInt32 - Emit a int32 directive.
+ void emitInt32(uint32_t Value) {
+ if (4 <= BufferEnd-CurBufferPtr) {
+ *((uint32_t*)CurBufferPtr) = Value;
+ CurBufferPtr += 4;
+ } else {
+ CurBufferPtr = BufferEnd;
+ }
+ }
+
+ /// emitInt64 - Emit a int64 directive.
+ void emitInt64(uint64_t Value) {
+ if (8 <= BufferEnd-CurBufferPtr) {
+ *((uint64_t*)CurBufferPtr) = Value;
+ CurBufferPtr += 8;
+ } else {
+ CurBufferPtr = BufferEnd;
+ }
+ }
+
+ /// emitInt32At - Emit the Int32 Value in Addr.
+ void emitInt32At(uintptr_t *Addr, uintptr_t Value) {
+ if (Addr >= (uintptr_t*)BufferBegin && Addr < (uintptr_t*)BufferEnd)
+ (*(uint32_t*)Addr) = (uint32_t)Value;
+ }
+
+ /// emitInt64At - Emit the Int64 Value in Addr.
+ void emitInt64At(uintptr_t *Addr, uintptr_t Value) {
+ if (Addr >= (uintptr_t*)BufferBegin && Addr < (uintptr_t*)BufferEnd)
+ (*(uint64_t*)Addr) = (uint64_t)Value;
+ }
+
+
+ /// emitLabel - Emits a label
+ void emitLabel(MCSymbol *Label) override = 0;
+
+ /// allocateSpace - Allocate a block of space in the current output buffer,
+ /// returning null (and setting conditions to indicate buffer overflow) on
+ /// failure. Alignment is the alignment in bytes of the buffer desired.
+ void *allocateSpace(uintptr_t Size, unsigned Alignment) override {
+ emitAlignment(Alignment);
+ void *Result;
+
+ // Check for buffer overflow.
+ if (Size >= (uintptr_t)(BufferEnd-CurBufferPtr)) {
+ CurBufferPtr = BufferEnd;
+ Result = nullptr;
+ } else {
+ // Allocate the space.
+ Result = CurBufferPtr;
+ CurBufferPtr += Size;
+ }
+
+ return Result;
+ }
+
+ /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace,
+ /// this method does not allocate memory in the current output buffer,
+ /// because a global may live longer than the current function.
+ virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment) = 0;
+
+ /// StartMachineBasicBlock - This should be called by the target when a new
+ /// basic block is about to be emitted. This way the MCE knows where the
+ /// start of the block is, and can implement getMachineBasicBlockAddress.
+ void StartMachineBasicBlock(MachineBasicBlock *MBB) override = 0;
+
+ /// getCurrentPCValue - This returns the address that the next emitted byte
+ /// will be output to.
+ ///
+ uintptr_t getCurrentPCValue() const override {
+ return (uintptr_t)CurBufferPtr;
+ }
+
+ /// getCurrentPCOffset - Return the offset from the start of the emitted
+ /// buffer that we are currently writing to.
+ uintptr_t getCurrentPCOffset() const override {
+ return CurBufferPtr-BufferBegin;
+ }
+
+ /// earlyResolveAddresses - True if the code emitter can use symbol addresses
+ /// during code emission time. The JIT is capable of doing this because it
+ /// creates jump tables or constant pools in memory on the fly while the
+ /// object code emitters rely on a linker to have real addresses and should
+ /// use relocations instead.
+ bool earlyResolveAddresses() const override { return true; }
+
+ /// addRelocation - Whenever a relocatable address is needed, it should be
+ /// noted with this interface.
+ void addRelocation(const MachineRelocation &MR) override = 0;
+
+ /// FIXME: These should all be handled with relocations!
+
+ /// getConstantPoolEntryAddress - Return the address of the 'Index' entry in
+ /// the constant pool that was last emitted with the emitConstantPool method.
+ ///
+ uintptr_t getConstantPoolEntryAddress(unsigned Index) const override = 0;
+
+ /// getJumpTableEntryAddress - Return the address of the jump table with index
+ /// 'Index' in the function that last called initJumpTableInfo.
+ ///
+ uintptr_t getJumpTableEntryAddress(unsigned Index) const override = 0;
+
+ /// getMachineBasicBlockAddress - Return the address of the specified
+ /// MachineBasicBlock, only usable after the label for the MBB has been
+ /// emitted.
+ ///
+ uintptr_t
+ getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override = 0;
+
+ /// getLabelAddress - Return the address of the specified Label, only usable
+ /// after the Label has been emitted.
+ ///
+ uintptr_t getLabelAddress(MCSymbol *Label) const override = 0;
+
+ /// Specifies the MachineModuleInfo object. This is used for exception handling
+ /// purposes.
+ void setModuleInfo(MachineModuleInfo* Info) override = 0;
+
+ /// getLabelLocations - Return the label locations map of the label IDs to
+ /// their address.
+ virtual DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() {
+ return nullptr;
+ }
+};
+
+} // End llvm namespace
+
+#endif
// To avoid having libexecutionengine depend on the JIT and interpreter
// libraries, the execution engine implementations set these functions to ctor
// pointers at startup time if they are linked in.
+ static ExecutionEngine *(*JITCtor)(
+ Module *M,
+ std::string *ErrorStr,
+ JITMemoryManager *JMM,
+ bool GVsWithCode,
+ TargetMachine *TM);
static ExecutionEngine *(*MCJITCtor)(
Module *M,
std::string *ErrorStr,
/// getFunctionAddress instead.
virtual void *getPointerToFunction(Function *F) = 0;
+ /// getPointerToBasicBlock - The different EE's represent basic blocks in
+ /// different ways. Return the representation for a blockaddress of the
+ /// specified block.
+ ///
+ /// This function will not be implemented for the MCJIT execution engine.
+ virtual void *getPointerToBasicBlock(BasicBlock *BB) = 0;
+
/// getPointerToFunctionOrStub - If the specified function has been
/// code-gen'd, return a pointer to the function. If not, compile it, or use
/// a stub to implement lazy compilation if available. See
void InitializeMemory(const Constant *Init, void *Addr);
+ /// recompileAndRelinkFunction - This method is used to force a function which
+ /// has already been compiled to be compiled again, possibly after it has been
+ /// modified. Then the entry to the old copy is overwritten with a branch to
+ /// the new copy. If there was no old copy, this acts just like
+ /// VM::getPointerToFunction().
+ virtual void *recompileAndRelinkFunction(Function *F) = 0;
+
+ /// freeMachineCodeForFunction - Release memory in the ExecutionEngine
+ /// corresponding to the machine code emitted to execute this function, useful
+ /// for garbage-collecting generated code.
+ virtual void freeMachineCodeForFunction(Function *F) = 0;
+
/// getOrEmitGlobalVariable - Return the address of the specified global
/// variable, possibly emitting it to memory if needed. This is used by the
/// Emitter.
CodeGenOpt::Level OptLevel;
RTDyldMemoryManager *MCJMM;
JITMemoryManager *JMM;
+ bool AllocateGVsWithCode;
TargetOptions Options;
Reloc::Model RelocModel;
CodeModel::Model CMModel;
std::string MArch;
std::string MCPU;
SmallVector<std::string, 4> MAttrs;
+ bool UseMCJIT;
bool VerifyModules;
/// InitEngine - Does the common initialization of default options.
return *this;
}
+ /// setAllocateGVsWithCode - Sets whether global values should be allocated
+ /// into the same buffer as code. For most applications this should be set
+ /// to false. Allocating globals with code breaks freeMachineCodeForFunction
+ /// and is probably unsafe and bad for performance. However, we have clients
+ /// who depend on this behavior, so we must support it. This option defaults
+ /// to false so that users of the new API can safely use the new memory
+ /// manager and free machine code.
+ EngineBuilder &setAllocateGVsWithCode(bool a) {
+ AllocateGVsWithCode = a;
+ return *this;
+ }
+
/// setMArch - Override the architecture set by the Module's triple.
EngineBuilder &setMArch(StringRef march) {
MArch.assign(march.begin(), march.end());
return *this;
}
+ /// setUseMCJIT - Set whether the MC-JIT implementation should be used
+ /// (experimental).
+ EngineBuilder &setUseMCJIT(bool Value) {
+ UseMCJIT = Value;
+ return *this;
+ }
+
/// setVerifyModules - Set whether the JIT implementation should verify
/// IR modules during compilation.
EngineBuilder &setVerifyModules(bool Verify) {
--- /dev/null
+//===-- JIT.h - Abstract Execution Engine Interface -------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file forces the JIT to link in on certain operating systems.
+// (Windows).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_JIT_H
+#define LLVM_EXECUTIONENGINE_JIT_H
+
+#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include <cstdlib>
+
+extern "C" void LLVMLinkInJIT();
+
+namespace {
+ struct ForceJITLinking {
+ ForceJITLinking() {
+ // We must reference JIT in such a way that compilers will not
+ // delete it all as dead code, even with whole program optimization,
+ // yet is effectively a NO-OP. As the compiler isn't smart enough
+ // to know that getenv() never returns -1, this will do the job.
+ if (std::getenv("bar") != (char*) -1)
+ return;
+
+ LLVMLinkInJIT();
+ }
+ } ForceJITLinking;
+}
+
+#endif
--- /dev/null
+//===- Target/TargetJITInfo.h - Target Information for JIT ------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file exposes an abstract interface used by the Just-In-Time code
+// generator to perform target-specific activities, such as emitting stubs. If
+// a TargetMachine supports JIT code generation, it should provide one of these
+// objects through the getJITInfo() method.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TARGET_TARGETJITINFO_H
+#define LLVM_TARGET_TARGETJITINFO_H
+
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <cassert>
+
+namespace llvm {
+ class Function;
+ class GlobalValue;
+ class JITCodeEmitter;
+ class MachineRelocation;
+
+ /// TargetJITInfo - Target specific information required by the Just-In-Time
+ /// code generator.
+ class TargetJITInfo {
+ virtual void anchor();
+ public:
+ virtual ~TargetJITInfo() {}
+
+ /// replaceMachineCodeForFunction - Make it so that calling the function
+ /// whose machine code is at OLD turns into a call to NEW, perhaps by
+ /// overwriting OLD with a branch to NEW. This is used for self-modifying
+ /// code.
+ ///
+ virtual void replaceMachineCodeForFunction(void *Old, void *New) = 0;
+
+ /// emitGlobalValueIndirectSym - Use the specified JITCodeEmitter object
+ /// to emit an indirect symbol which contains the address of the specified
+ /// ptr.
+ virtual void *emitGlobalValueIndirectSym(const GlobalValue* GV, void *ptr,
+ JITCodeEmitter &JCE) {
+ llvm_unreachable("This target doesn't implement "
+ "emitGlobalValueIndirectSym!");
+ }
+
+ /// Records the required size and alignment for a call stub in bytes.
+ struct StubLayout {
+ size_t Size;
+ size_t Alignment;
+ };
+ /// Returns the maximum size and alignment for a call stub on this target.
+ virtual StubLayout getStubLayout() {
+ llvm_unreachable("This target doesn't implement getStubLayout!");
+ }
+
+ /// emitFunctionStub - Use the specified JITCodeEmitter object to emit a
+ /// small native function that simply calls the function at the specified
+ /// address. The JITCodeEmitter must already have storage allocated for the
+ /// stub. Return the address of the resultant function, which may have been
+ /// aligned from the address the JCE was set up to emit at.
+ virtual void *emitFunctionStub(const Function* F, void *Target,
+ JITCodeEmitter &JCE) {
+ llvm_unreachable("This target doesn't implement emitFunctionStub!");
+ }
+
+ /// getPICJumpTableEntry - Returns the value of the jumptable entry for the
+ /// specific basic block.
+ virtual uintptr_t getPICJumpTableEntry(uintptr_t BB, uintptr_t JTBase) {
+ llvm_unreachable("This target doesn't implement getPICJumpTableEntry!");
+ }
+
+ /// LazyResolverFn - This typedef is used to represent the function that
+ /// unresolved call points should invoke. This is a target specific
+ /// function that knows how to walk the stack and find out which stub the
+ /// call is coming from.
+ typedef void (*LazyResolverFn)();
+
+ /// JITCompilerFn - This typedef is used to represent the JIT function that
+ /// lazily compiles the function corresponding to a stub. The JIT keeps
+ /// track of the mapping between stubs and LLVM Functions, the target
+ /// provides the ability to figure out the address of a stub that is called
+ /// by the LazyResolverFn.
+ typedef void* (*JITCompilerFn)(void *);
+
+ /// getLazyResolverFunction - This method is used to initialize the JIT,
+ /// giving the target the function that should be used to compile a
+ /// function, and giving the JIT the target function used to do the lazy
+ /// resolving.
+ virtual LazyResolverFn getLazyResolverFunction(JITCompilerFn) {
+ llvm_unreachable("Not implemented for this target!");
+ }
+
+ /// relocate - Before the JIT can run a block of code that has been emitted,
+ /// it must rewrite the code to contain the actual addresses of any
+ /// referenced global symbols.
+ virtual void relocate(void *Function, MachineRelocation *MR,
+ unsigned NumRelocs, unsigned char* GOTBase) {
+ assert(NumRelocs == 0 && "This target does not have relocations!");
+ }
+
+ /// allocateThreadLocalMemory - Each target has its own way of
+ /// handling thread local variables. This method returns a value only
+ /// meaningful to the target.
+ virtual char* allocateThreadLocalMemory(size_t size) {
+ llvm_unreachable("This target does not implement thread local storage!");
+ }
+
+ /// needsGOT - Allows a target to specify that it would like the
+ /// JIT to manage a GOT for it.
+ bool needsGOT() const { return useGOT; }
+
+ /// hasCustomConstantPool - Allows a target to specify that constant
+ /// pool address resolution is handled by the target.
+ virtual bool hasCustomConstantPool() const { return false; }
+
+ /// hasCustomJumpTables - Allows a target to specify that jumptables
+ /// are emitted by the target.
+ virtual bool hasCustomJumpTables() const { return false; }
+
+ /// allocateSeparateGVMemory - If true, globals should be placed in
+ /// separately allocated heap memory rather than in the same
+ /// code memory allocated by JITCodeEmitter.
+ virtual bool allocateSeparateGVMemory() const { return false; }
+ protected:
+ bool useGOT;
+ };
+} // End llvm namespace
+
+#endif
return UseUnderscoreLongJmp;
}
+ /// Return whether the target can generate code for jump tables.
+ bool supportJumpTables() const {
+ return SupportJumpTables;
+ }
+
/// Return integer threshold on number of blocks to use jump tables rather
/// than if sequence.
int getMinimumJumpTableEntries() const {
UseUnderscoreLongJmp = Val;
}
+ /// Indicate whether the target can generate code for jump tables.
+ void setSupportJumpTables(bool Val) {
+ SupportJumpTables = Val;
+ }
+
/// Indicate the number of blocks to generate jump tables rather than if
/// sequence.
void setMinimumJumpTableEntries(int Val) {
/// Defaults to false.
bool UseUnderscoreLongJmp;
+ /// Whether the target can generate code for jumptables. If it's not true,
+ /// then each jumptable must be lowered into if-then-else's.
+ bool SupportJumpTables;
+
/// Number of blocks threshold to use jump tables.
int MinimumJumpTableEntries;
namespace llvm {
class InstrItineraryData;
+class JITCodeEmitter;
class GlobalValue;
class Mangler;
class MCAsmInfo;
class TargetLibraryInfo;
class TargetFrameLowering;
class TargetIntrinsicInfo;
+class TargetJITInfo;
class TargetLowering;
class TargetPassConfig;
class TargetRegisterInfo;
virtual const TargetSubtargetInfo *getSubtargetImpl() const {
return nullptr;
}
+ TargetSubtargetInfo *getSubtargetImpl() {
+ const TargetMachine *TM = this;
+ return const_cast<TargetSubtargetInfo *>(TM->getSubtargetImpl());
+ }
/// getSubtarget - This method returns a pointer to the specified type of
/// TargetSubtargetInfo. In debug builds, it verifies that the object being
return true;
}
+ /// addPassesToEmitMachineCode - Add passes to the specified pass manager to
+ /// get machine code emitted. This uses a JITCodeEmitter object to handle
+ /// actually outputting the machine code and resolving things like the address
+ /// of functions. This method returns true if machine code emission is
+ /// not supported.
+ ///
+ virtual bool addPassesToEmitMachineCode(PassManagerBase &,
+ JITCodeEmitter &,
+ bool /*DisableVerify*/ = true) {
+ return true;
+ }
+
/// addPassesToEmitMC - Add passes to the specified pass manager to get
/// machine code emitted with the MCJIT. This method returns true if machine
/// code is not supported. It fills the MCContext Ctx pointer which can be
AnalysisID StartAfter = nullptr,
AnalysisID StopAfter = nullptr) override;
+ /// addPassesToEmitMachineCode - Add passes to the specified pass manager to
+ /// get machine code emitted. This uses a JITCodeEmitter object to handle
+ /// actually outputting the machine code and resolving things like the address
+ /// of functions. This method returns true if machine code emission is
+ /// not supported.
+ ///
+ bool addPassesToEmitMachineCode(PassManagerBase &PM, JITCodeEmitter &MCE,
+ bool DisableVerify = true) override;
+
/// addPassesToEmitMC - Add passes to the specified pass manager to get
/// machine code emitted with the MCJIT. This method returns true if machine
/// code is not supported. It fills the MCContext Ctx pointer which can be
///
bool addPassesToEmitMC(PassManagerBase &PM, MCContext *&Ctx,
raw_ostream &OS, bool DisableVerify = true) override;
+
+ /// addCodeEmitter - This pass should be overridden by the target to add a
+ /// code emitter, if supported. If this is not supported, 'true' should be
+ /// returned.
+ virtual bool addCodeEmitter(PassManagerBase &,
+ JITCodeEmitter &) {
+ return true;
+ }
};
} // End llvm namespace
class SUnit;
class TargetFrameLowering;
class TargetInstrInfo;
+class TargetJITInfo;
class TargetLowering;
class TargetRegisterClass;
class TargetRegisterInfo;
///
virtual const TargetRegisterInfo *getRegisterInfo() const { return nullptr; }
+ /// getJITInfo - If this target supports a JIT, return information for it,
+ /// otherwise return null.
+ ///
+ virtual TargetJITInfo *getJITInfo() { return nullptr; }
+
/// getInstrItineraryData - Returns instruction itinerary data for the target
/// or specific subtarget.
///
bool BasicTTI::shouldBuildLookupTables() const {
const TargetLoweringBase *TLI = getTLI();
- return TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
- TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
+ return TLI->supportJumpTables() &&
+ (TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
+ TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other));
}
bool BasicTTI::haveFastSqrt(Type *Ty) const {
InlineSpiller.cpp
InterferenceCache.cpp
IntrinsicLowering.cpp
+ JITCodeEmitter.cpp
JumpInstrTables.cpp
LLVMTargetMachine.cpp
LatencyPriorityQueue.cpp
--- /dev/null
+//===-- llvm/CodeGen/JITCodeEmitter.cpp - Code emission --------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/JITCodeEmitter.h"
+
+using namespace llvm;
+
+void JITCodeEmitter::anchor() { }
return false;
}
+/// addPassesToEmitMachineCode - Add passes to the specified pass manager to
+/// get machine code emitted. This uses a JITCodeEmitter object to handle
+/// actually outputting the machine code and resolving things like the address
+/// of functions. This method should return true if machine code emission is
+/// not supported.
+///
+bool LLVMTargetMachine::addPassesToEmitMachineCode(PassManagerBase &PM,
+ JITCodeEmitter &JCE,
+ bool DisableVerify) {
+ // Add common CodeGen passes.
+ MCContext *Context = addPassesToGenerateCode(this, PM, DisableVerify, nullptr,
+ nullptr);
+ if (!Context)
+ return true;
+
+ addCodeEmitter(PM, JCE);
+
+ return false; // success!
+}
+
/// addPassesToEmitMC - Add passes to the specified pass manager to get
/// machine code emitted with the MCJIT. This method returns true if machine
/// code is not supported. It fills the MCContext Ctx pointer which can be
}
static inline bool areJTsAllowed(const TargetLowering &TLI) {
- return TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
- TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
+ return TLI.supportJumpTables() &&
+ (TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
+ TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other));
}
static APInt ComputeRange(const APInt &First, const APInt &Last) {
PrefLoopAlignment = 0;
MinStackArgumentAlignment = 1;
InsertFencesForAtomic = false;
+ SupportJumpTables = true;
MinimumJumpTableEntries = 4;
InitLibcallNames(LibcallRoutineNames, Triple(TM.getTargetTriple()));
)
add_subdirectory(Interpreter)
+add_subdirectory(JIT)
add_subdirectory(MCJIT)
add_subdirectory(RuntimeDyld)
void ObjectBuffer::anchor() {}
void ObjectBufferStream::anchor() {}
+ExecutionEngine *(*ExecutionEngine::JITCtor)(
+ Module *M,
+ std::string *ErrorStr,
+ JITMemoryManager *JMM,
+ bool GVsWithCode,
+ TargetMachine *TM) = nullptr;
ExecutionEngine *(*ExecutionEngine::MCJITCtor)(
Module *M,
std::string *ErrorStr,
MCJMM = nullptr;
JMM = nullptr;
Options = TargetOptions();
+ AllocateGVsWithCode = false;
RelocModel = Reloc::Default;
CMModel = CodeModel::JITDefault;
+ UseMCJIT = false;
// IR module verification is enabled by default in debug builds, and disabled
// by default in release builds.
return nullptr;
}
}
+
+ if (MCJMM && ! UseMCJIT) {
+ if (ErrorStr)
+ *ErrorStr =
+ "Cannot create a legacy JIT with a runtime dyld memory "
+ "manager.";
+ return nullptr;
+ }
// Unless the interpreter was explicitly selected or the JIT is not linked,
// try making a JIT.
}
ExecutionEngine *EE = nullptr;
- if (ExecutionEngine::MCJITCtor)
+ if (UseMCJIT && ExecutionEngine::MCJITCtor)
EE = ExecutionEngine::MCJITCtor(M, ErrorStr, MCJMM ? MCJMM : JMM,
TheTM.release());
+ else if (ExecutionEngine::JITCtor)
+ EE = ExecutionEngine::JITCtor(M, ErrorStr, JMM,
+ AllocateGVsWithCode, TheTM.release());
if (EE) {
EE->setVerifyModules(VerifyModules);
return nullptr;
}
- if ((WhichEngine & EngineKind::JIT) && !ExecutionEngine::MCJITCtor) {
+ if ((WhichEngine & EngineKind::JIT) && !ExecutionEngine::JITCtor &&
+ !ExecutionEngine::MCJITCtor) {
if (ErrorStr)
*ErrorStr = "JIT has not been linked in.";
}
Result = PTOGV(getPointerToFunctionOrStub(const_cast<Function*>(F)));
else if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(C))
Result = PTOGV(getOrEmitGlobalVariable(const_cast<GlobalVariable*>(GV)));
+ else if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
+ Result = PTOGV(getPointerToBasicBlock(const_cast<BasicBlock*>(
+ BA->getBasicBlock())));
else
llvm_unreachable("Unknown constant pointer type!");
break;
EngineBuilder builder(unwrap(M));
builder.setEngineKind(EngineKind::JIT)
.setErrorStr(&Error)
+ .setUseMCJIT(true)
.setOptLevel((CodeGenOpt::Level)options.OptLevel)
.setCodeModel(unwrap(options.CodeModel))
.setTargetOptions(targetOptions);
}
void LLVMFreeMachineCodeForFunction(LLVMExecutionEngineRef EE, LLVMValueRef F) {
+ unwrap(EE)->freeMachineCodeForFunction(unwrap<Function>(F));
}
void LLVMAddModule(LLVMExecutionEngineRef EE, LLVMModuleRef M){
void *LLVMRecompileAndRelinkFunction(LLVMExecutionEngineRef EE,
LLVMValueRef Fn) {
- return nullptr;
+ return unwrap(EE)->recompileAndRelinkFunction(unwrap<Function>(Fn));
}
LLVMTargetDataRef LLVMGetExecutionEngineTargetData(LLVMExecutionEngineRef EE) {
return nullptr;
}
+ /// recompileAndRelinkFunction - For the interpreter, functions are always
+ /// up-to-date.
+ ///
+ void *recompileAndRelinkFunction(Function *F) override {
+ return getPointerToFunction(F);
+ }
+
+ /// freeMachineCodeForFunction - The interpreter does not generate any code.
+ ///
+ void freeMachineCodeForFunction(Function *F) override { }
+
// Methods used to execute code:
// Place a call on the stack
void callFunction(Function *F, const std::vector<GenericValue> &ArgVals);
void SwitchToNewBasicBlock(BasicBlock *Dest, ExecutionContext &SF);
void *getPointerToFunction(Function *F) override { return (void*)F; }
+ void *getPointerToBasicBlock(BasicBlock *BB) override { return (void*)BB; }
void initializeExecutionEngine() { }
void initializeExternalFunctions();
--- /dev/null
+# TODO: Support other architectures. See Makefile.
+add_definitions(-DENABLE_X86_JIT)
+
+add_llvm_library(LLVMJIT
+ JIT.cpp
+ JITEmitter.cpp
+ JITMemoryManager.cpp
+ )
--- /dev/null
+//===-- JIT.cpp - LLVM Just in Time Compiler ------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This tool implements a just-in-time compiler for LLVM, allowing direct
+// execution of LLVM bitcode in an efficient manner.
+//
+//===----------------------------------------------------------------------===//
+
+#include "JIT.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/CodeGen/MachineCodeInfo.h"
+#include "llvm/Config/config.h"
+#include "llvm/ExecutionEngine/GenericValue.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
+#include "llvm/ExecutionEngine/JITMemoryManager.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/Dwarf.h"
+#include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/MutexGuard.h"
+#include "llvm/Target/TargetJITInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+
+using namespace llvm;
+
+#ifdef __APPLE__
+// Apple gcc defaults to -fuse-cxa-atexit (i.e. calls __cxa_atexit instead
+// of atexit). It passes the address of linker generated symbol __dso_handle
+// to the function.
+// This configuration change happened at version 5330.
+# include <AvailabilityMacros.h>
+# if defined(MAC_OS_X_VERSION_10_4) && \
+ ((MAC_OS_X_VERSION_MIN_REQUIRED > MAC_OS_X_VERSION_10_4) || \
+ (MAC_OS_X_VERSION_MIN_REQUIRED == MAC_OS_X_VERSION_10_4 && \
+ __APPLE_CC__ >= 5330))
+# ifndef HAVE___DSO_HANDLE
+# define HAVE___DSO_HANDLE 1
+# endif
+# endif
+#endif
+
+#if HAVE___DSO_HANDLE
+extern void *__dso_handle __attribute__ ((__visibility__ ("hidden")));
+#endif
+
+namespace {
+
+static struct RegisterJIT {
+ RegisterJIT() { JIT::Register(); }
+} JITRegistrator;
+
+}
+
+extern "C" void LLVMLinkInJIT() {
+}
+
+/// createJIT - This is the factory method for creating a JIT for the current
+/// machine, it does not fall back to the interpreter. This takes ownership
+/// of the module.
+ExecutionEngine *JIT::createJIT(Module *M,
+ std::string *ErrorStr,
+ JITMemoryManager *JMM,
+ bool GVsWithCode,
+ TargetMachine *TM) {
+ // Try to register the program as a source of symbols to resolve against.
+ //
+ // FIXME: Don't do this here.
+ sys::DynamicLibrary::LoadLibraryPermanently(nullptr, nullptr);
+
+ // If the target supports JIT code generation, create the JIT.
+ if (TargetJITInfo *TJ = TM->getSubtargetImpl()->getJITInfo()) {
+ return new JIT(M, *TM, *TJ, JMM, GVsWithCode);
+ } else {
+ if (ErrorStr)
+ *ErrorStr = "target does not support JIT code generation";
+ return nullptr;
+ }
+}
+
+namespace {
+/// This class supports the global getPointerToNamedFunction(), which allows
+/// bugpoint or gdb users to search for a function by name without any context.
+class JitPool {
+ SmallPtrSet<JIT*, 1> JITs; // Optimize for process containing just 1 JIT.
+ mutable sys::Mutex Lock;
+public:
+ void Add(JIT *jit) {
+ MutexGuard guard(Lock);
+ JITs.insert(jit);
+ }
+ void Remove(JIT *jit) {
+ MutexGuard guard(Lock);
+ JITs.erase(jit);
+ }
+ void *getPointerToNamedFunction(const char *Name) const {
+ MutexGuard guard(Lock);
+ assert(JITs.size() != 0 && "No Jit registered");
+ //search function in every instance of JIT
+ for (SmallPtrSet<JIT*, 1>::const_iterator Jit = JITs.begin(),
+ end = JITs.end();
+ Jit != end; ++Jit) {
+ if (Function *F = (*Jit)->FindFunctionNamed(Name))
+ return (*Jit)->getPointerToFunction(F);
+ }
+ // The function is not available : fallback on the first created (will
+ // search in symbol of the current program/library)
+ return (*JITs.begin())->getPointerToNamedFunction(Name);
+ }
+};
+ManagedStatic<JitPool> AllJits;
+}
+extern "C" {
+ // getPointerToNamedFunction - This function is used as a global wrapper to
+ // JIT::getPointerToNamedFunction for the purpose of resolving symbols when
+ // bugpoint is debugging the JIT. In that scenario, we are loading an .so and
+ // need to resolve function(s) that are being mis-codegenerated, so we need to
+ // resolve their addresses at runtime, and this is the way to do it.
+ void *getPointerToNamedFunction(const char *Name) {
+ return AllJits->getPointerToNamedFunction(Name);
+ }
+}
+
+JIT::JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji,
+ JITMemoryManager *jmm, bool GVsWithCode)
+ : ExecutionEngine(M), TM(tm), TJI(tji),
+ JMM(jmm ? jmm : JITMemoryManager::CreateDefaultMemManager()),
+ AllocateGVsWithCode(GVsWithCode), isAlreadyCodeGenerating(false) {
+ setDataLayout(TM.getSubtargetImpl()->getDataLayout());
+
+ jitstate = new JITState(M);
+
+ // Initialize JCE
+ JCE = createEmitter(*this, JMM, TM);
+
+ // Register in global list of all JITs.
+ AllJits->Add(this);
+
+ // Add target data
+ MutexGuard locked(lock);
+ FunctionPassManager &PM = jitstate->getPM();
+ M->setDataLayout(TM.getSubtargetImpl()->getDataLayout());
+ PM.add(new DataLayoutPass(M));
+
+ // Turn the machine code intermediate representation into bytes in memory that
+ // may be executed.
+ if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) {
+ report_fatal_error("Target does not support machine code emission!");
+ }
+
+ // Initialize passes.
+ PM.doInitialization();
+}
+
+JIT::~JIT() {
+ // Cleanup.
+ AllJits->Remove(this);
+ delete jitstate;
+ delete JCE;
+ // JMM is a ownership of JCE, so we no need delete JMM here.
+ delete &TM;
+}
+
+/// addModule - Add a new Module to the JIT. If we previously removed the last
+/// Module, we need re-initialize jitstate with a valid Module.
+void JIT::addModule(Module *M) {
+ MutexGuard locked(lock);
+
+ if (Modules.empty()) {
+ assert(!jitstate && "jitstate should be NULL if Modules vector is empty!");
+
+ jitstate = new JITState(M);
+
+ FunctionPassManager &PM = jitstate->getPM();
+ M->setDataLayout(TM.getSubtargetImpl()->getDataLayout());
+ PM.add(new DataLayoutPass(M));
+
+ // Turn the machine code intermediate representation into bytes in memory
+ // that may be executed.
+ if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) {
+ report_fatal_error("Target does not support machine code emission!");
+ }
+
+ // Initialize passes.
+ PM.doInitialization();
+ }
+
+ ExecutionEngine::addModule(M);
+}
+
+/// removeModule - If we are removing the last Module, invalidate the jitstate
+/// since the PassManager it contains references a released Module.
+bool JIT::removeModule(Module *M) {
+ bool result = ExecutionEngine::removeModule(M);
+
+ MutexGuard locked(lock);
+
+ if (jitstate && jitstate->getModule() == M) {
+ delete jitstate;
+ jitstate = nullptr;
+ }
+
+ if (!jitstate && !Modules.empty()) {
+ jitstate = new JITState(Modules[0]);
+
+ FunctionPassManager &PM = jitstate->getPM();
+ M->setDataLayout(TM.getSubtargetImpl()->getDataLayout());
+ PM.add(new DataLayoutPass(M));
+
+ // Turn the machine code intermediate representation into bytes in memory
+ // that may be executed.
+ if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) {
+ report_fatal_error("Target does not support machine code emission!");
+ }
+
+ // Initialize passes.
+ PM.doInitialization();
+ }
+ return result;
+}
+
+/// run - Start execution with the specified function and arguments.
+///
+GenericValue JIT::runFunction(Function *F,
+ const std::vector<GenericValue> &ArgValues) {
+ assert(F && "Function *F was null at entry to run()");
+
+ void *FPtr = getPointerToFunction(F);
+ assert(FPtr && "Pointer to fn's code was null after getPointerToFunction");
+ FunctionType *FTy = F->getFunctionType();
+ Type *RetTy = FTy->getReturnType();
+
+ assert((FTy->getNumParams() == ArgValues.size() ||
+ (FTy->isVarArg() && FTy->getNumParams() <= ArgValues.size())) &&
+ "Wrong number of arguments passed into function!");
+ assert(FTy->getNumParams() == ArgValues.size() &&
+ "This doesn't support passing arguments through varargs (yet)!");
+
+ // Handle some common cases first. These cases correspond to common `main'
+ // prototypes.
+ if (RetTy->isIntegerTy(32) || RetTy->isVoidTy()) {
+ switch (ArgValues.size()) {
+ case 3:
+ if (FTy->getParamType(0)->isIntegerTy(32) &&
+ FTy->getParamType(1)->isPointerTy() &&
+ FTy->getParamType(2)->isPointerTy()) {
+ int (*PF)(int, char **, const char **) =
+ (int(*)(int, char **, const char **))(intptr_t)FPtr;
+
+ // Call the function.
+ GenericValue rv;
+ rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(),
+ (char **)GVTOP(ArgValues[1]),
+ (const char **)GVTOP(ArgValues[2])));
+ return rv;
+ }
+ break;
+ case 2:
+ if (FTy->getParamType(0)->isIntegerTy(32) &&
+ FTy->getParamType(1)->isPointerTy()) {
+ int (*PF)(int, char **) = (int(*)(int, char **))(intptr_t)FPtr;
+
+ // Call the function.
+ GenericValue rv;
+ rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(),
+ (char **)GVTOP(ArgValues[1])));
+ return rv;
+ }
+ break;
+ case 1:
+ if (FTy->getParamType(0)->isIntegerTy(32)) {
+ GenericValue rv;
+ int (*PF)(int) = (int(*)(int))(intptr_t)FPtr;
+ rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue()));
+ return rv;
+ }
+ if (FTy->getParamType(0)->isPointerTy()) {
+ GenericValue rv;
+ int (*PF)(char *) = (int(*)(char *))(intptr_t)FPtr;
+ rv.IntVal = APInt(32, PF((char*)GVTOP(ArgValues[0])));
+ return rv;
+ }
+ break;
+ }
+ }
+
+ // Handle cases where no arguments are passed first.
+ if (ArgValues.empty()) {
+ GenericValue rv;
+ switch (RetTy->getTypeID()) {
+ default: llvm_unreachable("Unknown return type for function call!");
+ case Type::IntegerTyID: {
+ unsigned BitWidth = cast<IntegerType>(RetTy)->getBitWidth();
+ if (BitWidth == 1)
+ rv.IntVal = APInt(BitWidth, ((bool(*)())(intptr_t)FPtr)());
+ else if (BitWidth <= 8)
+ rv.IntVal = APInt(BitWidth, ((char(*)())(intptr_t)FPtr)());
+ else if (BitWidth <= 16)
+ rv.IntVal = APInt(BitWidth, ((short(*)())(intptr_t)FPtr)());
+ else if (BitWidth <= 32)
+ rv.IntVal = APInt(BitWidth, ((int(*)())(intptr_t)FPtr)());
+ else if (BitWidth <= 64)
+ rv.IntVal = APInt(BitWidth, ((int64_t(*)())(intptr_t)FPtr)());
+ else
+ llvm_unreachable("Integer types > 64 bits not supported");
+ return rv;
+ }
+ case Type::VoidTyID:
+ rv.IntVal = APInt(32, ((int(*)())(intptr_t)FPtr)());
+ return rv;
+ case Type::FloatTyID:
+ rv.FloatVal = ((float(*)())(intptr_t)FPtr)();
+ return rv;
+ case Type::DoubleTyID:
+ rv.DoubleVal = ((double(*)())(intptr_t)FPtr)();
+ return rv;
+ case Type::X86_FP80TyID:
+ case Type::FP128TyID:
+ case Type::PPC_FP128TyID:
+ llvm_unreachable("long double not supported yet");
+ case Type::PointerTyID:
+ return PTOGV(((void*(*)())(intptr_t)FPtr)());
+ }
+ }
+
+ // Okay, this is not one of our quick and easy cases. Because we don't have a
+ // full FFI, we have to codegen a nullary stub function that just calls the
+ // function we are interested in, passing in constants for all of the
+ // arguments. Make this function and return.
+
+ // First, create the function.
+ FunctionType *STy=FunctionType::get(RetTy, false);
+ Function *Stub = Function::Create(STy, Function::InternalLinkage, "",
+ F->getParent());
+
+ // Insert a basic block.
+ BasicBlock *StubBB = BasicBlock::Create(F->getContext(), "", Stub);
+
+ // Convert all of the GenericValue arguments over to constants. Note that we
+ // currently don't support varargs.
+ SmallVector<Value*, 8> Args;
+ for (unsigned i = 0, e = ArgValues.size(); i != e; ++i) {
+ Constant *C = nullptr;
+ Type *ArgTy = FTy->getParamType(i);
+ const GenericValue &AV = ArgValues[i];
+ switch (ArgTy->getTypeID()) {
+ default: llvm_unreachable("Unknown argument type for function call!");
+ case Type::IntegerTyID:
+ C = ConstantInt::get(F->getContext(), AV.IntVal);
+ break;
+ case Type::FloatTyID:
+ C = ConstantFP::get(F->getContext(), APFloat(AV.FloatVal));
+ break;
+ case Type::DoubleTyID:
+ C = ConstantFP::get(F->getContext(), APFloat(AV.DoubleVal));
+ break;
+ case Type::PPC_FP128TyID:
+ case Type::X86_FP80TyID:
+ case Type::FP128TyID:
+ C = ConstantFP::get(F->getContext(), APFloat(ArgTy->getFltSemantics(),
+ AV.IntVal));
+ break;
+ case Type::PointerTyID:
+ void *ArgPtr = GVTOP(AV);
+ if (sizeof(void*) == 4)
+ C = ConstantInt::get(Type::getInt32Ty(F->getContext()),
+ (int)(intptr_t)ArgPtr);
+ else
+ C = ConstantInt::get(Type::getInt64Ty(F->getContext()),
+ (intptr_t)ArgPtr);
+ // Cast the integer to pointer
+ C = ConstantExpr::getIntToPtr(C, ArgTy);
+ break;
+ }
+ Args.push_back(C);
+ }
+
+ CallInst *TheCall = CallInst::Create(F, Args, "", StubBB);
+ TheCall->setCallingConv(F->getCallingConv());
+ TheCall->setTailCall();
+ if (!TheCall->getType()->isVoidTy())
+ // Return result of the call.
+ ReturnInst::Create(F->getContext(), TheCall, StubBB);
+ else
+ ReturnInst::Create(F->getContext(), StubBB); // Just return void.
+
+ // Finally, call our nullary stub function.
+ GenericValue Result = runFunction(Stub, std::vector<GenericValue>());
+ // Erase it, since no other function can have a reference to it.
+ Stub->eraseFromParent();
+ // And return the result.
+ return Result;
+}
+
+void JIT::RegisterJITEventListener(JITEventListener *L) {
+ if (!L)
+ return;
+ MutexGuard locked(lock);
+ EventListeners.push_back(L);
+}
+void JIT::UnregisterJITEventListener(JITEventListener *L) {
+ if (!L)
+ return;
+ MutexGuard locked(lock);
+ std::vector<JITEventListener*>::reverse_iterator I=
+ std::find(EventListeners.rbegin(), EventListeners.rend(), L);
+ if (I != EventListeners.rend()) {
+ std::swap(*I, EventListeners.back());
+ EventListeners.pop_back();
+ }
+}
+void JIT::NotifyFunctionEmitted(
+ const Function &F,
+ void *Code, size_t Size,
+ const JITEvent_EmittedFunctionDetails &Details) {
+ MutexGuard locked(lock);
+ for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
+ EventListeners[I]->NotifyFunctionEmitted(F, Code, Size, Details);
+ }
+}
+
+void JIT::NotifyFreeingMachineCode(void *OldPtr) {
+ MutexGuard locked(lock);
+ for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
+ EventListeners[I]->NotifyFreeingMachineCode(OldPtr);
+ }
+}
+
+/// runJITOnFunction - Run the FunctionPassManager full of
+/// just-in-time compilation passes on F, hopefully filling in
+/// GlobalAddress[F] with the address of F's machine code.
+///
+void JIT::runJITOnFunction(Function *F, MachineCodeInfo *MCI) {
+ MutexGuard locked(lock);
+
+ class MCIListener : public JITEventListener {
+ MachineCodeInfo *const MCI;
+ public:
+ MCIListener(MachineCodeInfo *mci) : MCI(mci) {}
+ void NotifyFunctionEmitted(const Function &, void *Code, size_t Size,
+ const EmittedFunctionDetails &) override {
+ MCI->setAddress(Code);
+ MCI->setSize(Size);
+ }
+ };
+ MCIListener MCIL(MCI);
+ if (MCI)
+ RegisterJITEventListener(&MCIL);
+
+ runJITOnFunctionUnlocked(F);
+
+ if (MCI)
+ UnregisterJITEventListener(&MCIL);
+}
+
+void JIT::runJITOnFunctionUnlocked(Function *F) {
+ assert(!isAlreadyCodeGenerating && "Error: Recursive compilation detected!");
+
+ jitTheFunctionUnlocked(F);
+
+ // If the function referred to another function that had not yet been
+ // read from bitcode, and we are jitting non-lazily, emit it now.
+ while (!jitstate->getPendingFunctions().empty()) {
+ Function *PF = jitstate->getPendingFunctions().back();
+ jitstate->getPendingFunctions().pop_back();
+
+ assert(!PF->hasAvailableExternallyLinkage() &&
+ "Externally-defined function should not be in pending list.");
+
+ jitTheFunctionUnlocked(PF);
+
+ // Now that the function has been jitted, ask the JITEmitter to rewrite
+ // the stub with real address of the function.
+ updateFunctionStubUnlocked(PF);
+ }
+}
+
+void JIT::jitTheFunctionUnlocked(Function *F) {
+ isAlreadyCodeGenerating = true;
+ jitstate->getPM().run(*F);
+ isAlreadyCodeGenerating = false;
+
+ // clear basic block addresses after this function is done
+ getBasicBlockAddressMap().clear();
+}
+
+/// getPointerToFunction - This method is used to get the address of the
+/// specified function, compiling it if necessary.
+///
+void *JIT::getPointerToFunction(Function *F) {
+
+ if (void *Addr = getPointerToGlobalIfAvailable(F))
+ return Addr; // Check if function already code gen'd
+
+ MutexGuard locked(lock);
+
+ // Now that this thread owns the lock, make sure we read in the function if it
+ // exists in this Module.
+ std::string ErrorMsg;
+ if (F->Materialize(&ErrorMsg)) {
+ report_fatal_error("Error reading function '" + F->getName()+
+ "' from bitcode file: " + ErrorMsg);
+ }
+
+ // ... and check if another thread has already code gen'd the function.
+ if (void *Addr = getPointerToGlobalIfAvailable(F))
+ return Addr;
+
+ if (F->isDeclaration() || F->hasAvailableExternallyLinkage()) {
+ bool AbortOnFailure = !F->hasExternalWeakLinkage();
+ void *Addr = getPointerToNamedFunction(F->getName(), AbortOnFailure);
+ addGlobalMapping(F, Addr);
+ return Addr;
+ }
+
+ runJITOnFunctionUnlocked(F);
+
+ void *Addr = getPointerToGlobalIfAvailable(F);
+ assert(Addr && "Code generation didn't add function to GlobalAddress table!");
+ return Addr;
+}
+
+void JIT::addPointerToBasicBlock(const BasicBlock *BB, void *Addr) {
+ MutexGuard locked(lock);
+
+ BasicBlockAddressMapTy::iterator I =
+ getBasicBlockAddressMap().find(BB);
+ if (I == getBasicBlockAddressMap().end()) {
+ getBasicBlockAddressMap()[BB] = Addr;
+ } else {
+ // ignore repeats: some BBs can be split into few MBBs?
+ }
+}
+
+void JIT::clearPointerToBasicBlock(const BasicBlock *BB) {
+ MutexGuard locked(lock);
+ getBasicBlockAddressMap().erase(BB);
+}
+
+void *JIT::getPointerToBasicBlock(BasicBlock *BB) {
+ // make sure it's function is compiled by JIT
+ (void)getPointerToFunction(BB->getParent());
+
+ // resolve basic block address
+ MutexGuard locked(lock);
+
+ BasicBlockAddressMapTy::iterator I =
+ getBasicBlockAddressMap().find(BB);
+ if (I != getBasicBlockAddressMap().end()) {
+ return I->second;
+ } else {
+ llvm_unreachable("JIT does not have BB address for address-of-label, was"
+ " it eliminated by optimizer?");
+ }
+}
+
+void *JIT::getPointerToNamedFunction(const std::string &Name,
+ bool AbortOnFailure){
+ if (!isSymbolSearchingDisabled()) {
+ void *ptr = JMM->getPointerToNamedFunction(Name, false);
+ if (ptr)
+ return ptr;
+ }
+
+ /// If a LazyFunctionCreator is installed, use it to get/create the function.
+ if (LazyFunctionCreator)
+ if (void *RP = LazyFunctionCreator(Name))
+ return RP;
+
+ if (AbortOnFailure) {
+ report_fatal_error("Program used external function '"+Name+
+ "' which could not be resolved!");
+ }
+ return nullptr;
+}
+
+
+/// getOrEmitGlobalVariable - Return the address of the specified global
+/// variable, possibly emitting it to memory if needed. This is used by the
+/// Emitter.
+void *JIT::getOrEmitGlobalVariable(const GlobalVariable *GV) {
+ MutexGuard locked(lock);
+
+ void *Ptr = getPointerToGlobalIfAvailable(GV);
+ if (Ptr) return Ptr;
+
+ // If the global is external, just remember the address.
+ if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) {
+#if HAVE___DSO_HANDLE
+ if (GV->getName() == "__dso_handle")
+ return (void*)&__dso_handle;
+#endif
+ Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(GV->getName());
+ if (!Ptr) {
+ report_fatal_error("Could not resolve external global address: "
+ +GV->getName());
+ }
+ addGlobalMapping(GV, Ptr);
+ } else {
+ // If the global hasn't been emitted to memory yet, allocate space and
+ // emit it into memory.
+ Ptr = getMemoryForGV(GV);
+ addGlobalMapping(GV, Ptr);
+ EmitGlobalVariable(GV); // Initialize the variable.
+ }
+ return Ptr;
+}
+
+/// recompileAndRelinkFunction - This method is used to force a function
+/// which has already been compiled, to be compiled again, possibly
+/// after it has been modified. Then the entry to the old copy is overwritten
+/// with a branch to the new copy. If there was no old copy, this acts
+/// just like JIT::getPointerToFunction().
+///
+void *JIT::recompileAndRelinkFunction(Function *F) {
+ void *OldAddr = getPointerToGlobalIfAvailable(F);
+
+ // If it's not already compiled there is no reason to patch it up.
+ if (!OldAddr) return getPointerToFunction(F);
+
+ // Delete the old function mapping.
+ addGlobalMapping(F, nullptr);
+
+ // Recodegen the function
+ runJITOnFunction(F);
+
+ // Update state, forward the old function to the new function.
+ void *Addr = getPointerToGlobalIfAvailable(F);
+ assert(Addr && "Code generation didn't add function to GlobalAddress table!");
+ TJI.replaceMachineCodeForFunction(OldAddr, Addr);
+ return Addr;
+}
+
+/// getMemoryForGV - This method abstracts memory allocation of global
+/// variable so that the JIT can allocate thread local variables depending
+/// on the target.
+///
+char* JIT::getMemoryForGV(const GlobalVariable* GV) {
+ char *Ptr;
+
+ // GlobalVariable's which are not "constant" will cause trouble in a server
+ // situation. It's returned in the same block of memory as code which may
+ // not be writable.
+ if (isGVCompilationDisabled() && !GV->isConstant()) {
+ report_fatal_error("Compilation of non-internal GlobalValue is disabled!");
+ }
+
+ // Some applications require globals and code to live together, so they may
+ // be allocated into the same buffer, but in general globals are allocated
+ // through the memory manager which puts them near the code but not in the
+ // same buffer.
+ Type *GlobalType = GV->getType()->getElementType();
+ size_t S = getDataLayout()->getTypeAllocSize(GlobalType);
+ size_t A = getDataLayout()->getPreferredAlignment(GV);
+ if (GV->isThreadLocal()) {
+ MutexGuard locked(lock);
+ Ptr = TJI.allocateThreadLocalMemory(S);
+ } else if (TJI.allocateSeparateGVMemory()) {
+ if (A <= 8) {
+ Ptr = (char*)malloc(S);
+ } else {
+ // Allocate S+A bytes of memory, then use an aligned pointer within that
+ // space.
+ Ptr = (char*)malloc(S+A);
+ unsigned MisAligned = ((intptr_t)Ptr & (A-1));
+ Ptr = Ptr + (MisAligned ? (A-MisAligned) : 0);
+ }
+ } else if (AllocateGVsWithCode) {
+ Ptr = (char*)JCE->allocateSpace(S, A);
+ } else {
+ Ptr = (char*)JCE->allocateGlobal(S, A);
+ }
+ return Ptr;
+}
+
+void JIT::addPendingFunction(Function *F) {
+ MutexGuard locked(lock);
+ jitstate->getPendingFunctions().push_back(F);
+}
+
+
+JITEventListener::~JITEventListener() {}
--- /dev/null
+//===-- JIT.h - Class definition for the JIT --------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the top-level JIT data structure.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef JIT_H
+#define JIT_H
+
+#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/IR/ValueHandle.h"
+#include "llvm/PassManager.h"
+
+namespace llvm {
+
+class Function;
+struct JITEvent_EmittedFunctionDetails;
+class MachineCodeEmitter;
+class MachineCodeInfo;
+class TargetJITInfo;
+class TargetMachine;
+
+class JITState {
+private:
+ FunctionPassManager PM; // Passes to compile a function
+ Module *M; // Module used to create the PM
+
+ /// PendingFunctions - Functions which have not been code generated yet, but
+ /// were called from a function being code generated.
+ std::vector<AssertingVH<Function> > PendingFunctions;
+
+public:
+ explicit JITState(Module *M) : PM(M), M(M) {}
+
+ FunctionPassManager &getPM() {
+ return PM;
+ }
+
+ Module *getModule() const { return M; }
+ std::vector<AssertingVH<Function> > &getPendingFunctions() {
+ return PendingFunctions;
+ }
+};
+
+
+class JIT : public ExecutionEngine {
+ /// types
+ typedef ValueMap<const BasicBlock *, void *>
+ BasicBlockAddressMapTy;
+ /// data
+ TargetMachine &TM; // The current target we are compiling to
+ TargetJITInfo &TJI; // The JITInfo for the target we are compiling to
+ JITCodeEmitter *JCE; // JCE object
+ JITMemoryManager *JMM;
+ std::vector<JITEventListener*> EventListeners;
+
+ /// AllocateGVsWithCode - Some applications require that global variables and
+ /// code be allocated into the same region of memory, in which case this flag
+ /// should be set to true. Doing so breaks freeMachineCodeForFunction.
+ bool AllocateGVsWithCode;
+
+ /// True while the JIT is generating code. Used to assert against recursive
+ /// entry.
+ bool isAlreadyCodeGenerating;
+
+ JITState *jitstate;
+
+ /// BasicBlockAddressMap - A mapping between LLVM basic blocks and their
+ /// actualized version, only filled for basic blocks that have their address
+ /// taken.
+ BasicBlockAddressMapTy BasicBlockAddressMap;
+
+
+ JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji,
+ JITMemoryManager *JMM, bool AllocateGVsWithCode);
+public:
+ ~JIT();
+
+ static void Register() {
+ JITCtor = createJIT;
+ }
+
+ /// getJITInfo - Return the target JIT information structure.
+ ///
+ TargetJITInfo &getJITInfo() const { return TJI; }
+
+ void addModule(Module *M) override;
+
+ /// removeModule - Remove a Module from the list of modules. Returns true if
+ /// M is found.
+ bool removeModule(Module *M) override;
+
+ /// runFunction - Start execution with the specified function and arguments.
+ ///
+ GenericValue runFunction(Function *F,
+ const std::vector<GenericValue> &ArgValues) override;
+
+ /// getPointerToNamedFunction - This method returns the address of the
+ /// specified function by using the MemoryManager. As such it is only
+ /// useful for resolving library symbols, not code generated symbols.
+ ///
+ /// If AbortOnFailure is false and no function with the given name is
+ /// found, this function silently returns a null pointer. Otherwise,
+ /// it prints a message to stderr and aborts.
+ ///
+ void *getPointerToNamedFunction(const std::string &Name,
+ bool AbortOnFailure = true) override;
+
+ // CompilationCallback - Invoked the first time that a call site is found,
+ // which causes lazy compilation of the target function.
+ //
+ static void CompilationCallback();
+
+ /// getPointerToFunction - This returns the address of the specified function,
+ /// compiling it if necessary.
+ ///
+ void *getPointerToFunction(Function *F) override;
+
+ /// addPointerToBasicBlock - Adds address of the specific basic block.
+ void addPointerToBasicBlock(const BasicBlock *BB, void *Addr);
+
+ /// clearPointerToBasicBlock - Removes address of specific basic block.
+ void clearPointerToBasicBlock(const BasicBlock *BB);
+
+ /// getPointerToBasicBlock - This returns the address of the specified basic
+ /// block, assuming function is compiled.
+ void *getPointerToBasicBlock(BasicBlock *BB) override;
+
+ /// getOrEmitGlobalVariable - Return the address of the specified global
+ /// variable, possibly emitting it to memory if needed. This is used by the
+ /// Emitter.
+ void *getOrEmitGlobalVariable(const GlobalVariable *GV) override;
+
+ /// getPointerToFunctionOrStub - If the specified function has been
+ /// code-gen'd, return a pointer to the function. If not, compile it, or use
+ /// a stub to implement lazy compilation if available.
+ ///
+ void *getPointerToFunctionOrStub(Function *F) override;
+
+ /// recompileAndRelinkFunction - This method is used to force a function
+ /// which has already been compiled, to be compiled again, possibly
+ /// after it has been modified. Then the entry to the old copy is overwritten
+ /// with a branch to the new copy. If there was no old copy, this acts
+ /// just like JIT::getPointerToFunction().
+ ///
+ void *recompileAndRelinkFunction(Function *F) override;
+
+ /// freeMachineCodeForFunction - deallocate memory used to code-generate this
+ /// Function.
+ ///
+ void freeMachineCodeForFunction(Function *F) override;
+
+ /// addPendingFunction - while jitting non-lazily, a called but non-codegen'd
+ /// function was encountered. Add it to a pending list to be processed after
+ /// the current function.
+ ///
+ void addPendingFunction(Function *F);
+
+ /// getCodeEmitter - Return the code emitter this JIT is emitting into.
+ ///
+ JITCodeEmitter *getCodeEmitter() const { return JCE; }
+
+ static ExecutionEngine *createJIT(Module *M,
+ std::string *ErrorStr,
+ JITMemoryManager *JMM,
+ bool GVsWithCode,
+ TargetMachine *TM);
+
+ // Run the JIT on F and return information about the generated code
+ void runJITOnFunction(Function *F, MachineCodeInfo *MCI = nullptr) override;
+
+ void RegisterJITEventListener(JITEventListener *L) override;
+ void UnregisterJITEventListener(JITEventListener *L) override;
+
+ TargetMachine *getTargetMachine() override { return &TM; }
+
+ /// These functions correspond to the methods on JITEventListener. They
+ /// iterate over the registered listeners and call the corresponding method on
+ /// each.
+ void NotifyFunctionEmitted(
+ const Function &F, void *Code, size_t Size,
+ const JITEvent_EmittedFunctionDetails &Details);
+ void NotifyFreeingMachineCode(void *OldPtr);
+
+ BasicBlockAddressMapTy &
+ getBasicBlockAddressMap() {
+ return BasicBlockAddressMap;
+ }
+
+
+private:
+ static JITCodeEmitter *createEmitter(JIT &J, JITMemoryManager *JMM,
+ TargetMachine &tm);
+ void runJITOnFunctionUnlocked(Function *F);
+ void updateFunctionStubUnlocked(Function *F);
+ void jitTheFunctionUnlocked(Function *F);
+
+protected:
+
+ /// getMemoryforGV - Allocate memory for a global variable.
+ char* getMemoryForGV(const GlobalVariable* GV) override;
+
+};
+
+} // End llvm namespace
+
+#endif
--- /dev/null
+//===-- JITEmitter.cpp - Write machine code to executable memory ----------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a MachineCodeEmitter object that is used by the JIT to
+// write machine code to memory and remember where relocatable values are.
+//
+//===----------------------------------------------------------------------===//
+
+#include "JIT.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/CodeGen/MachineCodeInfo.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineRelocation.h"
+#include "llvm/ExecutionEngine/GenericValue.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
+#include "llvm/ExecutionEngine/JITMemoryManager.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/ValueHandle.h"
+#include "llvm/IR/ValueMap.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/Memory.h"
+#include "llvm/Support/MutexGuard.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetJITInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
+#include <algorithm>
+#ifndef NDEBUG
+#include <iomanip>
+#endif
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+STATISTIC(NumBytes, "Number of bytes of machine code compiled");
+STATISTIC(NumRelos, "Number of relocations applied");
+STATISTIC(NumRetries, "Number of retries with more memory");
+
+
+// A declaration may stop being a declaration once it's fully read from bitcode.
+// This function returns true if F is fully read and is still a declaration.
+static bool isNonGhostDeclaration(const Function *F) {
+ return F->isDeclaration() && !F->isMaterializable();
+}
+
+//===----------------------------------------------------------------------===//
+// JIT lazy compilation code.
+//
+namespace {
+ class JITEmitter;
+ class JITResolverState;
+
+ template<typename ValueTy>
+ struct NoRAUWValueMapConfig : public ValueMapConfig<ValueTy> {
+ typedef JITResolverState *ExtraData;
+ static void onRAUW(JITResolverState *, Value *Old, Value *New) {
+ llvm_unreachable("The JIT doesn't know how to handle a"
+ " RAUW on a value it has emitted.");
+ }
+ };
+
+ struct CallSiteValueMapConfig : public NoRAUWValueMapConfig<Function*> {
+ typedef JITResolverState *ExtraData;
+ static void onDelete(JITResolverState *JRS, Function *F);
+ };
+
+ class JITResolverState {
+ public:
+ typedef ValueMap<Function*, void*, NoRAUWValueMapConfig<Function*> >
+ FunctionToLazyStubMapTy;
+ typedef std::map<void*, AssertingVH<Function> > CallSiteToFunctionMapTy;
+ typedef ValueMap<Function *, SmallPtrSet<void*, 1>,
+ CallSiteValueMapConfig> FunctionToCallSitesMapTy;
+ typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy;
+ private:
+ /// FunctionToLazyStubMap - Keep track of the lazy stub created for a
+ /// particular function so that we can reuse them if necessary.
+ FunctionToLazyStubMapTy FunctionToLazyStubMap;
+
+ /// CallSiteToFunctionMap - Keep track of the function that each lazy call
+ /// site corresponds to, and vice versa.
+ CallSiteToFunctionMapTy CallSiteToFunctionMap;
+ FunctionToCallSitesMapTy FunctionToCallSitesMap;
+
+ /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a
+ /// particular GlobalVariable so that we can reuse them if necessary.
+ GlobalToIndirectSymMapTy GlobalToIndirectSymMap;
+
+#ifndef NDEBUG
+ /// Instance of the JIT this ResolverState serves.
+ JIT *TheJIT;
+#endif
+
+ public:
+ JITResolverState(JIT *jit) : FunctionToLazyStubMap(this),
+ FunctionToCallSitesMap(this) {
+#ifndef NDEBUG
+ TheJIT = jit;
+#endif
+ }
+
+ FunctionToLazyStubMapTy& getFunctionToLazyStubMap() {
+ return FunctionToLazyStubMap;
+ }
+
+ GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap() {
+ return GlobalToIndirectSymMap;
+ }
+
+ std::pair<void *, Function *> LookupFunctionFromCallSite(
+ void *CallSite) const {
+ // The address given to us for the stub may not be exactly right, it
+ // might be a little bit after the stub. As such, use upper_bound to
+ // find it.
+ CallSiteToFunctionMapTy::const_iterator I =
+ CallSiteToFunctionMap.upper_bound(CallSite);
+ assert(I != CallSiteToFunctionMap.begin() &&
+ "This is not a known call site!");
+ --I;
+ return *I;
+ }
+
+ void AddCallSite(void *CallSite, Function *F) {
+ bool Inserted = CallSiteToFunctionMap.insert(
+ std::make_pair(CallSite, F)).second;
+ (void)Inserted;
+ assert(Inserted && "Pair was already in CallSiteToFunctionMap");
+ FunctionToCallSitesMap[F].insert(CallSite);
+ }
+
+ void EraseAllCallSitesForPrelocked(Function *F);
+
+ // Erases _all_ call sites regardless of their function. This is used to
+ // unregister the stub addresses from the StubToResolverMap in
+ // ~JITResolver().
+ void EraseAllCallSitesPrelocked();
+ };
+
+ /// JITResolver - Keep track of, and resolve, call sites for functions that
+ /// have not yet been compiled.
+ class JITResolver {
+ typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy;
+ typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy;
+ typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy;
+
+ /// LazyResolverFn - The target lazy resolver function that we actually
+ /// rewrite instructions to use.
+ TargetJITInfo::LazyResolverFn LazyResolverFn;
+
+ JITResolverState state;
+
+ /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap
+ /// for external functions. TODO: Of course, external functions don't need
+ /// a lazy stub. It's actually here to make it more likely that far calls
+ /// succeed, but no single stub can guarantee that. I'll remove this in a
+ /// subsequent checkin when I actually fix far calls.
+ std::map<void*, void*> ExternalFnToStubMap;
+
+ /// revGOTMap - map addresses to indexes in the GOT
+ std::map<void*, unsigned> revGOTMap;
+ unsigned nextGOTIndex;
+
+ JITEmitter &JE;
+
+ /// Instance of JIT corresponding to this Resolver.
+ JIT *TheJIT;
+
+ public:
+ explicit JITResolver(JIT &jit, JITEmitter &je)
+ : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) {
+ LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn);
+ }
+
+ ~JITResolver();
+
+ /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's
+ /// lazy-compilation stub if it has already been created.
+ void *getLazyFunctionStubIfAvailable(Function *F);
+
+ /// getLazyFunctionStub - This returns a pointer to a function's
+ /// lazy-compilation stub, creating one on demand as needed.
+ void *getLazyFunctionStub(Function *F);
+
+ /// getExternalFunctionStub - Return a stub for the function at the
+ /// specified address, created lazily on demand.
+ void *getExternalFunctionStub(void *FnAddr);
+
+ /// getGlobalValueIndirectSym - Return an indirect symbol containing the
+ /// specified GV address.
+ void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress);
+
+ /// getGOTIndexForAddress - Return a new or existing index in the GOT for
+ /// an address. This function only manages slots, it does not manage the
+ /// contents of the slots or the memory associated with the GOT.
+ unsigned getGOTIndexForAddr(void *addr);
+
+ /// JITCompilerFn - This function is called to resolve a stub to a compiled
+ /// address. If the LLVM Function corresponding to the stub has not yet
+ /// been compiled, this function compiles it first.
+ static void *JITCompilerFn(void *Stub);
+ };
+
+ class StubToResolverMapTy {
+ /// Map a stub address to a specific instance of a JITResolver so that
+ /// lazily-compiled functions can find the right resolver to use.
+ ///
+ /// Guarded by Lock.
+ std::map<void*, JITResolver*> Map;
+
+ /// Guards Map from concurrent accesses.
+ mutable sys::Mutex Lock;
+
+ public:
+ /// Registers a Stub to be resolved by Resolver.
+ void RegisterStubResolver(void *Stub, JITResolver *Resolver) {
+ MutexGuard guard(Lock);
+ Map.insert(std::make_pair(Stub, Resolver));
+ }
+ /// Unregisters the Stub when it's invalidated.
+ void UnregisterStubResolver(void *Stub) {
+ MutexGuard guard(Lock);
+ Map.erase(Stub);
+ }
+ /// Returns the JITResolver instance that owns the Stub.
+ JITResolver *getResolverFromStub(void *Stub) const {
+ MutexGuard guard(Lock);
+ // The address given to us for the stub may not be exactly right, it might
+ // be a little bit after the stub. As such, use upper_bound to find it.
+ // This is the same trick as in LookupFunctionFromCallSite from
+ // JITResolverState.
+ std::map<void*, JITResolver*>::const_iterator I = Map.upper_bound(Stub);
+ assert(I != Map.begin() && "This is not a known stub!");
+ --I;
+ return I->second;
+ }
+ /// True if any stubs refer to the given resolver. Only used in an assert().
+ /// O(N)
+ bool ResolverHasStubs(JITResolver* Resolver) const {
+ MutexGuard guard(Lock);
+ for (std::map<void*, JITResolver*>::const_iterator I = Map.begin(),
+ E = Map.end(); I != E; ++I) {
+ if (I->second == Resolver)
+ return true;
+ }
+ return false;
+ }
+ };
+ /// This needs to be static so that a lazy call stub can access it with no
+ /// context except the address of the stub.
+ ManagedStatic<StubToResolverMapTy> StubToResolverMap;
+
+ /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is
+ /// used to output functions to memory for execution.
+ class JITEmitter : public JITCodeEmitter {
+ JITMemoryManager *MemMgr;
+
+ // When outputting a function stub in the context of some other function, we
+ // save BufferBegin/BufferEnd/CurBufferPtr here.
+ uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr;
+
+ // When reattempting to JIT a function after running out of space, we store
+ // the estimated size of the function we're trying to JIT here, so we can
+ // ask the memory manager for at least this much space. When we
+ // successfully emit the function, we reset this back to zero.
+ uintptr_t SizeEstimate;
+
+ /// Relocations - These are the relocations that the function needs, as
+ /// emitted.
+ std::vector<MachineRelocation> Relocations;
+
+ /// MBBLocations - This vector is a mapping from MBB ID's to their address.
+ /// It is filled in by the StartMachineBasicBlock callback and queried by
+ /// the getMachineBasicBlockAddress callback.
+ std::vector<uintptr_t> MBBLocations;
+
+ /// ConstantPool - The constant pool for the current function.
+ ///
+ MachineConstantPool *ConstantPool;
+
+ /// ConstantPoolBase - A pointer to the first entry in the constant pool.
+ ///
+ void *ConstantPoolBase;
+
+ /// ConstPoolAddresses - Addresses of individual constant pool entries.
+ ///
+ SmallVector<uintptr_t, 8> ConstPoolAddresses;
+
+ /// JumpTable - The jump tables for the current function.
+ ///
+ MachineJumpTableInfo *JumpTable;
+
+ /// JumpTableBase - A pointer to the first entry in the jump table.
+ ///
+ void *JumpTableBase;
+
+ /// Resolver - This contains info about the currently resolved functions.
+ JITResolver Resolver;
+
+ /// LabelLocations - This vector is a mapping from Label ID's to their
+ /// address.
+ DenseMap<MCSymbol*, uintptr_t> LabelLocations;
+
+ /// MMI - Machine module info for exception informations
+ MachineModuleInfo* MMI;
+
+ // CurFn - The llvm function being emitted. Only valid during
+ // finishFunction().
+ const Function *CurFn;
+
+ /// Information about emitted code, which is passed to the
+ /// JITEventListeners. This is reset in startFunction and used in
+ /// finishFunction.
+ JITEvent_EmittedFunctionDetails EmissionDetails;
+
+ struct EmittedCode {
+ void *FunctionBody; // Beginning of the function's allocation.
+ void *Code; // The address the function's code actually starts at.
+ void *ExceptionTable;
+ EmittedCode() : FunctionBody(nullptr), Code(nullptr),
+ ExceptionTable(nullptr) {}
+ };
+ struct EmittedFunctionConfig : public ValueMapConfig<const Function*> {
+ typedef JITEmitter *ExtraData;
+ static void onDelete(JITEmitter *, const Function*);
+ static void onRAUW(JITEmitter *, const Function*, const Function*);
+ };
+ ValueMap<const Function *, EmittedCode,
+ EmittedFunctionConfig> EmittedFunctions;
+
+ DebugLoc PrevDL;
+
+ /// Instance of the JIT
+ JIT *TheJIT;
+
+ public:
+ JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM)
+ : SizeEstimate(0), Resolver(jit, *this), MMI(nullptr), CurFn(nullptr),
+ EmittedFunctions(this), TheJIT(&jit) {
+ MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager();
+ if (jit.getJITInfo().needsGOT()) {
+ MemMgr->AllocateGOT();
+ DEBUG(dbgs() << "JIT is managing a GOT\n");
+ }
+
+ }
+ ~JITEmitter() {
+ delete MemMgr;
+ }
+
+ JITResolver &getJITResolver() { return Resolver; }
+
+ void startFunction(MachineFunction &F) override;
+ bool finishFunction(MachineFunction &F) override;
+
+ void emitConstantPool(MachineConstantPool *MCP);
+ void initJumpTableInfo(MachineJumpTableInfo *MJTI);
+ void emitJumpTableInfo(MachineJumpTableInfo *MJTI);
+
+ void startGVStub(const GlobalValue* GV,
+ unsigned StubSize, unsigned Alignment = 1);
+ void startGVStub(void *Buffer, unsigned StubSize);
+ void finishGVStub();
+ void *allocIndirectGV(const GlobalValue *GV, const uint8_t *Buffer,
+ size_t Size, unsigned Alignment) override;
+
+ /// allocateSpace - Reserves space in the current block if any, or
+ /// allocate a new one of the given size.
+ void *allocateSpace(uintptr_t Size, unsigned Alignment) override;
+
+ /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace,
+ /// this method does not allocate memory in the current output buffer,
+ /// because a global may live longer than the current function.
+ void *allocateGlobal(uintptr_t Size, unsigned Alignment) override;
+
+ void addRelocation(const MachineRelocation &MR) override {
+ Relocations.push_back(MR);
+ }
+
+ void StartMachineBasicBlock(MachineBasicBlock *MBB) override {
+ if (MBBLocations.size() <= (unsigned)MBB->getNumber())
+ MBBLocations.resize((MBB->getNumber()+1)*2);
+ MBBLocations[MBB->getNumber()] = getCurrentPCValue();
+ if (MBB->hasAddressTaken())
+ TheJIT->addPointerToBasicBlock(MBB->getBasicBlock(),
+ (void*)getCurrentPCValue());
+ DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at ["
+ << (void*) getCurrentPCValue() << "]\n");
+ }
+
+ uintptr_t getConstantPoolEntryAddress(unsigned Entry) const override;
+ uintptr_t getJumpTableEntryAddress(unsigned Entry) const override;
+
+ uintptr_t
+ getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override {
+ assert(MBBLocations.size() > (unsigned)MBB->getNumber() &&
+ MBBLocations[MBB->getNumber()] && "MBB not emitted!");
+ return MBBLocations[MBB->getNumber()];
+ }
+
+ /// retryWithMoreMemory - Log a retry and deallocate all memory for the
+ /// given function. Increase the minimum allocation size so that we get
+ /// more memory next time.
+ void retryWithMoreMemory(MachineFunction &F);
+
+ /// deallocateMemForFunction - Deallocate all memory for the specified
+ /// function body.
+ void deallocateMemForFunction(const Function *F);
+
+ void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) override;
+
+ void emitLabel(MCSymbol *Label) override {
+ LabelLocations[Label] = getCurrentPCValue();
+ }
+
+ DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() override {
+ return &LabelLocations;
+ }
+
+ uintptr_t getLabelAddress(MCSymbol *Label) const override {
+ assert(LabelLocations.count(Label) && "Label not emitted!");
+ return LabelLocations.find(Label)->second;
+ }
+
+ void setModuleInfo(MachineModuleInfo* Info) override {
+ MMI = Info;
+ }
+
+ private:
+ void *getPointerToGlobal(GlobalValue *GV, void *Reference,
+ bool MayNeedFarStub);
+ void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference);
+ };
+}
+
+void CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) {
+ JRS->EraseAllCallSitesForPrelocked(F);
+}
+
+void JITResolverState::EraseAllCallSitesForPrelocked(Function *F) {
+ FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F);
+ if (F2C == FunctionToCallSitesMap.end())
+ return;
+ StubToResolverMapTy &S2RMap = *StubToResolverMap;
+ for (SmallPtrSet<void*, 1>::const_iterator I = F2C->second.begin(),
+ E = F2C->second.end(); I != E; ++I) {
+ S2RMap.UnregisterStubResolver(*I);
+ bool Erased = CallSiteToFunctionMap.erase(*I);
+ (void)Erased;
+ assert(Erased && "Missing call site->function mapping");
+ }
+ FunctionToCallSitesMap.erase(F2C);
+}
+
+void JITResolverState::EraseAllCallSitesPrelocked() {
+ StubToResolverMapTy &S2RMap = *StubToResolverMap;
+ for (CallSiteToFunctionMapTy::const_iterator
+ I = CallSiteToFunctionMap.begin(),
+ E = CallSiteToFunctionMap.end(); I != E; ++I) {
+ S2RMap.UnregisterStubResolver(I->first);
+ }
+ CallSiteToFunctionMap.clear();
+ FunctionToCallSitesMap.clear();
+}
+
+JITResolver::~JITResolver() {
+ // No need to lock because we're in the destructor, and state isn't shared.
+ state.EraseAllCallSitesPrelocked();
+ assert(!StubToResolverMap->ResolverHasStubs(this) &&
+ "Resolver destroyed with stubs still alive.");
+}
+
+/// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub
+/// if it has already been created.
+void *JITResolver::getLazyFunctionStubIfAvailable(Function *F) {
+ MutexGuard locked(TheJIT->lock);
+
+ // If we already have a stub for this function, recycle it.
+ return state.getFunctionToLazyStubMap().lookup(F);
+}
+
+/// getFunctionStub - This returns a pointer to a function stub, creating
+/// one on demand as needed.
+void *JITResolver::getLazyFunctionStub(Function *F) {
+ MutexGuard locked(TheJIT->lock);
+
+ // If we already have a lazy stub for this function, recycle it.
+ void *&Stub = state.getFunctionToLazyStubMap()[F];
+ if (Stub) return Stub;
+
+ // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we
+ // must resolve the symbol now.
+ void *Actual = TheJIT->isCompilingLazily()
+ ? (void *)(intptr_t)LazyResolverFn : (void *)nullptr;
+
+ // If this is an external declaration, attempt to resolve the address now
+ // to place in the stub.
+ if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) {
+ Actual = TheJIT->getPointerToFunction(F);
+
+ // If we resolved the symbol to a null address (eg. a weak external)
+ // don't emit a stub. Return a null pointer to the application.
+ if (!Actual) return nullptr;
+ }
+
+ TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout();
+ JE.startGVStub(F, SL.Size, SL.Alignment);
+ // Codegen a new stub, calling the lazy resolver or the actual address of the
+ // external function, if it was resolved.
+ Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE);
+ JE.finishGVStub();
+
+ if (Actual != (void*)(intptr_t)LazyResolverFn) {
+ // If we are getting the stub for an external function, we really want the
+ // address of the stub in the GlobalAddressMap for the JIT, not the address
+ // of the external function.
+ TheJIT->updateGlobalMapping(F, Stub);
+ }
+
+ DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '"
+ << F->getName() << "'\n");
+
+ if (TheJIT->isCompilingLazily()) {
+ // Register this JITResolver as the one corresponding to this call site so
+ // JITCompilerFn will be able to find it.
+ StubToResolverMap->RegisterStubResolver(Stub, this);
+
+ // Finally, keep track of the stub-to-Function mapping so that the
+ // JITCompilerFn knows which function to compile!
+ state.AddCallSite(Stub, F);
+ } else if (!Actual) {
+ // If we are JIT'ing non-lazily but need to call a function that does not
+ // exist yet, add it to the JIT's work list so that we can fill in the
+ // stub address later.
+ assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() &&
+ "'Actual' should have been set above.");
+ TheJIT->addPendingFunction(F);
+ }
+
+ return Stub;
+}
+
+/// getGlobalValueIndirectSym - Return a lazy pointer containing the specified
+/// GV address.
+void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) {
+ MutexGuard locked(TheJIT->lock);
+
+ // If we already have a stub for this global variable, recycle it.
+ void *&IndirectSym = state.getGlobalToIndirectSymMap()[GV];
+ if (IndirectSym) return IndirectSym;
+
+ // Otherwise, codegen a new indirect symbol.
+ IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress,
+ JE);
+
+ DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym
+ << "] for GV '" << GV->getName() << "'\n");
+
+ return IndirectSym;
+}
+
+/// getExternalFunctionStub - Return a stub for the function at the
+/// specified address, created lazily on demand.
+void *JITResolver::getExternalFunctionStub(void *FnAddr) {
+ // If we already have a stub for this function, recycle it.
+ void *&Stub = ExternalFnToStubMap[FnAddr];
+ if (Stub) return Stub;
+
+ TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout();
+ JE.startGVStub(nullptr, SL.Size, SL.Alignment);
+ Stub = TheJIT->getJITInfo().emitFunctionStub(nullptr, FnAddr, JE);
+ JE.finishGVStub();
+
+ DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub
+ << "] for external function at '" << FnAddr << "'\n");
+ return Stub;
+}
+
+unsigned JITResolver::getGOTIndexForAddr(void* addr) {
+ unsigned idx = revGOTMap[addr];
+ if (!idx) {
+ idx = ++nextGOTIndex;
+ revGOTMap[addr] = idx;
+ DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr ["
+ << addr << "]\n");
+ }
+ return idx;
+}
+
+/// JITCompilerFn - This function is called when a lazy compilation stub has
+/// been entered. It looks up which function this stub corresponds to, compiles
+/// it if necessary, then returns the resultant function pointer.
+void *JITResolver::JITCompilerFn(void *Stub) {
+ JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub);
+ assert(JR && "Unable to find the corresponding JITResolver to the call site");
+
+ Function* F = nullptr;
+ void* ActualPtr = nullptr;
+
+ {
+ // Only lock for getting the Function. The call getPointerToFunction made
+ // in this function might trigger function materializing, which requires
+ // JIT lock to be unlocked.
+ MutexGuard locked(JR->TheJIT->lock);
+
+ // The address given to us for the stub may not be exactly right, it might
+ // be a little bit after the stub. As such, use upper_bound to find it.
+ std::pair<void*, Function*> I =
+ JR->state.LookupFunctionFromCallSite(Stub);
+ F = I.second;
+ ActualPtr = I.first;
+ }
+
+ // If we have already code generated the function, just return the address.
+ void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F);
+
+ if (!Result) {
+ // Otherwise we don't have it, do lazy compilation now.
+
+ // If lazy compilation is disabled, emit a useful error message and abort.
+ if (!JR->TheJIT->isCompilingLazily()) {
+ report_fatal_error("LLVM JIT requested to do lazy compilation of"
+ " function '"
+ + F->getName() + "' when lazy compiles are disabled!");
+ }
+
+ DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName()
+ << "' In stub ptr = " << Stub << " actual ptr = "
+ << ActualPtr << "\n");
+ (void)ActualPtr;
+
+ Result = JR->TheJIT->getPointerToFunction(F);
+ }
+
+ // Reacquire the lock to update the GOT map.
+ MutexGuard locked(JR->TheJIT->lock);
+
+ // We might like to remove the call site from the CallSiteToFunction map, but
+ // we can't do that! Multiple threads could be stuck, waiting to acquire the
+ // lock above. As soon as the 1st function finishes compiling the function,
+ // the next one will be released, and needs to be able to find the function it
+ // needs to call.
+
+ // FIXME: We could rewrite all references to this stub if we knew them.
+
+ // What we will do is set the compiled function address to map to the
+ // same GOT entry as the stub so that later clients may update the GOT
+ // if they see it still using the stub address.
+ // Note: this is done so the Resolver doesn't have to manage GOT memory
+ // Do this without allocating map space if the target isn't using a GOT
+ if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end())
+ JR->revGOTMap[Result] = JR->revGOTMap[Stub];
+
+ return Result;
+}
+
+//===----------------------------------------------------------------------===//
+// JITEmitter code.
+//
+
+static GlobalObject *getSimpleAliasee(Constant *C) {
+ C = C->stripPointerCasts();
+ return dyn_cast<GlobalObject>(C);
+}
+
+void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference,
+ bool MayNeedFarStub) {
+ if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
+ return TheJIT->getOrEmitGlobalVariable(GV);
+
+ if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) {
+ // We can only handle simple cases.
+ if (GlobalValue *GV = getSimpleAliasee(GA->getAliasee()))
+ return TheJIT->getPointerToGlobal(GV);
+ return nullptr;
+ }
+
+ // If we have already compiled the function, return a pointer to its body.
+ Function *F = cast<Function>(V);
+
+ void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F);
+ if (FnStub) {
+ // Return the function stub if it's already created. We do this first so
+ // that we're returning the same address for the function as any previous
+ // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be
+ // close enough to call.
+ return FnStub;
+ }
+
+ // If we know the target can handle arbitrary-distance calls, try to
+ // return a direct pointer.
+ if (!MayNeedFarStub) {
+ // If we have code, go ahead and return that.
+ void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F);
+ if (ResultPtr) return ResultPtr;
+
+ // If this is an external function pointer, we can force the JIT to
+ // 'compile' it, which really just adds it to the map.
+ if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage())
+ return TheJIT->getPointerToFunction(F);
+ }
+
+ // Otherwise, we may need a to emit a stub, and, conservatively, we always do
+ // so. Note that it's possible to return null from getLazyFunctionStub in the
+ // case of a weak extern that fails to resolve.
+ return Resolver.getLazyFunctionStub(F);
+}
+
+void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) {
+ // Make sure GV is emitted first, and create a stub containing the fully
+ // resolved address.
+ void *GVAddress = getPointerToGlobal(V, Reference, false);
+ void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress);
+ return StubAddr;
+}
+
+void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) {
+ if (DL.isUnknown()) return;
+ if (!BeforePrintingInsn) return;
+
+ const LLVMContext &Context = EmissionDetails.MF->getFunction()->getContext();
+
+ if (DL.getScope(Context) != nullptr && PrevDL != DL) {
+ JITEvent_EmittedFunctionDetails::LineStart NextLine;
+ NextLine.Address = getCurrentPCValue();
+ NextLine.Loc = DL;
+ EmissionDetails.LineStarts.push_back(NextLine);
+ }
+
+ PrevDL = DL;
+}
+
+static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP,
+ const DataLayout *TD) {
+ const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants();
+ if (Constants.empty()) return 0;
+
+ unsigned Size = 0;
+ for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
+ MachineConstantPoolEntry CPE = Constants[i];
+ unsigned AlignMask = CPE.getAlignment() - 1;
+ Size = (Size + AlignMask) & ~AlignMask;
+ Type *Ty = CPE.getType();
+ Size += TD->getTypeAllocSize(Ty);
+ }
+ return Size;
+}
+
+void JITEmitter::startFunction(MachineFunction &F) {
+ DEBUG(dbgs() << "JIT: Starting CodeGen of Function "
+ << F.getName() << "\n");
+
+ uintptr_t ActualSize = 0;
+ // Set the memory writable, if it's not already
+ MemMgr->setMemoryWritable();
+
+ if (SizeEstimate > 0) {
+ // SizeEstimate will be non-zero on reallocation attempts.
+ ActualSize = SizeEstimate;
+ }
+
+ BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(),
+ ActualSize);
+ BufferEnd = BufferBegin+ActualSize;
+ EmittedFunctions[F.getFunction()].FunctionBody = BufferBegin;
+
+ // Ensure the constant pool/jump table info is at least 4-byte aligned.
+ emitAlignment(16);
+
+ emitConstantPool(F.getConstantPool());
+ if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo())
+ initJumpTableInfo(MJTI);
+
+ // About to start emitting the machine code for the function.
+ emitAlignment(std::max(F.getFunction()->getAlignment(), 8U));
+ TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr);
+ EmittedFunctions[F.getFunction()].Code = CurBufferPtr;
+
+ MBBLocations.clear();
+
+ EmissionDetails.MF = &F;
+ EmissionDetails.LineStarts.clear();
+}
+
+bool JITEmitter::finishFunction(MachineFunction &F) {
+ if (CurBufferPtr == BufferEnd) {
+ // We must call endFunctionBody before retrying, because
+ // deallocateMemForFunction requires it.
+ MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr);
+ retryWithMoreMemory(F);
+ return true;
+ }
+
+ if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo())
+ emitJumpTableInfo(MJTI);
+
+ // FnStart is the start of the text, not the start of the constant pool and
+ // other per-function data.
+ uint8_t *FnStart =
+ (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction());
+
+ // FnEnd is the end of the function's machine code.
+ uint8_t *FnEnd = CurBufferPtr;
+
+ if (!Relocations.empty()) {
+ CurFn = F.getFunction();
+ NumRelos += Relocations.size();
+
+ // Resolve the relocations to concrete pointers.
+ for (unsigned i = 0, e = Relocations.size(); i != e; ++i) {
+ MachineRelocation &MR = Relocations[i];
+ void *ResultPtr = nullptr;
+ if (!MR.letTargetResolve()) {
+ if (MR.isExternalSymbol()) {
+ ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(),
+ false);
+ DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to ["
+ << ResultPtr << "]\n");
+
+ // If the target REALLY wants a stub for this function, emit it now.
+ if (MR.mayNeedFarStub()) {
+ ResultPtr = Resolver.getExternalFunctionStub(ResultPtr);
+ }
+ } else if (MR.isGlobalValue()) {
+ ResultPtr = getPointerToGlobal(MR.getGlobalValue(),
+ BufferBegin+MR.getMachineCodeOffset(),
+ MR.mayNeedFarStub());
+ } else if (MR.isIndirectSymbol()) {
+ ResultPtr = getPointerToGVIndirectSym(
+ MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset());
+ } else if (MR.isBasicBlock()) {
+ ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock());
+ } else if (MR.isConstantPoolIndex()) {
+ ResultPtr =
+ (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex());
+ } else {
+ assert(MR.isJumpTableIndex());
+ ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex());
+ }
+
+ MR.setResultPointer(ResultPtr);
+ }
+
+ // if we are managing the GOT and the relocation wants an index,
+ // give it one
+ if (MR.isGOTRelative() && MemMgr->isManagingGOT()) {
+ unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr);
+ MR.setGOTIndex(idx);
+ if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) {
+ DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr
+ << " pointing at " << ((void**)MemMgr->getGOTBase())[idx]
+ << "\n");
+ ((void**)MemMgr->getGOTBase())[idx] = ResultPtr;
+ }
+ }
+ }
+
+ CurFn = nullptr;
+ TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0],
+ Relocations.size(), MemMgr->getGOTBase());
+ }
+
+ // Update the GOT entry for F to point to the new code.
+ if (MemMgr->isManagingGOT()) {
+ unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin);
+ if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) {
+ DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin
+ << " pointing at " << ((void**)MemMgr->getGOTBase())[idx]
+ << "\n");
+ ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin;
+ }
+ }
+
+ // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for
+ // global variables that were referenced in the relocations.
+ MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr);
+
+ if (CurBufferPtr == BufferEnd) {
+ retryWithMoreMemory(F);
+ return true;
+ } else {
+ // Now that we've succeeded in emitting the function, reset the
+ // SizeEstimate back down to zero.
+ SizeEstimate = 0;
+ }
+
+ BufferBegin = CurBufferPtr = nullptr;
+ NumBytes += FnEnd-FnStart;
+
+ // Invalidate the icache if necessary.
+ sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart);
+
+ TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart,
+ EmissionDetails);
+
+ // Reset the previous debug location.
+ PrevDL = DebugLoc();
+
+ DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart
+ << "] Function: " << F.getName()
+ << ": " << (FnEnd-FnStart) << " bytes of text, "
+ << Relocations.size() << " relocations\n");
+
+ Relocations.clear();
+ ConstPoolAddresses.clear();
+
+ // Mark code region readable and executable if it's not so already.
+ MemMgr->setMemoryExecutable();
+
+ DEBUG({
+ dbgs() << "JIT: Binary code:\n";
+ uint8_t* q = FnStart;
+ for (int i = 0; q < FnEnd; q += 4, ++i) {
+ if (i == 4)
+ i = 0;
+ if (i == 0)
+ dbgs() << "JIT: " << (long)(q - FnStart) << ": ";
+ bool Done = false;
+ for (int j = 3; j >= 0; --j) {
+ if (q + j >= FnEnd)
+ Done = true;
+ else
+ dbgs() << (unsigned short)q[j];
+ }
+ if (Done)
+ break;
+ dbgs() << ' ';
+ if (i == 3)
+ dbgs() << '\n';
+ }
+ dbgs()<< '\n';
+ });
+
+ if (MMI)
+ MMI->EndFunction();
+
+ return false;
+}
+
+void JITEmitter::retryWithMoreMemory(MachineFunction &F) {
+ DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n");
+ Relocations.clear(); // Clear the old relocations or we'll reapply them.
+ ConstPoolAddresses.clear();
+ ++NumRetries;
+ deallocateMemForFunction(F.getFunction());
+ // Try again with at least twice as much free space.
+ SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin));
+
+ for (MachineFunction::iterator MBB = F.begin(), E = F.end(); MBB != E; ++MBB){
+ if (MBB->hasAddressTaken())
+ TheJIT->clearPointerToBasicBlock(MBB->getBasicBlock());
+ }
+}
+
+/// deallocateMemForFunction - Deallocate all memory for the specified
+/// function body. Also drop any references the function has to stubs.
+/// May be called while the Function is being destroyed inside ~Value().
+void JITEmitter::deallocateMemForFunction(const Function *F) {
+ ValueMap<const Function *, EmittedCode, EmittedFunctionConfig>::iterator
+ Emitted = EmittedFunctions.find(F);
+ if (Emitted != EmittedFunctions.end()) {
+ MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody);
+ TheJIT->NotifyFreeingMachineCode(Emitted->second.Code);
+
+ EmittedFunctions.erase(Emitted);
+ }
+}
+
+
+void *JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) {
+ if (BufferBegin)
+ return JITCodeEmitter::allocateSpace(Size, Alignment);
+
+ // create a new memory block if there is no active one.
+ // care must be taken so that BufferBegin is invalidated when a
+ // block is trimmed
+ BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment);
+ BufferEnd = BufferBegin+Size;
+ return CurBufferPtr;
+}
+
+void *JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) {
+ // Delegate this call through the memory manager.
+ return MemMgr->allocateGlobal(Size, Alignment);
+}
+
+void JITEmitter::emitConstantPool(MachineConstantPool *MCP) {
+ if (TheJIT->getJITInfo().hasCustomConstantPool())
+ return;
+
+ const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants();
+ if (Constants.empty()) return;
+
+ unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getDataLayout());
+ unsigned Align = MCP->getConstantPoolAlignment();
+ ConstantPoolBase = allocateSpace(Size, Align);
+ ConstantPool = MCP;
+
+ if (!ConstantPoolBase) return; // Buffer overflow.
+
+ DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase
+ << "] (size: " << Size << ", alignment: " << Align << ")\n");
+
+ // Initialize the memory for all of the constant pool entries.
+ unsigned Offset = 0;
+ for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
+ MachineConstantPoolEntry CPE = Constants[i];
+ unsigned AlignMask = CPE.getAlignment() - 1;
+ Offset = (Offset + AlignMask) & ~AlignMask;
+
+ uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset;
+ ConstPoolAddresses.push_back(CAddr);
+ if (CPE.isMachineConstantPoolEntry()) {
+ // FIXME: add support to lower machine constant pool values into bytes!
+ report_fatal_error("Initialize memory with machine specific constant pool"
+ "entry has not been implemented!");
+ }
+ TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr);
+ DEBUG(dbgs() << "JIT: CP" << i << " at [0x";
+ dbgs().write_hex(CAddr) << "]\n");
+
+ Type *Ty = CPE.Val.ConstVal->getType();
+ Offset += TheJIT->getDataLayout()->getTypeAllocSize(Ty);
+ }
+}
+
+void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) {
+ if (TheJIT->getJITInfo().hasCustomJumpTables())
+ return;
+ if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline)
+ return;
+
+ const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
+ if (JT.empty()) return;
+
+ unsigned NumEntries = 0;
+ for (unsigned i = 0, e = JT.size(); i != e; ++i)
+ NumEntries += JT[i].MBBs.size();
+
+ unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getDataLayout());
+
+ // Just allocate space for all the jump tables now. We will fix up the actual
+ // MBB entries in the tables after we emit the code for each block, since then
+ // we will know the final locations of the MBBs in memory.
+ JumpTable = MJTI;
+ JumpTableBase = allocateSpace(NumEntries * EntrySize,
+ MJTI->getEntryAlignment(*TheJIT->getDataLayout()));
+}
+
+void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) {
+ if (TheJIT->getJITInfo().hasCustomJumpTables())
+ return;
+
+ const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
+ if (JT.empty() || !JumpTableBase) return;
+
+
+ switch (MJTI->getEntryKind()) {
+ case MachineJumpTableInfo::EK_Inline:
+ return;
+ case MachineJumpTableInfo::EK_BlockAddress: {
+ // EK_BlockAddress - Each entry is a plain address of block, e.g.:
+ // .word LBB123
+ assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == sizeof(void*) &&
+ "Cross JIT'ing?");
+
+ // For each jump table, map each target in the jump table to the address of
+ // an emitted MachineBasicBlock.
+ intptr_t *SlotPtr = (intptr_t*)JumpTableBase;
+
+ for (unsigned i = 0, e = JT.size(); i != e; ++i) {
+ const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs;
+ // Store the address of the basic block for this jump table slot in the
+ // memory we allocated for the jump table in 'initJumpTableInfo'
+ for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi)
+ *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]);
+ }
+ break;
+ }
+
+ case MachineJumpTableInfo::EK_Custom32:
+ case MachineJumpTableInfo::EK_GPRel32BlockAddress:
+ case MachineJumpTableInfo::EK_LabelDifference32: {
+ assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == 4&&"Cross JIT'ing?");
+ // For each jump table, place the offset from the beginning of the table
+ // to the target address.
+ int *SlotPtr = (int*)JumpTableBase;
+
+ for (unsigned i = 0, e = JT.size(); i != e; ++i) {
+ const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs;
+ // Store the offset of the basic block for this jump table slot in the
+ // memory we allocated for the jump table in 'initJumpTableInfo'
+ uintptr_t Base = (uintptr_t)SlotPtr;
+ for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) {
+ uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]);
+ /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook.
+ *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base);
+ }
+ }
+ break;
+ }
+ case MachineJumpTableInfo::EK_GPRel64BlockAddress:
+ llvm_unreachable(
+ "JT Info emission not implemented for GPRel64BlockAddress yet.");
+ }
+}
+
+void JITEmitter::startGVStub(const GlobalValue* GV,
+ unsigned StubSize, unsigned Alignment) {
+ SavedBufferBegin = BufferBegin;
+ SavedBufferEnd = BufferEnd;
+ SavedCurBufferPtr = CurBufferPtr;
+
+ BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment);
+ BufferEnd = BufferBegin+StubSize+1;
+}
+
+void JITEmitter::startGVStub(void *Buffer, unsigned StubSize) {
+ SavedBufferBegin = BufferBegin;
+ SavedBufferEnd = BufferEnd;
+ SavedCurBufferPtr = CurBufferPtr;
+
+ BufferBegin = CurBufferPtr = (uint8_t *)Buffer;
+ BufferEnd = BufferBegin+StubSize+1;
+}
+
+void JITEmitter::finishGVStub() {
+ assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space.");
+ NumBytes += getCurrentPCOffset();
+ BufferBegin = SavedBufferBegin;
+ BufferEnd = SavedBufferEnd;
+ CurBufferPtr = SavedCurBufferPtr;
+}
+
+void *JITEmitter::allocIndirectGV(const GlobalValue *GV,
+ const uint8_t *Buffer, size_t Size,
+ unsigned Alignment) {
+ uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment);
+ memcpy(IndGV, Buffer, Size);
+ return IndGV;
+}
+
+// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry
+// in the constant pool that was last emitted with the 'emitConstantPool'
+// method.
+//
+uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const {
+ assert(ConstantNum < ConstantPool->getConstants().size() &&
+ "Invalid ConstantPoolIndex!");
+ return ConstPoolAddresses[ConstantNum];
+}
+
+// getJumpTableEntryAddress - Return the address of the JumpTable with index
+// 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo'
+//
+uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const {
+ const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables();
+ assert(Index < JT.size() && "Invalid jump table index!");
+
+ unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getDataLayout());
+
+ unsigned Offset = 0;
+ for (unsigned i = 0; i < Index; ++i)
+ Offset += JT[i].MBBs.size();
+
+ Offset *= EntrySize;
+
+ return (uintptr_t)((char *)JumpTableBase + Offset);
+}
+
+void JITEmitter::EmittedFunctionConfig::onDelete(
+ JITEmitter *Emitter, const Function *F) {
+ Emitter->deallocateMemForFunction(F);
+}
+void JITEmitter::EmittedFunctionConfig::onRAUW(
+ JITEmitter *, const Function*, const Function*) {
+ llvm_unreachable("The JIT doesn't know how to handle a"
+ " RAUW on a value it has emitted.");
+}
+
+
+//===----------------------------------------------------------------------===//
+// Public interface to this file
+//===----------------------------------------------------------------------===//
+
+JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM,
+ TargetMachine &tm) {
+ return new JITEmitter(jit, JMM, tm);
+}
+
+// getPointerToFunctionOrStub - If the specified function has been
+// code-gen'd, return a pointer to the function. If not, compile it, or use
+// a stub to implement lazy compilation if available.
+//
+void *JIT::getPointerToFunctionOrStub(Function *F) {
+ // If we have already code generated the function, just return the address.
+ if (void *Addr = getPointerToGlobalIfAvailable(F))
+ return Addr;
+
+ // Get a stub if the target supports it.
+ JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter());
+ return JE->getJITResolver().getLazyFunctionStub(F);
+}
+
+void JIT::updateFunctionStubUnlocked(Function *F) {
+ // Get the empty stub we generated earlier.
+ JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter());
+ void *Stub = JE->getJITResolver().getLazyFunctionStub(F);
+ void *Addr = getPointerToGlobalIfAvailable(F);
+ assert(Addr != Stub && "Function must have non-stub address to be updated.");
+
+ // Tell the target jit info to rewrite the stub at the specified address,
+ // rather than creating a new one.
+ TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout();
+ JE->startGVStub(Stub, layout.Size);
+ getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter());
+ JE->finishGVStub();
+}
+
+/// freeMachineCodeForFunction - release machine code memory for given Function.
+///
+void JIT::freeMachineCodeForFunction(Function *F) {
+ // Delete translation for this from the ExecutionEngine, so it will get
+ // retranslated next time it is used.
+ updateGlobalMapping(F, nullptr);
+
+ // Free the actual memory for the function body and related stuff.
+ static_cast<JITEmitter*>(JCE)->deallocateMemForFunction(F);
+}
--- /dev/null
+//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the DefaultJITMemoryManager class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/JITMemoryManager.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Config/config.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Memory.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+#include <climits>
+#include <cstring>
+#include <vector>
+
+#if defined(__linux__)
+#if defined(HAVE_SYS_STAT_H)
+#include <sys/stat.h>
+#endif
+#include <fcntl.h>
+#include <unistd.h>
+#endif
+
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+STATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT");
+
+JITMemoryManager::~JITMemoryManager() {}
+
+//===----------------------------------------------------------------------===//
+// Memory Block Implementation.
+//===----------------------------------------------------------------------===//
+
+namespace {
+ /// MemoryRangeHeader - For a range of memory, this is the header that we put
+ /// on the block of memory. It is carefully crafted to be one word of memory.
+ /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader
+ /// which starts with this.
+ struct FreeRangeHeader;
+ struct MemoryRangeHeader {
+ /// ThisAllocated - This is true if this block is currently allocated. If
+ /// not, this can be converted to a FreeRangeHeader.
+ unsigned ThisAllocated : 1;
+
+ /// PrevAllocated - Keep track of whether the block immediately before us is
+ /// allocated. If not, the word immediately before this header is the size
+ /// of the previous block.
+ unsigned PrevAllocated : 1;
+
+ /// BlockSize - This is the size in bytes of this memory block,
+ /// including this header.
+ uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2);
+
+
+ /// getBlockAfter - Return the memory block immediately after this one.
+ ///
+ MemoryRangeHeader &getBlockAfter() const {
+ return *reinterpret_cast<MemoryRangeHeader *>(
+ reinterpret_cast<char*>(
+ const_cast<MemoryRangeHeader *>(this))+BlockSize);
+ }
+
+ /// getFreeBlockBefore - If the block before this one is free, return it,
+ /// otherwise return null.
+ FreeRangeHeader *getFreeBlockBefore() const {
+ if (PrevAllocated) return nullptr;
+ intptr_t PrevSize = reinterpret_cast<intptr_t *>(
+ const_cast<MemoryRangeHeader *>(this))[-1];
+ return reinterpret_cast<FreeRangeHeader *>(
+ reinterpret_cast<char*>(
+ const_cast<MemoryRangeHeader *>(this))-PrevSize);
+ }
+
+ /// FreeBlock - Turn an allocated block into a free block, adjusting
+ /// bits in the object headers, and adding an end of region memory block.
+ FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList);
+
+ /// TrimAllocationToSize - If this allocated block is significantly larger
+ /// than NewSize, split it into two pieces (where the former is NewSize
+ /// bytes, including the header), and add the new block to the free list.
+ FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList,
+ uint64_t NewSize);
+ };
+
+ /// FreeRangeHeader - For a memory block that isn't already allocated, this
+ /// keeps track of the current block and has a pointer to the next free block.
+ /// Free blocks are kept on a circularly linked list.
+ struct FreeRangeHeader : public MemoryRangeHeader {
+ FreeRangeHeader *Prev;
+ FreeRangeHeader *Next;
+
+ /// getMinBlockSize - Get the minimum size for a memory block. Blocks
+ /// smaller than this size cannot be created.
+ static unsigned getMinBlockSize() {
+ return sizeof(FreeRangeHeader)+sizeof(intptr_t);
+ }
+
+ /// SetEndOfBlockSizeMarker - The word at the end of every free block is
+ /// known to be the size of the free block. Set it for this block.
+ void SetEndOfBlockSizeMarker() {
+ void *EndOfBlock = (char*)this + BlockSize;
+ ((intptr_t *)EndOfBlock)[-1] = BlockSize;
+ }
+
+ FreeRangeHeader *RemoveFromFreeList() {
+ assert(Next->Prev == this && Prev->Next == this && "Freelist broken!");
+ Next->Prev = Prev;
+ return Prev->Next = Next;
+ }
+
+ void AddToFreeList(FreeRangeHeader *FreeList) {
+ Next = FreeList;
+ Prev = FreeList->Prev;
+ Prev->Next = this;
+ Next->Prev = this;
+ }
+
+ /// GrowBlock - The block after this block just got deallocated. Merge it
+ /// into the current block.
+ void GrowBlock(uintptr_t NewSize);
+
+ /// AllocateBlock - Mark this entire block allocated, updating freelists
+ /// etc. This returns a pointer to the circular free-list.
+ FreeRangeHeader *AllocateBlock();
+ };
+}
+
+
+/// AllocateBlock - Mark this entire block allocated, updating freelists
+/// etc. This returns a pointer to the circular free-list.
+FreeRangeHeader *FreeRangeHeader::AllocateBlock() {
+ assert(!ThisAllocated && !getBlockAfter().PrevAllocated &&
+ "Cannot allocate an allocated block!");
+ // Mark this block allocated.
+ ThisAllocated = 1;
+ getBlockAfter().PrevAllocated = 1;
+
+ // Remove it from the free list.
+ return RemoveFromFreeList();
+}
+
+/// FreeBlock - Turn an allocated block into a free block, adjusting
+/// bits in the object headers, and adding an end of region memory block.
+/// If possible, coalesce this block with neighboring blocks. Return the
+/// FreeRangeHeader to allocate from.
+FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) {
+ MemoryRangeHeader *FollowingBlock = &getBlockAfter();
+ assert(ThisAllocated && "This block is already free!");
+ assert(FollowingBlock->PrevAllocated && "Flags out of sync!");
+
+ FreeRangeHeader *FreeListToReturn = FreeList;
+
+ // If the block after this one is free, merge it into this block.
+ if (!FollowingBlock->ThisAllocated) {
+ FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock;
+ // "FreeList" always needs to be a valid free block. If we're about to
+ // coalesce with it, update our notion of what the free list is.
+ if (&FollowingFreeBlock == FreeList) {
+ FreeList = FollowingFreeBlock.Next;
+ FreeListToReturn = nullptr;
+ assert(&FollowingFreeBlock != FreeList && "No tombstone block?");
+ }
+ FollowingFreeBlock.RemoveFromFreeList();
+
+ // Include the following block into this one.
+ BlockSize += FollowingFreeBlock.BlockSize;
+ FollowingBlock = &FollowingFreeBlock.getBlockAfter();
+
+ // Tell the block after the block we are coalescing that this block is
+ // allocated.
+ FollowingBlock->PrevAllocated = 1;
+ }
+
+ assert(FollowingBlock->ThisAllocated && "Missed coalescing?");
+
+ if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) {
+ PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize);
+ return FreeListToReturn ? FreeListToReturn : PrevFreeBlock;
+ }
+
+ // Otherwise, mark this block free.
+ FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this;
+ FollowingBlock->PrevAllocated = 0;
+ FreeBlock.ThisAllocated = 0;
+
+ // Link this into the linked list of free blocks.
+ FreeBlock.AddToFreeList(FreeList);
+
+ // Add a marker at the end of the block, indicating the size of this free
+ // block.
+ FreeBlock.SetEndOfBlockSizeMarker();
+ return FreeListToReturn ? FreeListToReturn : &FreeBlock;
+}
+
+/// GrowBlock - The block after this block just got deallocated. Merge it
+/// into the current block.
+void FreeRangeHeader::GrowBlock(uintptr_t NewSize) {
+ assert(NewSize > BlockSize && "Not growing block?");
+ BlockSize = NewSize;
+ SetEndOfBlockSizeMarker();
+ getBlockAfter().PrevAllocated = 0;
+}
+
+/// TrimAllocationToSize - If this allocated block is significantly larger
+/// than NewSize, split it into two pieces (where the former is NewSize
+/// bytes, including the header), and add the new block to the free list.
+FreeRangeHeader *MemoryRangeHeader::
+TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) {
+ assert(ThisAllocated && getBlockAfter().PrevAllocated &&
+ "Cannot deallocate part of an allocated block!");
+
+ // Don't allow blocks to be trimmed below minimum required size
+ NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize);
+
+ // Round up size for alignment of header.
+ unsigned HeaderAlign = __alignof(FreeRangeHeader);
+ NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1);
+
+ // Size is now the size of the block we will remove from the start of the
+ // current block.
+ assert(NewSize <= BlockSize &&
+ "Allocating more space from this block than exists!");
+
+ // If splitting this block will cause the remainder to be too small, do not
+ // split the block.
+ if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize())
+ return FreeList;
+
+ // Otherwise, we splice the required number of bytes out of this block, form
+ // a new block immediately after it, then mark this block allocated.
+ MemoryRangeHeader &FormerNextBlock = getBlockAfter();
+
+ // Change the size of this block.
+ BlockSize = NewSize;
+
+ // Get the new block we just sliced out and turn it into a free block.
+ FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter();
+ NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock;
+ NewNextBlock.ThisAllocated = 0;
+ NewNextBlock.PrevAllocated = 1;
+ NewNextBlock.SetEndOfBlockSizeMarker();
+ FormerNextBlock.PrevAllocated = 0;
+ NewNextBlock.AddToFreeList(FreeList);
+ return &NewNextBlock;
+}
+
+//===----------------------------------------------------------------------===//
+// Memory Block Implementation.
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+ class DefaultJITMemoryManager;
+
+ class JITAllocator {
+ DefaultJITMemoryManager &JMM;
+ public:
+ JITAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { }
+ void *Allocate(size_t Size, size_t /*Alignment*/);
+ void Deallocate(void *Slab, size_t Size);
+ };
+
+ /// DefaultJITMemoryManager - Manage memory for the JIT code generation.
+ /// This splits a large block of MAP_NORESERVE'd memory into two
+ /// sections, one for function stubs, one for the functions themselves. We
+ /// have to do this because we may need to emit a function stub while in the
+ /// middle of emitting a function, and we don't know how large the function we
+ /// are emitting is.
+ class DefaultJITMemoryManager : public JITMemoryManager {
+ public:
+ /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at
+ /// least this much unless more is requested. Currently, in 512k slabs.
+ static const size_t DefaultCodeSlabSize = 512 * 1024;
+
+ /// DefaultSlabSize - Allocate globals and stubs into slabs of 64K (probably
+ /// 16 pages) unless we get an allocation above SizeThreshold.
+ static const size_t DefaultSlabSize = 64 * 1024;
+
+ /// DefaultSizeThreshold - For any allocation larger than 16K (probably
+ /// 4 pages), we should allocate a separate slab to avoid wasted space at
+ /// the end of a normal slab.
+ static const size_t DefaultSizeThreshold = 16 * 1024;
+
+ private:
+ // Whether to poison freed memory.
+ bool PoisonMemory;
+
+ /// LastSlab - This points to the last slab allocated and is used as the
+ /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all
+ /// stubs, data, and code contiguously in memory. In general, however, this
+ /// is not possible because the NearBlock parameter is ignored on Windows
+ /// platforms and even on Unix it works on a best-effort pasis.
+ sys::MemoryBlock LastSlab;
+
+ // Memory slabs allocated by the JIT. We refer to them as slabs so we don't
+ // confuse them with the blocks of memory described above.
+ std::vector<sys::MemoryBlock> CodeSlabs;
+ BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize,
+ DefaultSizeThreshold> StubAllocator;
+ BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize,
+ DefaultSizeThreshold> DataAllocator;
+
+ // Circular list of free blocks.
+ FreeRangeHeader *FreeMemoryList;
+
+ // When emitting code into a memory block, this is the block.
+ MemoryRangeHeader *CurBlock;
+
+ uint8_t *GOTBase; // Target Specific reserved memory
+ public:
+ DefaultJITMemoryManager();
+ ~DefaultJITMemoryManager();
+
+ /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the
+ /// last slab it allocated, so that subsequent allocations follow it.
+ sys::MemoryBlock allocateNewSlab(size_t size);
+
+ /// getPointerToNamedFunction - This method returns the address of the
+ /// specified function by using the dlsym function call.
+ void *getPointerToNamedFunction(const std::string &Name,
+ bool AbortOnFailure = true) override;
+
+ void AllocateGOT() override;
+
+ // Testing methods.
+ bool CheckInvariants(std::string &ErrorStr) override;
+ size_t GetDefaultCodeSlabSize() override { return DefaultCodeSlabSize; }
+ size_t GetDefaultDataSlabSize() override { return DefaultSlabSize; }
+ size_t GetDefaultStubSlabSize() override { return DefaultSlabSize; }
+ unsigned GetNumCodeSlabs() override { return CodeSlabs.size(); }
+ unsigned GetNumDataSlabs() override { return DataAllocator.GetNumSlabs(); }
+ unsigned GetNumStubSlabs() override { return StubAllocator.GetNumSlabs(); }
+
+ /// startFunctionBody - When a function starts, allocate a block of free
+ /// executable memory, returning a pointer to it and its actual size.
+ uint8_t *startFunctionBody(const Function *F,
+ uintptr_t &ActualSize) override {
+
+ FreeRangeHeader* candidateBlock = FreeMemoryList;
+ FreeRangeHeader* head = FreeMemoryList;
+ FreeRangeHeader* iter = head->Next;
+
+ uintptr_t largest = candidateBlock->BlockSize;
+
+ // Search for the largest free block
+ while (iter != head) {
+ if (iter->BlockSize > largest) {
+ largest = iter->BlockSize;
+ candidateBlock = iter;
+ }
+ iter = iter->Next;
+ }
+
+ largest = largest - sizeof(MemoryRangeHeader);
+
+ // If this block isn't big enough for the allocation desired, allocate
+ // another block of memory and add it to the free list.
+ if (largest < ActualSize ||
+ largest <= FreeRangeHeader::getMinBlockSize()) {
+ DEBUG(dbgs() << "JIT: Allocating another slab of memory for function.");
+ candidateBlock = allocateNewCodeSlab((size_t)ActualSize);
+ }
+
+ // Select this candidate block for allocation
+ CurBlock = candidateBlock;
+
+ // Allocate the entire memory block.
+ FreeMemoryList = candidateBlock->AllocateBlock();
+ ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader);
+ return (uint8_t *)(CurBlock + 1);
+ }
+
+ /// allocateNewCodeSlab - Helper method to allocate a new slab of code
+ /// memory from the OS and add it to the free list. Returns the new
+ /// FreeRangeHeader at the base of the slab.
+ FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) {
+ // If the user needs at least MinSize free memory, then we account for
+ // two MemoryRangeHeaders: the one in the user's block, and the one at the
+ // end of the slab.
+ size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader);
+ size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin);
+ sys::MemoryBlock B = allocateNewSlab(SlabSize);
+ CodeSlabs.push_back(B);
+ char *MemBase = (char*)(B.base());
+
+ // Put a tiny allocated block at the end of the memory chunk, so when
+ // FreeBlock calls getBlockAfter it doesn't fall off the end.
+ MemoryRangeHeader *EndBlock =
+ (MemoryRangeHeader*)(MemBase + B.size()) - 1;
+ EndBlock->ThisAllocated = 1;
+ EndBlock->PrevAllocated = 0;
+ EndBlock->BlockSize = sizeof(MemoryRangeHeader);
+
+ // Start out with a vast new block of free memory.
+ FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase;
+ NewBlock->ThisAllocated = 0;
+ // Make sure getFreeBlockBefore doesn't look into unmapped memory.
+ NewBlock->PrevAllocated = 1;
+ NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock;
+ NewBlock->SetEndOfBlockSizeMarker();
+ NewBlock->AddToFreeList(FreeMemoryList);
+
+ assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize &&
+ "The block was too small!");
+ return NewBlock;
+ }
+
+ /// endFunctionBody - The function F is now allocated, and takes the memory
+ /// in the range [FunctionStart,FunctionEnd).
+ void endFunctionBody(const Function *F, uint8_t *FunctionStart,
+ uint8_t *FunctionEnd) override {
+ assert(FunctionEnd > FunctionStart);
+ assert(FunctionStart == (uint8_t *)(CurBlock+1) &&
+ "Mismatched function start/end!");
+
+ uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock;
+
+ // Release the memory at the end of this block that isn't needed.
+ FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
+ }
+
+ /// allocateSpace - Allocate a memory block of the given size. This method
+ /// cannot be called between calls to startFunctionBody and endFunctionBody.
+ uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) override {
+ CurBlock = FreeMemoryList;
+ FreeMemoryList = FreeMemoryList->AllocateBlock();
+
+ uint8_t *result = (uint8_t *)(CurBlock + 1);
+
+ if (Alignment == 0) Alignment = 1;
+ result = (uint8_t*)(((intptr_t)result+Alignment-1) &
+ ~(intptr_t)(Alignment-1));
+
+ uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock;
+ FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
+
+ return result;
+ }
+
+ /// allocateStub - Allocate memory for a function stub.
+ uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize,
+ unsigned Alignment) override {
+ return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment);
+ }
+
+ /// allocateGlobal - Allocate memory for a global.
+ uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) override {
+ return (uint8_t*)DataAllocator.Allocate(Size, Alignment);
+ }
+
+ /// allocateCodeSection - Allocate memory for a code section.
+ uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
+ unsigned SectionID,
+ StringRef SectionName) override {
+ // Grow the required block size to account for the block header
+ Size += sizeof(*CurBlock);
+
+ // Alignment handling.
+ if (!Alignment)
+ Alignment = 16;
+ Size += Alignment - 1;
+
+ FreeRangeHeader* candidateBlock = FreeMemoryList;
+ FreeRangeHeader* head = FreeMemoryList;
+ FreeRangeHeader* iter = head->Next;
+
+ uintptr_t largest = candidateBlock->BlockSize;
+
+ // Search for the largest free block.
+ while (iter != head) {
+ if (iter->BlockSize > largest) {
+ largest = iter->BlockSize;
+ candidateBlock = iter;
+ }
+ iter = iter->Next;
+ }
+
+ largest = largest - sizeof(MemoryRangeHeader);
+
+ // If this block isn't big enough for the allocation desired, allocate
+ // another block of memory and add it to the free list.
+ if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) {
+ DEBUG(dbgs() << "JIT: Allocating another slab of memory for function.");
+ candidateBlock = allocateNewCodeSlab((size_t)Size);
+ }
+
+ // Select this candidate block for allocation
+ CurBlock = candidateBlock;
+
+ // Allocate the entire memory block.
+ FreeMemoryList = candidateBlock->AllocateBlock();
+ // Release the memory at the end of this block that isn't needed.
+ FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size);
+ uintptr_t unalignedAddr = (uintptr_t)CurBlock + sizeof(*CurBlock);
+ return (uint8_t*)RoundUpToAlignment((uint64_t)unalignedAddr, Alignment);
+ }
+
+ /// allocateDataSection - Allocate memory for a data section.
+ uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
+ unsigned SectionID, StringRef SectionName,
+ bool IsReadOnly) override {
+ return (uint8_t*)DataAllocator.Allocate(Size, Alignment);
+ }
+
+ bool finalizeMemory(std::string *ErrMsg) override {
+ return false;
+ }
+
+ uint8_t *getGOTBase() const override {
+ return GOTBase;
+ }
+
+ void deallocateBlock(void *Block) {
+ // Find the block that is allocated for this function.
+ MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1;
+ assert(MemRange->ThisAllocated && "Block isn't allocated!");
+
+ // Fill the buffer with garbage!
+ if (PoisonMemory) {
+ memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
+ }
+
+ // Free the memory.
+ FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
+ }
+
+ /// deallocateFunctionBody - Deallocate all memory for the specified
+ /// function body.
+ void deallocateFunctionBody(void *Body) override {
+ if (Body) deallocateBlock(Body);
+ }
+
+ /// setMemoryWritable - When code generation is in progress,
+ /// the code pages may need permissions changed.
+ void setMemoryWritable() override {
+ for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
+ sys::Memory::setWritable(CodeSlabs[i]);
+ }
+ /// setMemoryExecutable - When code generation is done and we're ready to
+ /// start execution, the code pages may need permissions changed.
+ void setMemoryExecutable() override {
+ for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
+ sys::Memory::setExecutable(CodeSlabs[i]);
+ }
+
+ /// setPoisonMemory - Controls whether we write garbage over freed memory.
+ ///
+ void setPoisonMemory(bool poison) override {
+ PoisonMemory = poison;
+ }
+ };
+}
+
+void *JITAllocator::Allocate(size_t Size, size_t /*Alignment*/) {
+ sys::MemoryBlock B = JMM.allocateNewSlab(Size);
+ return B.base();
+}
+
+void JITAllocator::Deallocate(void *Slab, size_t Size) {
+ sys::MemoryBlock B(Slab, Size);
+ sys::Memory::ReleaseRWX(B);
+}
+
+DefaultJITMemoryManager::DefaultJITMemoryManager()
+ :
+#ifdef NDEBUG
+ PoisonMemory(false),
+#else
+ PoisonMemory(true),
+#endif
+ LastSlab(nullptr, 0), StubAllocator(*this), DataAllocator(*this) {
+
+ // Allocate space for code.
+ sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize);
+ CodeSlabs.push_back(MemBlock);
+ uint8_t *MemBase = (uint8_t*)MemBlock.base();
+
+ // We set up the memory chunk with 4 mem regions, like this:
+ // [ START
+ // [ Free #0 ] -> Large space to allocate functions from.
+ // [ Allocated #1 ] -> Tiny space to separate regions.
+ // [ Free #2 ] -> Tiny space so there is always at least 1 free block.
+ // [ Allocated #3 ] -> Tiny space to prevent looking past end of block.
+ // END ]
+ //
+ // The last three blocks are never deallocated or touched.
+
+ // Add MemoryRangeHeader to the end of the memory region, indicating that
+ // the space after the block of memory is allocated. This is block #3.
+ MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1;
+ Mem3->ThisAllocated = 1;
+ Mem3->PrevAllocated = 0;
+ Mem3->BlockSize = sizeof(MemoryRangeHeader);
+
+ /// Add a tiny free region so that the free list always has one entry.
+ FreeRangeHeader *Mem2 =
+ (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize());
+ Mem2->ThisAllocated = 0;
+ Mem2->PrevAllocated = 1;
+ Mem2->BlockSize = FreeRangeHeader::getMinBlockSize();
+ Mem2->SetEndOfBlockSizeMarker();
+ Mem2->Prev = Mem2; // Mem2 *is* the free list for now.
+ Mem2->Next = Mem2;
+
+ /// Add a tiny allocated region so that Mem2 is never coalesced away.
+ MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1;
+ Mem1->ThisAllocated = 1;
+ Mem1->PrevAllocated = 0;
+ Mem1->BlockSize = sizeof(MemoryRangeHeader);
+
+ // Add a FreeRangeHeader to the start of the function body region, indicating
+ // that the space is free. Mark the previous block allocated so we never look
+ // at it.
+ FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase;
+ Mem0->ThisAllocated = 0;
+ Mem0->PrevAllocated = 1;
+ Mem0->BlockSize = (char*)Mem1-(char*)Mem0;
+ Mem0->SetEndOfBlockSizeMarker();
+ Mem0->AddToFreeList(Mem2);
+
+ // Start out with the freelist pointing to Mem0.
+ FreeMemoryList = Mem0;
+
+ GOTBase = nullptr;
+}
+
+void DefaultJITMemoryManager::AllocateGOT() {
+ assert(!GOTBase && "Cannot allocate the got multiple times");
+ GOTBase = new uint8_t[sizeof(void*) * 8192];
+ HasGOT = true;
+}
+
+DefaultJITMemoryManager::~DefaultJITMemoryManager() {
+ for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
+ sys::Memory::ReleaseRWX(CodeSlabs[i]);
+
+ delete[] GOTBase;
+}
+
+sys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) {
+ // Allocate a new block close to the last one.
+ std::string ErrMsg;
+ sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : nullptr;
+ sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg);
+ if (!B.base()) {
+ report_fatal_error("Allocation failed when allocating new memory in the"
+ " JIT\n" + Twine(ErrMsg));
+ }
+ LastSlab = B;
+ ++NumSlabs;
+ // Initialize the slab to garbage when debugging.
+ if (PoisonMemory) {
+ memset(B.base(), 0xCD, B.size());
+ }
+ return B;
+}
+
+/// CheckInvariants - For testing only. Return "" if all internal invariants
+/// are preserved, and a helpful error message otherwise. For free and
+/// allocated blocks, make sure that adding BlockSize gives a valid block.
+/// For free blocks, make sure they're in the free list and that their end of
+/// block size marker is correct. This function should return an error before
+/// accessing bad memory. This function is defined here instead of in
+/// JITMemoryManagerTest.cpp so that we don't have to expose all of the
+/// implementation details of DefaultJITMemoryManager.
+bool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) {
+ raw_string_ostream Err(ErrorStr);
+
+ // Construct a the set of FreeRangeHeader pointers so we can query it
+ // efficiently.
+ llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet;
+ FreeRangeHeader* FreeHead = FreeMemoryList;
+ FreeRangeHeader* FreeRange = FreeHead;
+
+ do {
+ // Check that the free range pointer is in the blocks we've allocated.
+ bool Found = false;
+ for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(),
+ E = CodeSlabs.end(); I != E && !Found; ++I) {
+ char *Start = (char*)I->base();
+ char *End = Start + I->size();
+ Found = (Start <= (char*)FreeRange && (char*)FreeRange < End);
+ }
+ if (!Found) {
+ Err << "Corrupt free list; points to " << FreeRange;
+ return false;
+ }
+
+ if (FreeRange->Next->Prev != FreeRange) {
+ Err << "Next and Prev pointers do not match.";
+ return false;
+ }
+
+ // Otherwise, add it to the set.
+ FreeHdrSet.insert(FreeRange);
+ FreeRange = FreeRange->Next;
+ } while (FreeRange != FreeHead);
+
+ // Go over each block, and look at each MemoryRangeHeader.
+ for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(),
+ E = CodeSlabs.end(); I != E; ++I) {
+ char *Start = (char*)I->base();
+ char *End = Start + I->size();
+
+ // Check each memory range.
+ for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = nullptr;
+ Start <= (char*)Hdr && (char*)Hdr < End;
+ Hdr = &Hdr->getBlockAfter()) {
+ if (Hdr->ThisAllocated == 0) {
+ // Check that this range is in the free list.
+ if (!FreeHdrSet.count(Hdr)) {
+ Err << "Found free header at " << Hdr << " that is not in free list.";
+ return false;
+ }
+
+ // Now make sure the size marker at the end of the block is correct.
+ uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1;
+ if (!(Start <= (char*)Marker && (char*)Marker < End)) {
+ Err << "Block size in header points out of current MemoryBlock.";
+ return false;
+ }
+ if (Hdr->BlockSize != *Marker) {
+ Err << "End of block size marker (" << *Marker << ") "
+ << "and BlockSize (" << Hdr->BlockSize << ") don't match.";
+ return false;
+ }
+ }
+
+ if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) {
+ Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != "
+ << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")";
+ return false;
+ } else if (!LastHdr && !Hdr->PrevAllocated) {
+ Err << "The first header should have PrevAllocated true.";
+ return false;
+ }
+
+ // Remember the last header.
+ LastHdr = Hdr;
+ }
+ }
+
+ // All invariants are preserved.
+ return true;
+}
+
+//===----------------------------------------------------------------------===//
+// getPointerToNamedFunction() implementation.
+//===----------------------------------------------------------------------===//
+
+// AtExitHandlers - List of functions to call when the program exits,
+// registered with the atexit() library function.
+static std::vector<void (*)()> AtExitHandlers;
+
+/// runAtExitHandlers - Run any functions registered by the program's
+/// calls to atexit(3), which we intercept and store in
+/// AtExitHandlers.
+///
+static void runAtExitHandlers() {
+ while (!AtExitHandlers.empty()) {
+ void (*Fn)() = AtExitHandlers.back();
+ AtExitHandlers.pop_back();
+ Fn();
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// Function stubs that are invoked instead of certain library calls
+//
+// Force the following functions to be linked in to anything that uses the
+// JIT. This is a hack designed to work around the all-too-clever Glibc
+// strategy of making these functions work differently when inlined vs. when
+// not inlined, and hiding their real definitions in a separate archive file
+// that the dynamic linker can't see. For more info, search for
+// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274.
+#if defined(__linux__) && defined(__GLIBC__)
+/* stat functions are redirecting to __xstat with a version number. On x86-64
+ * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat'
+ * available as an exported symbol, so we have to add it explicitly.
+ */
+namespace {
+class StatSymbols {
+public:
+ StatSymbols() {
+ sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat);
+ sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat);
+ sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat);
+ sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64);
+ sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64);
+ sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64);
+ sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64);
+ sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64);
+ sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64);
+ sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit);
+ sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod);
+ }
+};
+}
+static StatSymbols initStatSymbols;
+#endif // __linux__
+
+// jit_exit - Used to intercept the "exit" library call.
+static void jit_exit(int Status) {
+ runAtExitHandlers(); // Run atexit handlers...
+ exit(Status);
+}
+
+// jit_atexit - Used to intercept the "atexit" library call.
+static int jit_atexit(void (*Fn)()) {
+ AtExitHandlers.push_back(Fn); // Take note of atexit handler...
+ return 0; // Always successful
+}
+
+static int jit_noop() {
+ return 0;
+}
+
+//===----------------------------------------------------------------------===//
+//
+/// getPointerToNamedFunction - This method returns the address of the specified
+/// function by using the dynamic loader interface. As such it is only useful
+/// for resolving library symbols, not code generated symbols.
+///
+void *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name,
+ bool AbortOnFailure) {
+ // Check to see if this is one of the functions we want to intercept. Note,
+ // we cast to intptr_t here to silence a -pedantic warning that complains
+ // about casting a function pointer to a normal pointer.
+ if (Name == "exit") return (void*)(intptr_t)&jit_exit;
+ if (Name == "atexit") return (void*)(intptr_t)&jit_atexit;
+
+ // We should not invoke parent's ctors/dtors from generated main()!
+ // On Mingw and Cygwin, the symbol __main is resolved to
+ // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors
+ // (and register wrong callee's dtors with atexit(3)).
+ // We expect ExecutionEngine::runStaticConstructorsDestructors()
+ // is called before ExecutionEngine::runFunctionAsMain() is called.
+ if (Name == "__main") return (void*)(intptr_t)&jit_noop;
+
+ const char *NameStr = Name.c_str();
+ // If this is an asm specifier, skip the sentinal.
+ if (NameStr[0] == 1) ++NameStr;
+
+ // If it's an external function, look it up in the process image...
+ void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr);
+ if (Ptr) return Ptr;
+
+ // If it wasn't found and if it starts with an underscore ('_') character,
+ // try again without the underscore.
+ if (NameStr[0] == '_') {
+ Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1);
+ if (Ptr) return Ptr;
+ }
+
+ // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These
+ // are references to hidden visibility symbols that dlsym cannot resolve.
+ // If we have one of these, strip off $LDBLStub and try again.
+#if defined(__APPLE__) && defined(__ppc__)
+ if (Name.size() > 9 && Name[Name.size()-9] == '$' &&
+ memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) {
+ // First try turning $LDBLStub into $LDBL128. If that fails, strip it off.
+ // This mirrors logic in libSystemStubs.a.
+ std::string Prefix = std::string(Name.begin(), Name.end()-9);
+ if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false))
+ return Ptr;
+ if (void *Ptr = getPointerToNamedFunction(Prefix, false))
+ return Ptr;
+ }
+#endif
+
+ if (AbortOnFailure) {
+ report_fatal_error("Program used external function '"+Name+
+ "' which could not be resolved!");
+ }
+ return nullptr;
+}
+
+
+
+JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() {
+ return new DefaultJITMemoryManager();
+}
+
+const size_t DefaultJITMemoryManager::DefaultCodeSlabSize;
+const size_t DefaultJITMemoryManager::DefaultSlabSize;
+const size_t DefaultJITMemoryManager::DefaultSizeThreshold;
--- /dev/null
+;===- ./lib/ExecutionEngine/JIT/LLVMBuild.txt ------------------*- Conf -*--===;
+;
+; The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+; http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = JIT
+parent = ExecutionEngine
+required_libraries = CodeGen Core ExecutionEngine Support
--- /dev/null
+##===- lib/ExecutionEngine/JIT/Makefile --------------------*- Makefile -*-===##
+#
+# The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../../..
+LIBRARYNAME = LLVMJIT
+
+# Get the $(ARCH) setting
+include $(LEVEL)/Makefile.config
+
+# Enable the X86 JIT if compiling on X86
+ifeq ($(ARCH), x86)
+ ENABLE_X86_JIT = 1
+endif
+
+# This flag can also be used on the command line to force inclusion
+# of the X86 JIT on non-X86 hosts
+ifdef ENABLE_X86_JIT
+ CPPFLAGS += -DENABLE_X86_JIT
+endif
+
+# Enable the Sparc JIT if compiling on Sparc
+ifeq ($(ARCH), Sparc)
+ ENABLE_SPARC_JIT = 1
+endif
+
+# This flag can also be used on the command line to force inclusion
+# of the Sparc JIT on non-Sparc hosts
+ifdef ENABLE_SPARC_JIT
+ CPPFLAGS += -DENABLE_SPARC_JIT
+endif
+
+include $(LEVEL)/Makefile.common
;===------------------------------------------------------------------------===;
[common]
-subdirectories = Interpreter MCJIT RuntimeDyld IntelJITEvents OProfileJIT
+subdirectories = Interpreter JIT MCJIT RuntimeDyld IntelJITEvents OProfileJIT
[component_0]
type = Library
add_llvm_library(LLVMMCJIT
- JITMemoryManager.cpp
MCJIT.cpp
SectionMemoryManager.cpp
)
+++ /dev/null
-//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the DefaultJITMemoryManager class.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ExecutionEngine/JITMemoryManager.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Config/config.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/Support/Allocator.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/DynamicLibrary.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/Memory.h"
-#include "llvm/Support/raw_ostream.h"
-#include <cassert>
-#include <climits>
-#include <cstring>
-#include <vector>
-
-#if defined(__linux__)
-#if defined(HAVE_SYS_STAT_H)
-#include <sys/stat.h>
-#endif
-#include <fcntl.h>
-#include <unistd.h>
-#endif
-
-using namespace llvm;
-
-#define DEBUG_TYPE "jit"
-
-STATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT");
-
-JITMemoryManager::~JITMemoryManager() {}
-
-//===----------------------------------------------------------------------===//
-// Memory Block Implementation.
-//===----------------------------------------------------------------------===//
-
-namespace {
- /// MemoryRangeHeader - For a range of memory, this is the header that we put
- /// on the block of memory. It is carefully crafted to be one word of memory.
- /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader
- /// which starts with this.
- struct FreeRangeHeader;
- struct MemoryRangeHeader {
- /// ThisAllocated - This is true if this block is currently allocated. If
- /// not, this can be converted to a FreeRangeHeader.
- unsigned ThisAllocated : 1;
-
- /// PrevAllocated - Keep track of whether the block immediately before us is
- /// allocated. If not, the word immediately before this header is the size
- /// of the previous block.
- unsigned PrevAllocated : 1;
-
- /// BlockSize - This is the size in bytes of this memory block,
- /// including this header.
- uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2);
-
-
- /// getBlockAfter - Return the memory block immediately after this one.
- ///
- MemoryRangeHeader &getBlockAfter() const {
- return *reinterpret_cast<MemoryRangeHeader *>(
- reinterpret_cast<char*>(
- const_cast<MemoryRangeHeader *>(this))+BlockSize);
- }
-
- /// getFreeBlockBefore - If the block before this one is free, return it,
- /// otherwise return null.
- FreeRangeHeader *getFreeBlockBefore() const {
- if (PrevAllocated) return nullptr;
- intptr_t PrevSize = reinterpret_cast<intptr_t *>(
- const_cast<MemoryRangeHeader *>(this))[-1];
- return reinterpret_cast<FreeRangeHeader *>(
- reinterpret_cast<char*>(
- const_cast<MemoryRangeHeader *>(this))-PrevSize);
- }
-
- /// FreeBlock - Turn an allocated block into a free block, adjusting
- /// bits in the object headers, and adding an end of region memory block.
- FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList);
-
- /// TrimAllocationToSize - If this allocated block is significantly larger
- /// than NewSize, split it into two pieces (where the former is NewSize
- /// bytes, including the header), and add the new block to the free list.
- FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList,
- uint64_t NewSize);
- };
-
- /// FreeRangeHeader - For a memory block that isn't already allocated, this
- /// keeps track of the current block and has a pointer to the next free block.
- /// Free blocks are kept on a circularly linked list.
- struct FreeRangeHeader : public MemoryRangeHeader {
- FreeRangeHeader *Prev;
- FreeRangeHeader *Next;
-
- /// getMinBlockSize - Get the minimum size for a memory block. Blocks
- /// smaller than this size cannot be created.
- static unsigned getMinBlockSize() {
- return sizeof(FreeRangeHeader)+sizeof(intptr_t);
- }
-
- /// SetEndOfBlockSizeMarker - The word at the end of every free block is
- /// known to be the size of the free block. Set it for this block.
- void SetEndOfBlockSizeMarker() {
- void *EndOfBlock = (char*)this + BlockSize;
- ((intptr_t *)EndOfBlock)[-1] = BlockSize;
- }
-
- FreeRangeHeader *RemoveFromFreeList() {
- assert(Next->Prev == this && Prev->Next == this && "Freelist broken!");
- Next->Prev = Prev;
- return Prev->Next = Next;
- }
-
- void AddToFreeList(FreeRangeHeader *FreeList) {
- Next = FreeList;
- Prev = FreeList->Prev;
- Prev->Next = this;
- Next->Prev = this;
- }
-
- /// GrowBlock - The block after this block just got deallocated. Merge it
- /// into the current block.
- void GrowBlock(uintptr_t NewSize);
-
- /// AllocateBlock - Mark this entire block allocated, updating freelists
- /// etc. This returns a pointer to the circular free-list.
- FreeRangeHeader *AllocateBlock();
- };
-}
-
-
-/// AllocateBlock - Mark this entire block allocated, updating freelists
-/// etc. This returns a pointer to the circular free-list.
-FreeRangeHeader *FreeRangeHeader::AllocateBlock() {
- assert(!ThisAllocated && !getBlockAfter().PrevAllocated &&
- "Cannot allocate an allocated block!");
- // Mark this block allocated.
- ThisAllocated = 1;
- getBlockAfter().PrevAllocated = 1;
-
- // Remove it from the free list.
- return RemoveFromFreeList();
-}
-
-/// FreeBlock - Turn an allocated block into a free block, adjusting
-/// bits in the object headers, and adding an end of region memory block.
-/// If possible, coalesce this block with neighboring blocks. Return the
-/// FreeRangeHeader to allocate from.
-FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) {
- MemoryRangeHeader *FollowingBlock = &getBlockAfter();
- assert(ThisAllocated && "This block is already free!");
- assert(FollowingBlock->PrevAllocated && "Flags out of sync!");
-
- FreeRangeHeader *FreeListToReturn = FreeList;
-
- // If the block after this one is free, merge it into this block.
- if (!FollowingBlock->ThisAllocated) {
- FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock;
- // "FreeList" always needs to be a valid free block. If we're about to
- // coalesce with it, update our notion of what the free list is.
- if (&FollowingFreeBlock == FreeList) {
- FreeList = FollowingFreeBlock.Next;
- FreeListToReturn = nullptr;
- assert(&FollowingFreeBlock != FreeList && "No tombstone block?");
- }
- FollowingFreeBlock.RemoveFromFreeList();
-
- // Include the following block into this one.
- BlockSize += FollowingFreeBlock.BlockSize;
- FollowingBlock = &FollowingFreeBlock.getBlockAfter();
-
- // Tell the block after the block we are coalescing that this block is
- // allocated.
- FollowingBlock->PrevAllocated = 1;
- }
-
- assert(FollowingBlock->ThisAllocated && "Missed coalescing?");
-
- if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) {
- PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize);
- return FreeListToReturn ? FreeListToReturn : PrevFreeBlock;
- }
-
- // Otherwise, mark this block free.
- FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this;
- FollowingBlock->PrevAllocated = 0;
- FreeBlock.ThisAllocated = 0;
-
- // Link this into the linked list of free blocks.
- FreeBlock.AddToFreeList(FreeList);
-
- // Add a marker at the end of the block, indicating the size of this free
- // block.
- FreeBlock.SetEndOfBlockSizeMarker();
- return FreeListToReturn ? FreeListToReturn : &FreeBlock;
-}
-
-/// GrowBlock - The block after this block just got deallocated. Merge it
-/// into the current block.
-void FreeRangeHeader::GrowBlock(uintptr_t NewSize) {
- assert(NewSize > BlockSize && "Not growing block?");
- BlockSize = NewSize;
- SetEndOfBlockSizeMarker();
- getBlockAfter().PrevAllocated = 0;
-}
-
-/// TrimAllocationToSize - If this allocated block is significantly larger
-/// than NewSize, split it into two pieces (where the former is NewSize
-/// bytes, including the header), and add the new block to the free list.
-FreeRangeHeader *MemoryRangeHeader::
-TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) {
- assert(ThisAllocated && getBlockAfter().PrevAllocated &&
- "Cannot deallocate part of an allocated block!");
-
- // Don't allow blocks to be trimmed below minimum required size
- NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize);
-
- // Round up size for alignment of header.
- unsigned HeaderAlign = __alignof(FreeRangeHeader);
- NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1);
-
- // Size is now the size of the block we will remove from the start of the
- // current block.
- assert(NewSize <= BlockSize &&
- "Allocating more space from this block than exists!");
-
- // If splitting this block will cause the remainder to be too small, do not
- // split the block.
- if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize())
- return FreeList;
-
- // Otherwise, we splice the required number of bytes out of this block, form
- // a new block immediately after it, then mark this block allocated.
- MemoryRangeHeader &FormerNextBlock = getBlockAfter();
-
- // Change the size of this block.
- BlockSize = NewSize;
-
- // Get the new block we just sliced out and turn it into a free block.
- FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter();
- NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock;
- NewNextBlock.ThisAllocated = 0;
- NewNextBlock.PrevAllocated = 1;
- NewNextBlock.SetEndOfBlockSizeMarker();
- FormerNextBlock.PrevAllocated = 0;
- NewNextBlock.AddToFreeList(FreeList);
- return &NewNextBlock;
-}
-
-//===----------------------------------------------------------------------===//
-// Memory Block Implementation.
-//===----------------------------------------------------------------------===//
-
-namespace {
-
- class DefaultJITMemoryManager;
-
- class JITAllocator {
- DefaultJITMemoryManager &JMM;
- public:
- JITAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { }
- void *Allocate(size_t Size, size_t /*Alignment*/);
- void Deallocate(void *Slab, size_t Size);
- };
-
- /// DefaultJITMemoryManager - Manage memory for the JIT code generation.
- /// This splits a large block of MAP_NORESERVE'd memory into two
- /// sections, one for function stubs, one for the functions themselves. We
- /// have to do this because we may need to emit a function stub while in the
- /// middle of emitting a function, and we don't know how large the function we
- /// are emitting is.
- class DefaultJITMemoryManager : public JITMemoryManager {
- public:
- /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at
- /// least this much unless more is requested. Currently, in 512k slabs.
- static const size_t DefaultCodeSlabSize = 512 * 1024;
-
- /// DefaultSlabSize - Allocate globals and stubs into slabs of 64K (probably
- /// 16 pages) unless we get an allocation above SizeThreshold.
- static const size_t DefaultSlabSize = 64 * 1024;
-
- /// DefaultSizeThreshold - For any allocation larger than 16K (probably
- /// 4 pages), we should allocate a separate slab to avoid wasted space at
- /// the end of a normal slab.
- static const size_t DefaultSizeThreshold = 16 * 1024;
-
- private:
- // Whether to poison freed memory.
- bool PoisonMemory;
-
- /// LastSlab - This points to the last slab allocated and is used as the
- /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all
- /// stubs, data, and code contiguously in memory. In general, however, this
- /// is not possible because the NearBlock parameter is ignored on Windows
- /// platforms and even on Unix it works on a best-effort pasis.
- sys::MemoryBlock LastSlab;
-
- // Memory slabs allocated by the JIT. We refer to them as slabs so we don't
- // confuse them with the blocks of memory described above.
- std::vector<sys::MemoryBlock> CodeSlabs;
- BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize,
- DefaultSizeThreshold> StubAllocator;
- BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize,
- DefaultSizeThreshold> DataAllocator;
-
- // Circular list of free blocks.
- FreeRangeHeader *FreeMemoryList;
-
- // When emitting code into a memory block, this is the block.
- MemoryRangeHeader *CurBlock;
-
- uint8_t *GOTBase; // Target Specific reserved memory
- public:
- DefaultJITMemoryManager();
- ~DefaultJITMemoryManager();
-
- /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the
- /// last slab it allocated, so that subsequent allocations follow it.
- sys::MemoryBlock allocateNewSlab(size_t size);
-
- /// getPointerToNamedFunction - This method returns the address of the
- /// specified function by using the dlsym function call.
- void *getPointerToNamedFunction(const std::string &Name,
- bool AbortOnFailure = true) override;
-
- void AllocateGOT() override;
-
- // Testing methods.
- bool CheckInvariants(std::string &ErrorStr) override;
- size_t GetDefaultCodeSlabSize() override { return DefaultCodeSlabSize; }
- size_t GetDefaultDataSlabSize() override { return DefaultSlabSize; }
- size_t GetDefaultStubSlabSize() override { return DefaultSlabSize; }
- unsigned GetNumCodeSlabs() override { return CodeSlabs.size(); }
- unsigned GetNumDataSlabs() override { return DataAllocator.GetNumSlabs(); }
- unsigned GetNumStubSlabs() override { return StubAllocator.GetNumSlabs(); }
-
- /// startFunctionBody - When a function starts, allocate a block of free
- /// executable memory, returning a pointer to it and its actual size.
- uint8_t *startFunctionBody(const Function *F,
- uintptr_t &ActualSize) override {
-
- FreeRangeHeader* candidateBlock = FreeMemoryList;
- FreeRangeHeader* head = FreeMemoryList;
- FreeRangeHeader* iter = head->Next;
-
- uintptr_t largest = candidateBlock->BlockSize;
-
- // Search for the largest free block
- while (iter != head) {
- if (iter->BlockSize > largest) {
- largest = iter->BlockSize;
- candidateBlock = iter;
- }
- iter = iter->Next;
- }
-
- largest = largest - sizeof(MemoryRangeHeader);
-
- // If this block isn't big enough for the allocation desired, allocate
- // another block of memory and add it to the free list.
- if (largest < ActualSize ||
- largest <= FreeRangeHeader::getMinBlockSize()) {
- DEBUG(dbgs() << "JIT: Allocating another slab of memory for function.");
- candidateBlock = allocateNewCodeSlab((size_t)ActualSize);
- }
-
- // Select this candidate block for allocation
- CurBlock = candidateBlock;
-
- // Allocate the entire memory block.
- FreeMemoryList = candidateBlock->AllocateBlock();
- ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader);
- return (uint8_t *)(CurBlock + 1);
- }
-
- /// allocateNewCodeSlab - Helper method to allocate a new slab of code
- /// memory from the OS and add it to the free list. Returns the new
- /// FreeRangeHeader at the base of the slab.
- FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) {
- // If the user needs at least MinSize free memory, then we account for
- // two MemoryRangeHeaders: the one in the user's block, and the one at the
- // end of the slab.
- size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader);
- size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin);
- sys::MemoryBlock B = allocateNewSlab(SlabSize);
- CodeSlabs.push_back(B);
- char *MemBase = (char*)(B.base());
-
- // Put a tiny allocated block at the end of the memory chunk, so when
- // FreeBlock calls getBlockAfter it doesn't fall off the end.
- MemoryRangeHeader *EndBlock =
- (MemoryRangeHeader*)(MemBase + B.size()) - 1;
- EndBlock->ThisAllocated = 1;
- EndBlock->PrevAllocated = 0;
- EndBlock->BlockSize = sizeof(MemoryRangeHeader);
-
- // Start out with a vast new block of free memory.
- FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase;
- NewBlock->ThisAllocated = 0;
- // Make sure getFreeBlockBefore doesn't look into unmapped memory.
- NewBlock->PrevAllocated = 1;
- NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock;
- NewBlock->SetEndOfBlockSizeMarker();
- NewBlock->AddToFreeList(FreeMemoryList);
-
- assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize &&
- "The block was too small!");
- return NewBlock;
- }
-
- /// endFunctionBody - The function F is now allocated, and takes the memory
- /// in the range [FunctionStart,FunctionEnd).
- void endFunctionBody(const Function *F, uint8_t *FunctionStart,
- uint8_t *FunctionEnd) override {
- assert(FunctionEnd > FunctionStart);
- assert(FunctionStart == (uint8_t *)(CurBlock+1) &&
- "Mismatched function start/end!");
-
- uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock;
-
- // Release the memory at the end of this block that isn't needed.
- FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
- }
-
- /// allocateSpace - Allocate a memory block of the given size. This method
- /// cannot be called between calls to startFunctionBody and endFunctionBody.
- uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) override {
- CurBlock = FreeMemoryList;
- FreeMemoryList = FreeMemoryList->AllocateBlock();
-
- uint8_t *result = (uint8_t *)(CurBlock + 1);
-
- if (Alignment == 0) Alignment = 1;
- result = (uint8_t*)(((intptr_t)result+Alignment-1) &
- ~(intptr_t)(Alignment-1));
-
- uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock;
- FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
-
- return result;
- }
-
- /// allocateStub - Allocate memory for a function stub.
- uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize,
- unsigned Alignment) override {
- return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment);
- }
-
- /// allocateGlobal - Allocate memory for a global.
- uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) override {
- return (uint8_t*)DataAllocator.Allocate(Size, Alignment);
- }
-
- /// allocateCodeSection - Allocate memory for a code section.
- uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
- unsigned SectionID,
- StringRef SectionName) override {
- // Grow the required block size to account for the block header
- Size += sizeof(*CurBlock);
-
- // Alignment handling.
- if (!Alignment)
- Alignment = 16;
- Size += Alignment - 1;
-
- FreeRangeHeader* candidateBlock = FreeMemoryList;
- FreeRangeHeader* head = FreeMemoryList;
- FreeRangeHeader* iter = head->Next;
-
- uintptr_t largest = candidateBlock->BlockSize;
-
- // Search for the largest free block.
- while (iter != head) {
- if (iter->BlockSize > largest) {
- largest = iter->BlockSize;
- candidateBlock = iter;
- }
- iter = iter->Next;
- }
-
- largest = largest - sizeof(MemoryRangeHeader);
-
- // If this block isn't big enough for the allocation desired, allocate
- // another block of memory and add it to the free list.
- if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) {
- DEBUG(dbgs() << "JIT: Allocating another slab of memory for function.");
- candidateBlock = allocateNewCodeSlab((size_t)Size);
- }
-
- // Select this candidate block for allocation
- CurBlock = candidateBlock;
-
- // Allocate the entire memory block.
- FreeMemoryList = candidateBlock->AllocateBlock();
- // Release the memory at the end of this block that isn't needed.
- FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size);
- uintptr_t unalignedAddr = (uintptr_t)CurBlock + sizeof(*CurBlock);
- return (uint8_t*)RoundUpToAlignment((uint64_t)unalignedAddr, Alignment);
- }
-
- /// allocateDataSection - Allocate memory for a data section.
- uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
- unsigned SectionID, StringRef SectionName,
- bool IsReadOnly) override {
- return (uint8_t*)DataAllocator.Allocate(Size, Alignment);
- }
-
- bool finalizeMemory(std::string *ErrMsg) override {
- return false;
- }
-
- uint8_t *getGOTBase() const override {
- return GOTBase;
- }
-
- void deallocateBlock(void *Block) {
- // Find the block that is allocated for this function.
- MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1;
- assert(MemRange->ThisAllocated && "Block isn't allocated!");
-
- // Fill the buffer with garbage!
- if (PoisonMemory) {
- memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
- }
-
- // Free the memory.
- FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
- }
-
- /// deallocateFunctionBody - Deallocate all memory for the specified
- /// function body.
- void deallocateFunctionBody(void *Body) override {
- if (Body) deallocateBlock(Body);
- }
-
- /// setMemoryWritable - When code generation is in progress,
- /// the code pages may need permissions changed.
- void setMemoryWritable() override {
- for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
- sys::Memory::setWritable(CodeSlabs[i]);
- }
- /// setMemoryExecutable - When code generation is done and we're ready to
- /// start execution, the code pages may need permissions changed.
- void setMemoryExecutable() override {
- for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
- sys::Memory::setExecutable(CodeSlabs[i]);
- }
-
- /// setPoisonMemory - Controls whether we write garbage over freed memory.
- ///
- void setPoisonMemory(bool poison) override {
- PoisonMemory = poison;
- }
- };
-}
-
-void *JITAllocator::Allocate(size_t Size, size_t /*Alignment*/) {
- sys::MemoryBlock B = JMM.allocateNewSlab(Size);
- return B.base();
-}
-
-void JITAllocator::Deallocate(void *Slab, size_t Size) {
- sys::MemoryBlock B(Slab, Size);
- sys::Memory::ReleaseRWX(B);
-}
-
-DefaultJITMemoryManager::DefaultJITMemoryManager()
- :
-#ifdef NDEBUG
- PoisonMemory(false),
-#else
- PoisonMemory(true),
-#endif
- LastSlab(nullptr, 0), StubAllocator(*this), DataAllocator(*this) {
-
- // Allocate space for code.
- sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize);
- CodeSlabs.push_back(MemBlock);
- uint8_t *MemBase = (uint8_t*)MemBlock.base();
-
- // We set up the memory chunk with 4 mem regions, like this:
- // [ START
- // [ Free #0 ] -> Large space to allocate functions from.
- // [ Allocated #1 ] -> Tiny space to separate regions.
- // [ Free #2 ] -> Tiny space so there is always at least 1 free block.
- // [ Allocated #3 ] -> Tiny space to prevent looking past end of block.
- // END ]
- //
- // The last three blocks are never deallocated or touched.
-
- // Add MemoryRangeHeader to the end of the memory region, indicating that
- // the space after the block of memory is allocated. This is block #3.
- MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1;
- Mem3->ThisAllocated = 1;
- Mem3->PrevAllocated = 0;
- Mem3->BlockSize = sizeof(MemoryRangeHeader);
-
- /// Add a tiny free region so that the free list always has one entry.
- FreeRangeHeader *Mem2 =
- (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize());
- Mem2->ThisAllocated = 0;
- Mem2->PrevAllocated = 1;
- Mem2->BlockSize = FreeRangeHeader::getMinBlockSize();
- Mem2->SetEndOfBlockSizeMarker();
- Mem2->Prev = Mem2; // Mem2 *is* the free list for now.
- Mem2->Next = Mem2;
-
- /// Add a tiny allocated region so that Mem2 is never coalesced away.
- MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1;
- Mem1->ThisAllocated = 1;
- Mem1->PrevAllocated = 0;
- Mem1->BlockSize = sizeof(MemoryRangeHeader);
-
- // Add a FreeRangeHeader to the start of the function body region, indicating
- // that the space is free. Mark the previous block allocated so we never look
- // at it.
- FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase;
- Mem0->ThisAllocated = 0;
- Mem0->PrevAllocated = 1;
- Mem0->BlockSize = (char*)Mem1-(char*)Mem0;
- Mem0->SetEndOfBlockSizeMarker();
- Mem0->AddToFreeList(Mem2);
-
- // Start out with the freelist pointing to Mem0.
- FreeMemoryList = Mem0;
-
- GOTBase = nullptr;
-}
-
-void DefaultJITMemoryManager::AllocateGOT() {
- assert(!GOTBase && "Cannot allocate the got multiple times");
- GOTBase = new uint8_t[sizeof(void*) * 8192];
- HasGOT = true;
-}
-
-DefaultJITMemoryManager::~DefaultJITMemoryManager() {
- for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
- sys::Memory::ReleaseRWX(CodeSlabs[i]);
-
- delete[] GOTBase;
-}
-
-sys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) {
- // Allocate a new block close to the last one.
- std::string ErrMsg;
- sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : nullptr;
- sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg);
- if (!B.base()) {
- report_fatal_error("Allocation failed when allocating new memory in the"
- " JIT\n" + Twine(ErrMsg));
- }
- LastSlab = B;
- ++NumSlabs;
- // Initialize the slab to garbage when debugging.
- if (PoisonMemory) {
- memset(B.base(), 0xCD, B.size());
- }
- return B;
-}
-
-/// CheckInvariants - For testing only. Return "" if all internal invariants
-/// are preserved, and a helpful error message otherwise. For free and
-/// allocated blocks, make sure that adding BlockSize gives a valid block.
-/// For free blocks, make sure they're in the free list and that their end of
-/// block size marker is correct. This function should return an error before
-/// accessing bad memory. This function is defined here instead of in
-/// JITMemoryManagerTest.cpp so that we don't have to expose all of the
-/// implementation details of DefaultJITMemoryManager.
-bool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) {
- raw_string_ostream Err(ErrorStr);
-
- // Construct a the set of FreeRangeHeader pointers so we can query it
- // efficiently.
- llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet;
- FreeRangeHeader* FreeHead = FreeMemoryList;
- FreeRangeHeader* FreeRange = FreeHead;
-
- do {
- // Check that the free range pointer is in the blocks we've allocated.
- bool Found = false;
- for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(),
- E = CodeSlabs.end(); I != E && !Found; ++I) {
- char *Start = (char*)I->base();
- char *End = Start + I->size();
- Found = (Start <= (char*)FreeRange && (char*)FreeRange < End);
- }
- if (!Found) {
- Err << "Corrupt free list; points to " << FreeRange;
- return false;
- }
-
- if (FreeRange->Next->Prev != FreeRange) {
- Err << "Next and Prev pointers do not match.";
- return false;
- }
-
- // Otherwise, add it to the set.
- FreeHdrSet.insert(FreeRange);
- FreeRange = FreeRange->Next;
- } while (FreeRange != FreeHead);
-
- // Go over each block, and look at each MemoryRangeHeader.
- for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(),
- E = CodeSlabs.end(); I != E; ++I) {
- char *Start = (char*)I->base();
- char *End = Start + I->size();
-
- // Check each memory range.
- for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = nullptr;
- Start <= (char*)Hdr && (char*)Hdr < End;
- Hdr = &Hdr->getBlockAfter()) {
- if (Hdr->ThisAllocated == 0) {
- // Check that this range is in the free list.
- if (!FreeHdrSet.count(Hdr)) {
- Err << "Found free header at " << Hdr << " that is not in free list.";
- return false;
- }
-
- // Now make sure the size marker at the end of the block is correct.
- uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1;
- if (!(Start <= (char*)Marker && (char*)Marker < End)) {
- Err << "Block size in header points out of current MemoryBlock.";
- return false;
- }
- if (Hdr->BlockSize != *Marker) {
- Err << "End of block size marker (" << *Marker << ") "
- << "and BlockSize (" << Hdr->BlockSize << ") don't match.";
- return false;
- }
- }
-
- if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) {
- Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != "
- << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")";
- return false;
- } else if (!LastHdr && !Hdr->PrevAllocated) {
- Err << "The first header should have PrevAllocated true.";
- return false;
- }
-
- // Remember the last header.
- LastHdr = Hdr;
- }
- }
-
- // All invariants are preserved.
- return true;
-}
-
-//===----------------------------------------------------------------------===//
-// getPointerToNamedFunction() implementation.
-//===----------------------------------------------------------------------===//
-
-// AtExitHandlers - List of functions to call when the program exits,
-// registered with the atexit() library function.
-static std::vector<void (*)()> AtExitHandlers;
-
-/// runAtExitHandlers - Run any functions registered by the program's
-/// calls to atexit(3), which we intercept and store in
-/// AtExitHandlers.
-///
-static void runAtExitHandlers() {
- while (!AtExitHandlers.empty()) {
- void (*Fn)() = AtExitHandlers.back();
- AtExitHandlers.pop_back();
- Fn();
- }
-}
-
-//===----------------------------------------------------------------------===//
-// Function stubs that are invoked instead of certain library calls
-//
-// Force the following functions to be linked in to anything that uses the
-// JIT. This is a hack designed to work around the all-too-clever Glibc
-// strategy of making these functions work differently when inlined vs. when
-// not inlined, and hiding their real definitions in a separate archive file
-// that the dynamic linker can't see. For more info, search for
-// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274.
-#if defined(__linux__) && defined(__GLIBC__)
-/* stat functions are redirecting to __xstat with a version number. On x86-64
- * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat'
- * available as an exported symbol, so we have to add it explicitly.
- */
-namespace {
-class StatSymbols {
-public:
- StatSymbols() {
- sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat);
- sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat);
- sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat);
- sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64);
- sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64);
- sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64);
- sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64);
- sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64);
- sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64);
- sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit);
- sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod);
- }
-};
-}
-static StatSymbols initStatSymbols;
-#endif // __linux__
-
-// jit_exit - Used to intercept the "exit" library call.
-static void jit_exit(int Status) {
- runAtExitHandlers(); // Run atexit handlers...
- exit(Status);
-}
-
-// jit_atexit - Used to intercept the "atexit" library call.
-static int jit_atexit(void (*Fn)()) {
- AtExitHandlers.push_back(Fn); // Take note of atexit handler...
- return 0; // Always successful
-}
-
-static int jit_noop() {
- return 0;
-}
-
-//===----------------------------------------------------------------------===//
-//
-/// getPointerToNamedFunction - This method returns the address of the specified
-/// function by using the dynamic loader interface. As such it is only useful
-/// for resolving library symbols, not code generated symbols.
-///
-void *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name,
- bool AbortOnFailure) {
- // Check to see if this is one of the functions we want to intercept. Note,
- // we cast to intptr_t here to silence a -pedantic warning that complains
- // about casting a function pointer to a normal pointer.
- if (Name == "exit") return (void*)(intptr_t)&jit_exit;
- if (Name == "atexit") return (void*)(intptr_t)&jit_atexit;
-
- // We should not invoke parent's ctors/dtors from generated main()!
- // On Mingw and Cygwin, the symbol __main is resolved to
- // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors
- // (and register wrong callee's dtors with atexit(3)).
- // We expect ExecutionEngine::runStaticConstructorsDestructors()
- // is called before ExecutionEngine::runFunctionAsMain() is called.
- if (Name == "__main") return (void*)(intptr_t)&jit_noop;
-
- const char *NameStr = Name.c_str();
- // If this is an asm specifier, skip the sentinal.
- if (NameStr[0] == 1) ++NameStr;
-
- // If it's an external function, look it up in the process image...
- void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr);
- if (Ptr) return Ptr;
-
- // If it wasn't found and if it starts with an underscore ('_') character,
- // try again without the underscore.
- if (NameStr[0] == '_') {
- Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1);
- if (Ptr) return Ptr;
- }
-
- // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These
- // are references to hidden visibility symbols that dlsym cannot resolve.
- // If we have one of these, strip off $LDBLStub and try again.
-#if defined(__APPLE__) && defined(__ppc__)
- if (Name.size() > 9 && Name[Name.size()-9] == '$' &&
- memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) {
- // First try turning $LDBLStub into $LDBL128. If that fails, strip it off.
- // This mirrors logic in libSystemStubs.a.
- std::string Prefix = std::string(Name.begin(), Name.end()-9);
- if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false))
- return Ptr;
- if (void *Ptr = getPointerToNamedFunction(Prefix, false))
- return Ptr;
- }
-#endif
-
- if (AbortOnFailure) {
- report_fatal_error("Program used external function '"+Name+
- "' which could not be resolved!");
- }
- return nullptr;
-}
-
-
-
-JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() {
- return new DefaultJITMemoryManager();
-}
-
-const size_t DefaultJITMemoryManager::DefaultCodeSlabSize;
-const size_t DefaultJITMemoryManager::DefaultSlabSize;
-const size_t DefaultJITMemoryManager::DefaultSizeThreshold;
finalizeLoadedModules();
}
+void *MCJIT::getPointerToBasicBlock(BasicBlock *BB) {
+ report_fatal_error("not yet implemented");
+}
+
uint64_t MCJIT::getExistingSymbolAddress(const std::string &Name) {
Mangler Mang(TM->getSubtargetImpl()->getDataLayout());
SmallString<128> FullName;
return (void*)Dyld.getSymbolLoadAddress(Name);
}
+void *MCJIT::recompileAndRelinkFunction(Function *F) {
+ report_fatal_error("not yet implemented");
+}
+
+void MCJIT::freeMachineCodeForFunction(Function *F) {
+ report_fatal_error("not yet implemented");
+}
+
void MCJIT::runStaticConstructorsDestructorsInModulePtrSet(
bool isDtors, ModulePtrSet::iterator I, ModulePtrSet::iterator E) {
for (; I != E; ++I) {
if (!L)
return;
MutexGuard locked(lock);
- auto I = std::find(EventListeners.rbegin(), EventListeners.rend(), L);
+ SmallVector<JITEventListener*, 2>::reverse_iterator I=
+ std::find(EventListeners.rbegin(), EventListeners.rend(), L);
if (I != EventListeners.rend()) {
std::swap(*I, EventListeners.back());
EventListeners.pop_back();
void MCJIT::NotifyFreeingObject(const ObjectImage& Obj) {
MutexGuard locked(lock);
for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
- JITEventListener *L = EventListeners[I];
- L->NotifyFreeingObject(Obj);
+ EventListeners[I]->NotifyFreeingObject(Obj);
}
}
MCContext *Ctx;
LinkingMemoryManager MemMgr;
RuntimeDyld Dyld;
- std::vector<JITEventListener*> EventListeners;
+ SmallVector<JITEventListener*, 2> EventListeners;
OwningModuleContainer OwnedModules;
/// \param isDtors - Run the destructors instead of constructors.
void runStaticConstructorsDestructors(bool isDtors) override;
+ void *getPointerToBasicBlock(BasicBlock *BB) override;
+
void *getPointerToFunction(Function *F) override;
+ void *recompileAndRelinkFunction(Function *F) override;
+
+ void freeMachineCodeForFunction(Function *F) override;
+
GenericValue runFunction(Function *F,
const std::vector<GenericValue> &ArgValues) override;
include $(LEVEL)/Makefile.config
-PARALLEL_DIRS = Interpreter MCJIT RuntimeDyld
+PARALLEL_DIRS = Interpreter JIT MCJIT RuntimeDyld
ifeq ($(USE_INTEL_JITEVENTS), 1)
PARALLEL_DIRS += IntelJITEvents
// MCJIT can generate code for remote targets, but the old JIT and Interpreter
// must use the host architecture.
- if (WhichEngine != EngineKind::Interpreter && M)
+ if (UseMCJIT && WhichEngine != EngineKind::Interpreter && M)
TT.setTriple(M->getTargetTriple());
return selectTarget(TT, MArch, MCPU, MAttrs);
}
// FIXME: non-iOS ARM FastISel is broken with MCJIT.
- if (TheTriple.getArch() == Triple::arm &&
+ if (UseMCJIT &&
+ TheTriple.getArch() == Triple::arm &&
!TheTriple.isiOS() &&
OptLevel == CodeGenOpt::None) {
OptLevel = CodeGenOpt::Less;
tablegen(LLVM AArch64GenRegisterInfo.inc -gen-register-info)
tablegen(LLVM AArch64GenInstrInfo.inc -gen-instr-info)
-tablegen(LLVM AArch64GenMCCodeEmitter.inc -gen-emitter)
+tablegen(LLVM AArch64GenMCCodeEmitter.inc -gen-emitter -mc-emitter)
tablegen(LLVM AArch64GenMCPseudoLowering.inc -gen-pseudo-lowering)
tablegen(LLVM AArch64GenAsmWriter.inc -gen-asm-writer)
tablegen(LLVM AArch64GenAsmWriter1.inc -gen-asm-writer -asmwriternum=1)
class ARMBaseTargetMachine;
class FunctionPass;
class ImmutablePass;
+class JITCodeEmitter;
class MachineInstr;
class MCInst;
class TargetLowering;
FunctionPass *createARMISelDag(ARMBaseTargetMachine &TM,
CodeGenOpt::Level OptLevel);
+
+FunctionPass *createARMJITCodeEmitterPass(ARMBaseTargetMachine &TM,
+ JITCodeEmitter &JCE);
+
FunctionPass *createA15SDOptimizerPass();
FunctionPass *createARMLoadStoreOptimizationPass(bool PreAlloc = false);
FunctionPass *createARMExpandPseudoPass();
--- /dev/null
+//===-- ARM/ARMCodeEmitter.cpp - Convert ARM code to machine code ---------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the pass that transforms the ARM machine instructions into
+// relocatable machine code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ARM.h"
+#include "ARMBaseInstrInfo.h"
+#include "ARMConstantPoolValue.h"
+#include "ARMMachineFunctionInfo.h"
+#include "ARMRelocations.h"
+#include "ARMSubtarget.h"
+#include "ARMTargetMachine.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Function.h"
+#include "llvm/PassManager.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#ifndef NDEBUG
+#include <iomanip>
+#endif
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+STATISTIC(NumEmitted, "Number of machine instructions emitted");
+
+namespace {
+
+ class ARMCodeEmitter : public MachineFunctionPass {
+ ARMJITInfo *JTI;
+ const ARMBaseInstrInfo *II;
+ const DataLayout *TD;
+ const ARMSubtarget *Subtarget;
+ TargetMachine &TM;
+ JITCodeEmitter &MCE;
+ MachineModuleInfo *MMI;
+ const std::vector<MachineConstantPoolEntry> *MCPEs;
+ const std::vector<MachineJumpTableEntry> *MJTEs;
+ bool IsPIC;
+ bool IsThumb;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<MachineModuleInfo>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ static char ID;
+ public:
+ ARMCodeEmitter(TargetMachine &tm, JITCodeEmitter &mce)
+ : MachineFunctionPass(ID), JTI(nullptr),
+ II((const ARMBaseInstrInfo *)tm.getSubtargetImpl()->getInstrInfo()),
+ TD(tm.getSubtargetImpl()->getDataLayout()), TM(tm), MCE(mce),
+ MCPEs(nullptr), MJTEs(nullptr),
+ IsPIC(TM.getRelocationModel() == Reloc::PIC_), IsThumb(false) {}
+
+ /// getBinaryCodeForInstr - This function, generated by the
+ /// CodeEmitterGenerator using TableGen, produces the binary encoding for
+ /// machine instructions.
+ uint64_t getBinaryCodeForInstr(const MachineInstr &MI) const;
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ const char *getPassName() const override {
+ return "ARM Machine Code Emitter";
+ }
+
+ void emitInstruction(const MachineInstr &MI);
+
+ private:
+
+ void emitWordLE(unsigned Binary);
+ void emitDWordLE(uint64_t Binary);
+ void emitConstPoolInstruction(const MachineInstr &MI);
+ void emitMOVi32immInstruction(const MachineInstr &MI);
+ void emitMOVi2piecesInstruction(const MachineInstr &MI);
+ void emitLEApcrelJTInstruction(const MachineInstr &MI);
+ void emitPseudoMoveInstruction(const MachineInstr &MI);
+ void addPCLabel(unsigned LabelID);
+ void emitPseudoInstruction(const MachineInstr &MI);
+ unsigned getMachineSoRegOpValue(const MachineInstr &MI,
+ const MCInstrDesc &MCID,
+ const MachineOperand &MO,
+ unsigned OpIdx);
+
+ unsigned getMachineSoImmOpValue(unsigned SoImm);
+ unsigned getAddrModeSBit(const MachineInstr &MI,
+ const MCInstrDesc &MCID) const;
+
+ void emitDataProcessingInstruction(const MachineInstr &MI,
+ unsigned ImplicitRd = 0,
+ unsigned ImplicitRn = 0);
+
+ void emitLoadStoreInstruction(const MachineInstr &MI,
+ unsigned ImplicitRd = 0,
+ unsigned ImplicitRn = 0);
+
+ void emitMiscLoadStoreInstruction(const MachineInstr &MI,
+ unsigned ImplicitRn = 0);
+
+ void emitLoadStoreMultipleInstruction(const MachineInstr &MI);
+
+ void emitMulFrmInstruction(const MachineInstr &MI);
+
+ void emitExtendInstruction(const MachineInstr &MI);
+
+ void emitMiscArithInstruction(const MachineInstr &MI);
+
+ void emitSaturateInstruction(const MachineInstr &MI);
+
+ void emitBranchInstruction(const MachineInstr &MI);
+
+ void emitInlineJumpTable(unsigned JTIndex);
+
+ void emitMiscBranchInstruction(const MachineInstr &MI);
+
+ void emitVFPArithInstruction(const MachineInstr &MI);
+
+ void emitVFPConversionInstruction(const MachineInstr &MI);
+
+ void emitVFPLoadStoreInstruction(const MachineInstr &MI);
+
+ void emitVFPLoadStoreMultipleInstruction(const MachineInstr &MI);
+
+ void emitNEONLaneInstruction(const MachineInstr &MI);
+ void emitNEONDupInstruction(const MachineInstr &MI);
+ void emitNEON1RegModImmInstruction(const MachineInstr &MI);
+ void emitNEON2RegInstruction(const MachineInstr &MI);
+ void emitNEON3RegInstruction(const MachineInstr &MI);
+
+ /// getMachineOpValue - Return binary encoding of operand. If the machine
+ /// operand requires relocation, record the relocation and return zero.
+ unsigned getMachineOpValue(const MachineInstr &MI,
+ const MachineOperand &MO) const;
+ unsigned getMachineOpValue(const MachineInstr &MI, unsigned OpIdx) const {
+ return getMachineOpValue(MI, MI.getOperand(OpIdx));
+ }
+
+ // FIXME: The legacy JIT ARMCodeEmitter doesn't rely on the the
+ // TableGen'erated getBinaryCodeForInstr() function to encode any
+ // operand values, instead querying getMachineOpValue() directly for
+ // each operand it needs to encode. Thus, any of the new encoder
+ // helper functions can simply return 0 as the values the return
+ // are already handled elsewhere. They are placeholders to allow this
+ // encoder to continue to function until the MC encoder is sufficiently
+ // far along that this one can be eliminated entirely.
+ unsigned NEONThumb2DataIPostEncoder(const MachineInstr &MI, unsigned Val)
+ const { return 0; }
+ unsigned NEONThumb2LoadStorePostEncoder(const MachineInstr &MI,unsigned Val)
+ const { return 0; }
+ unsigned NEONThumb2DupPostEncoder(const MachineInstr &MI,unsigned Val)
+ const { return 0; }
+ unsigned NEONThumb2V8PostEncoder(const MachineInstr &MI,unsigned Val)
+ const { return 0; }
+ unsigned VFPThumb2PostEncoder(const MachineInstr&MI, unsigned Val)
+ const { return 0; }
+ unsigned getAdrLabelOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbAdrLabelOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbBLTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbBLXTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbBRTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbBCCTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbCBTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getBranchTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getUnconditionalBranchTargetOpValue(const MachineInstr &MI,
+ unsigned Op) const { return 0; }
+ unsigned getARMBranchTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getARMBLTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getARMBLXTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getCCOutOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getSOImmOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2SOImmOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getSORegRegOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getSORegImmOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbAddrModeRegRegOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2AddrModeImm8OpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2Imm8s4OpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2AddrModeImm8s4OpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2AddrModeImm0_1020s4OpValue(const MachineInstr &MI,unsigned Op)
+ const { return 0; }
+ unsigned getT2AddrModeImm8OffsetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2AddrModeSORegOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2SORegOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2AdrLabelOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getAddrMode6AddressOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getAddrMode6OneLane32AddressOpValue(const MachineInstr &MI,
+ unsigned Op)
+ const { return 0; }
+ unsigned getAddrMode6DupAddressOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getAddrMode6OffsetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getBitfieldInvertedMaskOpValue(const MachineInstr &MI,
+ unsigned Op) const { return 0; }
+ uint32_t getLdStSORegOpValue(const MachineInstr &MI, unsigned OpIdx)
+ const { return 0; }
+
+ unsigned getAddrModeImm12OpValue(const MachineInstr &MI, unsigned Op)
+ const {
+ // {17-13} = reg
+ // {12} = (U)nsigned (add == '1', sub == '0')
+ // {11-0} = imm12
+ const MachineOperand &MO = MI.getOperand(Op);
+ const MachineOperand &MO1 = MI.getOperand(Op + 1);
+ if (!MO.isReg()) {
+ emitConstPoolAddress(MO.getIndex(), ARM::reloc_arm_cp_entry);
+ return 0;
+ }
+ unsigned Reg = II->getRegisterInfo().getEncodingValue(MO.getReg());
+ int32_t Imm12 = MO1.getImm();
+ uint32_t Binary;
+ Binary = Imm12 & 0xfff;
+ if (Imm12 >= 0)
+ Binary |= (1 << 12);
+ Binary |= (Reg << 13);
+ return Binary;
+ }
+
+ unsigned getHiLo16ImmOpValue(const MachineInstr &MI, unsigned Op) const {
+ return 0;
+ }
+
+ uint32_t getAddrMode2OffsetOpValue(const MachineInstr &MI, unsigned OpIdx)
+ const { return 0;}
+ uint32_t getPostIdxRegOpValue(const MachineInstr &MI, unsigned OpIdx)
+ const { return 0;}
+ uint32_t getAddrMode3OffsetOpValue(const MachineInstr &MI, unsigned OpIdx)
+ const { return 0;}
+ uint32_t getAddrMode3OpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ uint32_t getAddrModeThumbSPOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ uint32_t getAddrModeISOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ uint32_t getAddrModePCOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ uint32_t getAddrMode5OpValue(const MachineInstr &MI, unsigned Op) const {
+ // {17-13} = reg
+ // {12} = (U)nsigned (add == '1', sub == '0')
+ // {11-0} = imm12
+ const MachineOperand &MO = MI.getOperand(Op);
+ const MachineOperand &MO1 = MI.getOperand(Op + 1);
+ if (!MO.isReg()) {
+ emitConstPoolAddress(MO.getIndex(), ARM::reloc_arm_cp_entry);
+ return 0;
+ }
+ unsigned Reg = II->getRegisterInfo().getEncodingValue(MO.getReg());
+ int32_t Imm12 = MO1.getImm();
+
+ // Special value for #-0
+ if (Imm12 == INT32_MIN)
+ Imm12 = 0;
+
+ // Immediate is always encoded as positive. The 'U' bit controls add vs
+ // sub.
+ bool isAdd = true;
+ if (Imm12 < 0) {
+ Imm12 = -Imm12;
+ isAdd = false;
+ }
+
+ uint32_t Binary = Imm12 & 0xfff;
+ if (isAdd)
+ Binary |= (1 << 12);
+ Binary |= (Reg << 13);
+ return Binary;
+ }
+ unsigned getNEONVcvtImm32OpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+
+ unsigned getRegisterListOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+
+ unsigned getShiftRight8Imm(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getShiftRight16Imm(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getShiftRight32Imm(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getShiftRight64Imm(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+
+ /// getMovi32Value - Return binary encoding of operand for movw/movt. If the
+ /// machine operand requires relocation, record the relocation and return
+ /// zero.
+ unsigned getMovi32Value(const MachineInstr &MI,const MachineOperand &MO,
+ unsigned Reloc);
+
+ /// getShiftOp - Return the shift opcode (bit[6:5]) of the immediate value.
+ ///
+ unsigned getShiftOp(unsigned Imm) const ;
+
+ /// Routines that handle operands which add machine relocations which are
+ /// fixed up by the relocation stage.
+ void emitGlobalAddress(const GlobalValue *GV, unsigned Reloc,
+ bool MayNeedFarStub, bool Indirect,
+ intptr_t ACPV = 0) const;
+ void emitExternalSymbolAddress(const char *ES, unsigned Reloc) const;
+ void emitConstPoolAddress(unsigned CPI, unsigned Reloc) const;
+ void emitJumpTableAddress(unsigned JTIndex, unsigned Reloc) const;
+ void emitMachineBasicBlock(MachineBasicBlock *BB, unsigned Reloc,
+ intptr_t JTBase = 0) const;
+ unsigned encodeVFPRd(const MachineInstr &MI, unsigned OpIdx) const;
+ unsigned encodeVFPRn(const MachineInstr &MI, unsigned OpIdx) const;
+ unsigned encodeVFPRm(const MachineInstr &MI, unsigned OpIdx) const;
+ unsigned encodeNEONRd(const MachineInstr &MI, unsigned OpIdx) const;
+ unsigned encodeNEONRn(const MachineInstr &MI, unsigned OpIdx) const;
+ unsigned encodeNEONRm(const MachineInstr &MI, unsigned OpIdx) const;
+ };
+}
+
+char ARMCodeEmitter::ID = 0;
+
+/// createARMJITCodeEmitterPass - Return a pass that emits the collected ARM
+/// code to the specified MCE object.
+FunctionPass *llvm::createARMJITCodeEmitterPass(ARMBaseTargetMachine &TM,
+ JITCodeEmitter &JCE) {
+ return new ARMCodeEmitter(TM, JCE);
+}
+
+bool ARMCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
+ TargetMachine &Target = const_cast<TargetMachine&>(MF.getTarget());
+
+ assert((Target.getRelocationModel() != Reloc::Default ||
+ Target.getRelocationModel() != Reloc::Static) &&
+ "JIT relocation model must be set to static or default!");
+ // Initialize the subtarget first so we can grab all of the
+ // subtarget dependent variables from there.
+ Subtarget = &TM.getSubtarget<ARMSubtarget>();
+ JTI = static_cast<ARMJITInfo *>(Target.getSubtargetImpl()->getJITInfo());
+ II = static_cast<const ARMBaseInstrInfo *>(Subtarget->getInstrInfo());
+ TD = Target.getSubtargetImpl()->getDataLayout();
+
+ MCPEs = &MF.getConstantPool()->getConstants();
+ MJTEs = nullptr;
+ if (MF.getJumpTableInfo()) MJTEs = &MF.getJumpTableInfo()->getJumpTables();
+ IsPIC = TM.getRelocationModel() == Reloc::PIC_;
+ IsThumb = MF.getInfo<ARMFunctionInfo>()->isThumbFunction();
+ JTI->Initialize(MF, IsPIC);
+ MMI = &getAnalysis<MachineModuleInfo>();
+ MCE.setModuleInfo(MMI);
+
+ do {
+ DEBUG(errs() << "JITTing function '"
+ << MF.getName() << "'\n");
+ MCE.startFunction(MF);
+ for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
+ MBB != E; ++MBB) {
+ MCE.StartMachineBasicBlock(MBB);
+ for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
+ I != E; ++I)
+ emitInstruction(*I);
+ }
+ } while (MCE.finishFunction(MF));
+
+ return false;
+}
+
+/// getShiftOp - Return the shift opcode (bit[6:5]) of the immediate value.
+///
+unsigned ARMCodeEmitter::getShiftOp(unsigned Imm) const {
+ switch (ARM_AM::getAM2ShiftOpc(Imm)) {
+ default: llvm_unreachable("Unknown shift opc!");
+ case ARM_AM::asr: return 2;
+ case ARM_AM::lsl: return 0;
+ case ARM_AM::lsr: return 1;
+ case ARM_AM::ror:
+ case ARM_AM::rrx: return 3;
+ }
+}
+
+/// getMovi32Value - Return binary encoding of operand for movw/movt. If the
+/// machine operand requires relocation, record the relocation and return zero.
+unsigned ARMCodeEmitter::getMovi32Value(const MachineInstr &MI,
+ const MachineOperand &MO,
+ unsigned Reloc) {
+ assert(((Reloc == ARM::reloc_arm_movt) || (Reloc == ARM::reloc_arm_movw))
+ && "Relocation to this function should be for movt or movw");
+
+ if (MO.isImm())
+ return static_cast<unsigned>(MO.getImm());
+ else if (MO.isGlobal())
+ emitGlobalAddress(MO.getGlobal(), Reloc, true, false);
+ else if (MO.isSymbol())
+ emitExternalSymbolAddress(MO.getSymbolName(), Reloc);
+ else if (MO.isMBB())
+ emitMachineBasicBlock(MO.getMBB(), Reloc);
+ else {
+#ifndef NDEBUG
+ errs() << MO;
+#endif
+ llvm_unreachable("Unsupported operand type for movw/movt");
+ }
+ return 0;
+}
+
+/// getMachineOpValue - Return binary encoding of operand. If the machine
+/// operand requires relocation, record the relocation and return zero.
+unsigned ARMCodeEmitter::getMachineOpValue(const MachineInstr &MI,
+ const MachineOperand &MO) const {
+ if (MO.isReg())
+ return II->getRegisterInfo().getEncodingValue(MO.getReg());
+ else if (MO.isImm())
+ return static_cast<unsigned>(MO.getImm());
+ else if (MO.isGlobal())
+ emitGlobalAddress(MO.getGlobal(), ARM::reloc_arm_branch, true, false);
+ else if (MO.isSymbol())
+ emitExternalSymbolAddress(MO.getSymbolName(), ARM::reloc_arm_branch);
+ else if (MO.isCPI()) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ // For VFP load, the immediate offset is multiplied by 4.
+ unsigned Reloc = ((MCID.TSFlags & ARMII::FormMask) == ARMII::VFPLdStFrm)
+ ? ARM::reloc_arm_vfp_cp_entry : ARM::reloc_arm_cp_entry;
+ emitConstPoolAddress(MO.getIndex(), Reloc);
+ } else if (MO.isJTI())
+ emitJumpTableAddress(MO.getIndex(), ARM::reloc_arm_relative);
+ else if (MO.isMBB())
+ emitMachineBasicBlock(MO.getMBB(), ARM::reloc_arm_branch);
+ else
+ llvm_unreachable("Unable to encode MachineOperand!");
+ return 0;
+}
+
+/// emitGlobalAddress - Emit the specified address to the code stream.
+///
+void ARMCodeEmitter::emitGlobalAddress(const GlobalValue *GV, unsigned Reloc,
+ bool MayNeedFarStub, bool Indirect,
+ intptr_t ACPV) const {
+ MachineRelocation MR = Indirect
+ ? MachineRelocation::getIndirectSymbol(MCE.getCurrentPCOffset(), Reloc,
+ const_cast<GlobalValue *>(GV),
+ ACPV, MayNeedFarStub)
+ : MachineRelocation::getGV(MCE.getCurrentPCOffset(), Reloc,
+ const_cast<GlobalValue *>(GV), ACPV,
+ MayNeedFarStub);
+ MCE.addRelocation(MR);
+}
+
+/// emitExternalSymbolAddress - Arrange for the address of an external symbol to
+/// be emitted to the current location in the function, and allow it to be PC
+/// relative.
+void ARMCodeEmitter::
+emitExternalSymbolAddress(const char *ES, unsigned Reloc) const {
+ MCE.addRelocation(MachineRelocation::getExtSym(MCE.getCurrentPCOffset(),
+ Reloc, ES));
+}
+
+/// emitConstPoolAddress - Arrange for the address of an constant pool
+/// to be emitted to the current location in the function, and allow it to be PC
+/// relative.
+void ARMCodeEmitter::emitConstPoolAddress(unsigned CPI, unsigned Reloc) const {
+ // Tell JIT emitter we'll resolve the address.
+ MCE.addRelocation(MachineRelocation::getConstPool(MCE.getCurrentPCOffset(),
+ Reloc, CPI, 0, true));
+}
+
+/// emitJumpTableAddress - Arrange for the address of a jump table to
+/// be emitted to the current location in the function, and allow it to be PC
+/// relative.
+void ARMCodeEmitter::
+emitJumpTableAddress(unsigned JTIndex, unsigned Reloc) const {
+ MCE.addRelocation(MachineRelocation::getJumpTable(MCE.getCurrentPCOffset(),
+ Reloc, JTIndex, 0, true));
+}
+
+/// emitMachineBasicBlock - Emit the specified address basic block.
+void ARMCodeEmitter::emitMachineBasicBlock(MachineBasicBlock *BB,
+ unsigned Reloc,
+ intptr_t JTBase) const {
+ MCE.addRelocation(MachineRelocation::getBB(MCE.getCurrentPCOffset(),
+ Reloc, BB, JTBase));
+}
+
+void ARMCodeEmitter::emitWordLE(unsigned Binary) {
+ DEBUG(errs() << " 0x";
+ errs().write_hex(Binary) << "\n");
+ MCE.emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitDWordLE(uint64_t Binary) {
+ DEBUG(errs() << " 0x";
+ errs().write_hex(Binary) << "\n");
+ MCE.emitDWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitInstruction(const MachineInstr &MI) {
+ DEBUG(errs() << "JIT: " << (void*)MCE.getCurrentPCValue() << ":\t" << MI);
+
+ MCE.processDebugLoc(MI.getDebugLoc(), true);
+
+ ++NumEmitted; // Keep track of the # of mi's emitted
+ switch (MI.getDesc().TSFlags & ARMII::FormMask) {
+ default: {
+ llvm_unreachable("Unhandled instruction encoding format!");
+ }
+ case ARMII::MiscFrm:
+ if (MI.getOpcode() == ARM::LEApcrelJT) {
+ // Materialize jumptable address.
+ emitLEApcrelJTInstruction(MI);
+ break;
+ }
+ llvm_unreachable("Unhandled instruction encoding!");
+ case ARMII::Pseudo:
+ emitPseudoInstruction(MI);
+ break;
+ case ARMII::DPFrm:
+ case ARMII::DPSoRegFrm:
+ emitDataProcessingInstruction(MI);
+ break;
+ case ARMII::LdFrm:
+ case ARMII::StFrm:
+ emitLoadStoreInstruction(MI);
+ break;
+ case ARMII::LdMiscFrm:
+ case ARMII::StMiscFrm:
+ emitMiscLoadStoreInstruction(MI);
+ break;
+ case ARMII::LdStMulFrm:
+ emitLoadStoreMultipleInstruction(MI);
+ break;
+ case ARMII::MulFrm:
+ emitMulFrmInstruction(MI);
+ break;
+ case ARMII::ExtFrm:
+ emitExtendInstruction(MI);
+ break;
+ case ARMII::ArithMiscFrm:
+ emitMiscArithInstruction(MI);
+ break;
+ case ARMII::SatFrm:
+ emitSaturateInstruction(MI);
+ break;
+ case ARMII::BrFrm:
+ emitBranchInstruction(MI);
+ break;
+ case ARMII::BrMiscFrm:
+ emitMiscBranchInstruction(MI);
+ break;
+ // VFP instructions.
+ case ARMII::VFPUnaryFrm:
+ case ARMII::VFPBinaryFrm:
+ emitVFPArithInstruction(MI);
+ break;
+ case ARMII::VFPConv1Frm:
+ case ARMII::VFPConv2Frm:
+ case ARMII::VFPConv3Frm:
+ case ARMII::VFPConv4Frm:
+ case ARMII::VFPConv5Frm:
+ emitVFPConversionInstruction(MI);
+ break;
+ case ARMII::VFPLdStFrm:
+ emitVFPLoadStoreInstruction(MI);
+ break;
+ case ARMII::VFPLdStMulFrm:
+ emitVFPLoadStoreMultipleInstruction(MI);
+ break;
+
+ // NEON instructions.
+ case ARMII::NGetLnFrm:
+ case ARMII::NSetLnFrm:
+ emitNEONLaneInstruction(MI);
+ break;
+ case ARMII::NDupFrm:
+ emitNEONDupInstruction(MI);
+ break;
+ case ARMII::N1RegModImmFrm:
+ emitNEON1RegModImmInstruction(MI);
+ break;
+ case ARMII::N2RegFrm:
+ emitNEON2RegInstruction(MI);
+ break;
+ case ARMII::N3RegFrm:
+ emitNEON3RegInstruction(MI);
+ break;
+ }
+ MCE.processDebugLoc(MI.getDebugLoc(), false);
+}
+
+void ARMCodeEmitter::emitConstPoolInstruction(const MachineInstr &MI) {
+ unsigned CPI = MI.getOperand(0).getImm(); // CP instruction index.
+ unsigned CPIndex = MI.getOperand(1).getIndex(); // Actual cp entry index.
+ const MachineConstantPoolEntry &MCPE = (*MCPEs)[CPIndex];
+
+ // Remember the CONSTPOOL_ENTRY address for later relocation.
+ JTI->addConstantPoolEntryAddr(CPI, MCE.getCurrentPCValue());
+
+ // Emit constpool island entry. In most cases, the actual values will be
+ // resolved and relocated after code emission.
+ if (MCPE.isMachineConstantPoolEntry()) {
+ ARMConstantPoolValue *ACPV =
+ static_cast<ARMConstantPoolValue*>(MCPE.Val.MachineCPVal);
+
+ DEBUG(errs() << " ** ARM constant pool #" << CPI << " @ "
+ << (void*)MCE.getCurrentPCValue() << " " << *ACPV << '\n');
+
+ assert(ACPV->isGlobalValue() && "unsupported constant pool value");
+ const GlobalValue *GV = cast<ARMConstantPoolConstant>(ACPV)->getGV();
+ if (GV) {
+ Reloc::Model RelocM = TM.getRelocationModel();
+ emitGlobalAddress(GV, ARM::reloc_arm_machine_cp_entry,
+ isa<Function>(GV),
+ Subtarget->GVIsIndirectSymbol(GV, RelocM),
+ (intptr_t)ACPV);
+ } else {
+ const char *Sym = cast<ARMConstantPoolSymbol>(ACPV)->getSymbol();
+ emitExternalSymbolAddress(Sym, ARM::reloc_arm_absolute);
+ }
+ emitWordLE(0);
+ } else {
+ const Constant *CV = MCPE.Val.ConstVal;
+
+ DEBUG({
+ errs() << " ** Constant pool #" << CPI << " @ "
+ << (void*)MCE.getCurrentPCValue() << " ";
+ if (const Function *F = dyn_cast<Function>(CV))
+ errs() << F->getName();
+ else
+ errs() << *CV;
+ errs() << '\n';
+ });
+
+ if (const GlobalValue *GV = dyn_cast<GlobalValue>(CV)) {
+ emitGlobalAddress(GV, ARM::reloc_arm_absolute, isa<Function>(GV), false);
+ emitWordLE(0);
+ } else if (const ConstantInt *CI = dyn_cast<ConstantInt>(CV)) {
+ uint32_t Val = uint32_t(*CI->getValue().getRawData());
+ emitWordLE(Val);
+ } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CV)) {
+ if (CFP->getType()->isFloatTy())
+ emitWordLE(CFP->getValueAPF().bitcastToAPInt().getZExtValue());
+ else if (CFP->getType()->isDoubleTy())
+ emitDWordLE(CFP->getValueAPF().bitcastToAPInt().getZExtValue());
+ else {
+ llvm_unreachable("Unable to handle this constantpool entry!");
+ }
+ } else {
+ llvm_unreachable("Unable to handle this constantpool entry!");
+ }
+ }
+}
+
+void ARMCodeEmitter::emitMOVi32immInstruction(const MachineInstr &MI) {
+ const MachineOperand &MO0 = MI.getOperand(0);
+ const MachineOperand &MO1 = MI.getOperand(1);
+
+ // Emit the 'movw' instruction.
+ unsigned Binary = 0x30 << 20; // mov: Insts{27-20} = 0b00110000
+
+ unsigned Lo16 = getMovi32Value(MI, MO1, ARM::reloc_arm_movw) & 0xFFFF;
+
+ // Set the conditional execution predicate.
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode Rd.
+ Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRdShift;
+
+ // Encode imm16 as imm4:imm12
+ Binary |= Lo16 & 0xFFF; // Insts{11-0} = imm12
+ Binary |= ((Lo16 >> 12) & 0xF) << 16; // Insts{19-16} = imm4
+ emitWordLE(Binary);
+
+ unsigned Hi16 = getMovi32Value(MI, MO1, ARM::reloc_arm_movt) >> 16;
+ // Emit the 'movt' instruction.
+ Binary = 0x34 << 20; // movt: Insts{27-20} = 0b00110100
+
+ // Set the conditional execution predicate.
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode Rd.
+ Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRdShift;
+
+ // Encode imm16 as imm4:imm1, same as movw above.
+ Binary |= Hi16 & 0xFFF;
+ Binary |= ((Hi16 >> 12) & 0xF) << 16;
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitMOVi2piecesInstruction(const MachineInstr &MI) {
+ const MachineOperand &MO0 = MI.getOperand(0);
+ const MachineOperand &MO1 = MI.getOperand(1);
+ assert(MO1.isImm() && ARM_AM::isSOImmTwoPartVal(MO1.getImm()) &&
+ "Not a valid so_imm value!");
+ unsigned V1 = ARM_AM::getSOImmTwoPartFirst(MO1.getImm());
+ unsigned V2 = ARM_AM::getSOImmTwoPartSecond(MO1.getImm());
+
+ // Emit the 'mov' instruction.
+ unsigned Binary = 0xd << 21; // mov: Insts{24-21} = 0b1101
+
+ // Set the conditional execution predicate.
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode Rd.
+ Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRdShift;
+
+ // Encode so_imm.
+ // Set bit I(25) to identify this is the immediate form of <shifter_op>
+ Binary |= 1 << ARMII::I_BitShift;
+ Binary |= getMachineSoImmOpValue(V1);
+ emitWordLE(Binary);
+
+ // Now the 'orr' instruction.
+ Binary = 0xc << 21; // orr: Insts{24-21} = 0b1100
+
+ // Set the conditional execution predicate.
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode Rd.
+ Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRdShift;
+
+ // Encode Rn.
+ Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRnShift;
+
+ // Encode so_imm.
+ // Set bit I(25) to identify this is the immediate form of <shifter_op>
+ Binary |= 1 << ARMII::I_BitShift;
+ Binary |= getMachineSoImmOpValue(V2);
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitLEApcrelJTInstruction(const MachineInstr &MI) {
+ // It's basically add r, pc, (LJTI - $+8)
+
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Emit the 'add' instruction.
+ unsigned Binary = 0x4 << 21; // add: Insts{24-21} = 0b0100
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode S bit if MI modifies CPSR.
+ Binary |= getAddrModeSBit(MI, MCID);
+
+ // Encode Rd.
+ Binary |= getMachineOpValue(MI, 0) << ARMII::RegRdShift;
+
+ // Encode Rn which is PC.
+ Binary |= II->getRegisterInfo().getEncodingValue(ARM::PC) << ARMII::RegRnShift;
+
+ // Encode the displacement.
+ Binary |= 1 << ARMII::I_BitShift;
+ emitJumpTableAddress(MI.getOperand(1).getIndex(), ARM::reloc_arm_jt_base);
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitPseudoMoveInstruction(const MachineInstr &MI) {
+ unsigned Opcode = MI.getDesc().Opcode;
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode S bit if MI modifies CPSR.
+ if (Opcode == ARM::MOVsrl_flag || Opcode == ARM::MOVsra_flag)
+ Binary |= 1 << ARMII::S_BitShift;
+
+ // Encode register def if there is one.
+ Binary |= getMachineOpValue(MI, 0) << ARMII::RegRdShift;
+
+ // Encode the shift operation.
+ switch (Opcode) {
+ default: break;
+ case ARM::RRX:
+ // rrx
+ Binary |= 0x6 << 4;
+ break;
+ case ARM::MOVsrl_flag:
+ // lsr #1
+ Binary |= (0x2 << 4) | (1 << 7);
+ break;
+ case ARM::MOVsra_flag:
+ // asr #1
+ Binary |= (0x4 << 4) | (1 << 7);
+ break;
+ }
+
+ // Encode register Rm.
+ Binary |= getMachineOpValue(MI, 1);
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::addPCLabel(unsigned LabelID) {
+ DEBUG(errs() << " ** LPC" << LabelID << " @ "
+ << (void*)MCE.getCurrentPCValue() << '\n');
+ JTI->addPCLabelAddr(LabelID, MCE.getCurrentPCValue());
+}
+
+void ARMCodeEmitter::emitPseudoInstruction(const MachineInstr &MI) {
+ unsigned Opcode = MI.getDesc().Opcode;
+ switch (Opcode) {
+ default:
+ llvm_unreachable("ARMCodeEmitter::emitPseudoInstruction");
+ case ARM::BX_CALL:
+ case ARM::BMOVPCRX_CALL: {
+ // First emit mov lr, pc
+ unsigned Binary = 0x01a0e00f;
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+ emitWordLE(Binary);
+
+ // and then emit the branch.
+ emitMiscBranchInstruction(MI);
+ break;
+ }
+ case TargetOpcode::INLINEASM: {
+ // We allow inline assembler nodes with empty bodies - they can
+ // implicitly define registers, which is ok for JIT.
+ if (MI.getOperand(0).getSymbolName()[0]) {
+ report_fatal_error("JIT does not support inline asm!");
+ }
+ break;
+ }
+ case TargetOpcode::CFI_INSTRUCTION:
+ break;
+ case TargetOpcode::EH_LABEL:
+ MCE.emitLabel(MI.getOperand(0).getMCSymbol());
+ break;
+ case TargetOpcode::IMPLICIT_DEF:
+ case TargetOpcode::KILL:
+ // Do nothing.
+ break;
+ case ARM::CONSTPOOL_ENTRY:
+ emitConstPoolInstruction(MI);
+ break;
+ case ARM::PICADD: {
+ // Remember of the address of the PC label for relocation later.
+ addPCLabel(MI.getOperand(2).getImm());
+ // PICADD is just an add instruction that implicitly read pc.
+ emitDataProcessingInstruction(MI, 0, ARM::PC);
+ break;
+ }
+ case ARM::PICLDR:
+ case ARM::PICLDRB:
+ case ARM::PICSTR:
+ case ARM::PICSTRB: {
+ // Remember of the address of the PC label for relocation later.
+ addPCLabel(MI.getOperand(2).getImm());
+ // These are just load / store instructions that implicitly read pc.
+ emitLoadStoreInstruction(MI, 0, ARM::PC);
+ break;
+ }
+ case ARM::PICLDRH:
+ case ARM::PICLDRSH:
+ case ARM::PICLDRSB:
+ case ARM::PICSTRH: {
+ // Remember of the address of the PC label for relocation later.
+ addPCLabel(MI.getOperand(2).getImm());
+ // These are just load / store instructions that implicitly read pc.
+ emitMiscLoadStoreInstruction(MI, ARM::PC);
+ break;
+ }
+
+ case ARM::MOVi32imm:
+ // Two instructions to materialize a constant.
+ if (Subtarget->hasV6T2Ops())
+ emitMOVi32immInstruction(MI);
+ else
+ emitMOVi2piecesInstruction(MI);
+ break;
+
+ case ARM::LEApcrelJT:
+ // Materialize jumptable address.
+ emitLEApcrelJTInstruction(MI);
+ break;
+ case ARM::RRX:
+ case ARM::MOVsrl_flag:
+ case ARM::MOVsra_flag:
+ emitPseudoMoveInstruction(MI);
+ break;
+ }
+}
+
+unsigned ARMCodeEmitter::getMachineSoRegOpValue(const MachineInstr &MI,
+ const MCInstrDesc &MCID,
+ const MachineOperand &MO,
+ unsigned OpIdx) {
+ unsigned Binary = getMachineOpValue(MI, MO);
+
+ const MachineOperand &MO1 = MI.getOperand(OpIdx + 1);
+ const MachineOperand &MO2 = MI.getOperand(OpIdx + 2);
+ ARM_AM::ShiftOpc SOpc = ARM_AM::getSORegShOp(MO2.getImm());
+
+ // Encode the shift opcode.
+ unsigned SBits = 0;
+ unsigned Rs = MO1.getReg();
+ if (Rs) {
+ // Set shift operand (bit[7:4]).
+ // LSL - 0001
+ // LSR - 0011
+ // ASR - 0101
+ // ROR - 0111
+ // RRX - 0110 and bit[11:8] clear.
+ switch (SOpc) {
+ default: llvm_unreachable("Unknown shift opc!");
+ case ARM_AM::lsl: SBits = 0x1; break;
+ case ARM_AM::lsr: SBits = 0x3; break;
+ case ARM_AM::asr: SBits = 0x5; break;
+ case ARM_AM::ror: SBits = 0x7; break;
+ case ARM_AM::rrx: SBits = 0x6; break;
+ }
+ } else {
+ // Set shift operand (bit[6:4]).
+ // LSL - 000
+ // LSR - 010
+ // ASR - 100
+ // ROR - 110
+ switch (SOpc) {
+ default: llvm_unreachable("Unknown shift opc!");
+ case ARM_AM::lsl: SBits = 0x0; break;
+ case ARM_AM::lsr: SBits = 0x2; break;
+ case ARM_AM::asr: SBits = 0x4; break;
+ case ARM_AM::ror: SBits = 0x6; break;
+ }
+ }
+ Binary |= SBits << 4;
+ if (SOpc == ARM_AM::rrx)
+ return Binary;
+
+ // Encode the shift operation Rs or shift_imm (except rrx).
+ if (Rs) {
+ // Encode Rs bit[11:8].
+ assert(ARM_AM::getSORegOffset(MO2.getImm()) == 0);
+ return Binary | (II->getRegisterInfo().getEncodingValue(Rs) << ARMII::RegRsShift);
+ }
+
+ // Encode shift_imm bit[11:7].
+ return Binary | ARM_AM::getSORegOffset(MO2.getImm()) << 7;
+}
+
+unsigned ARMCodeEmitter::getMachineSoImmOpValue(unsigned SoImm) {
+ int SoImmVal = ARM_AM::getSOImmVal(SoImm);
+ assert(SoImmVal != -1 && "Not a valid so_imm value!");
+
+ // Encode rotate_imm.
+ unsigned Binary = (ARM_AM::getSOImmValRot((unsigned)SoImmVal) >> 1)
+ << ARMII::SoRotImmShift;
+
+ // Encode immed_8.
+ Binary |= ARM_AM::getSOImmValImm((unsigned)SoImmVal);
+ return Binary;
+}
+
+unsigned ARMCodeEmitter::getAddrModeSBit(const MachineInstr &MI,
+ const MCInstrDesc &MCID) const {
+ for (unsigned i = MI.getNumOperands(), e = MCID.getNumOperands(); i >= e;--i){
+ const MachineOperand &MO = MI.getOperand(i-1);
+ if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR)
+ return 1 << ARMII::S_BitShift;
+ }
+ return 0;
+}
+
+void ARMCodeEmitter::emitDataProcessingInstruction(const MachineInstr &MI,
+ unsigned ImplicitRd,
+ unsigned ImplicitRn) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode S bit if MI modifies CPSR.
+ Binary |= getAddrModeSBit(MI, MCID);
+
+ // Encode register def if there is one.
+ unsigned NumDefs = MCID.getNumDefs();
+ unsigned OpIdx = 0;
+ if (NumDefs)
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
+ else if (ImplicitRd)
+ // Special handling for implicit use (e.g. PC).
+ Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRd) << ARMII::RegRdShift);
+
+ if (MCID.Opcode == ARM::MOVi16) {
+ // Get immediate from MI.
+ unsigned Lo16 = getMovi32Value(MI, MI.getOperand(OpIdx),
+ ARM::reloc_arm_movw);
+ // Encode imm which is the same as in emitMOVi32immInstruction().
+ Binary |= Lo16 & 0xFFF;
+ Binary |= ((Lo16 >> 12) & 0xF) << 16;
+ emitWordLE(Binary);
+ return;
+ } else if(MCID.Opcode == ARM::MOVTi16) {
+ unsigned Hi16 = (getMovi32Value(MI, MI.getOperand(OpIdx),
+ ARM::reloc_arm_movt) >> 16);
+ Binary |= Hi16 & 0xFFF;
+ Binary |= ((Hi16 >> 12) & 0xF) << 16;
+ emitWordLE(Binary);
+ return;
+ } else if ((MCID.Opcode == ARM::BFC) || (MCID.Opcode == ARM::BFI)) {
+ uint32_t v = ~MI.getOperand(2).getImm();
+ int32_t lsb = countTrailingZeros(v);
+ int32_t msb = (32 - countLeadingZeros(v)) - 1;
+ // Instr{20-16} = msb, Instr{11-7} = lsb
+ Binary |= (msb & 0x1F) << 16;
+ Binary |= (lsb & 0x1F) << 7;
+ emitWordLE(Binary);
+ return;
+ } else if ((MCID.Opcode == ARM::UBFX) || (MCID.Opcode == ARM::SBFX)) {
+ // Encode Rn in Instr{0-3}
+ Binary |= getMachineOpValue(MI, OpIdx++);
+
+ uint32_t lsb = MI.getOperand(OpIdx++).getImm();
+ uint32_t widthm1 = MI.getOperand(OpIdx++).getImm() - 1;
+
+ // Instr{20-16} = widthm1, Instr{11-7} = lsb
+ Binary |= (widthm1 & 0x1F) << 16;
+ Binary |= (lsb & 0x1F) << 7;
+ emitWordLE(Binary);
+ return;
+ }
+
+ // If this is a two-address operand, skip it. e.g. MOVCCr operand 1.
+ if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
+ ++OpIdx;
+
+ // Encode first non-shifter register operand if there is one.
+ bool isUnary = MCID.TSFlags & ARMII::UnaryDP;
+ if (!isUnary) {
+ if (ImplicitRn)
+ // Special handling for implicit use (e.g. PC).
+ Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRn) << ARMII::RegRnShift);
+ else {
+ Binary |= getMachineOpValue(MI, OpIdx) << ARMII::RegRnShift;
+ ++OpIdx;
+ }
+ }
+
+ // Encode shifter operand.
+ const MachineOperand &MO = MI.getOperand(OpIdx);
+ if ((MCID.TSFlags & ARMII::FormMask) == ARMII::DPSoRegFrm) {
+ // Encode SoReg.
+ emitWordLE(Binary | getMachineSoRegOpValue(MI, MCID, MO, OpIdx));
+ return;
+ }
+
+ if (MO.isReg()) {
+ // Encode register Rm.
+ emitWordLE(Binary | II->getRegisterInfo().getEncodingValue(MO.getReg()));
+ return;
+ }
+
+ // Encode so_imm.
+ Binary |= getMachineSoImmOpValue((unsigned)MO.getImm());
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitLoadStoreInstruction(const MachineInstr &MI,
+ unsigned ImplicitRd,
+ unsigned ImplicitRn) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ unsigned Form = MCID.TSFlags & ARMII::FormMask;
+ bool IsPrePost = (MCID.TSFlags & ARMII::IndexModeMask) != 0;
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // If this is an LDRi12, STRi12 or LDRcp, nothing more needs be done.
+ if (MI.getOpcode() == ARM::LDRi12 || MI.getOpcode() == ARM::LDRcp ||
+ MI.getOpcode() == ARM::STRi12) {
+ emitWordLE(Binary);
+ return;
+ }
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ unsigned OpIdx = 0;
+
+ // Operand 0 of a pre- and post-indexed store is the address base
+ // writeback. Skip it.
+ bool Skipped = false;
+ if (IsPrePost && Form == ARMII::StFrm) {
+ ++OpIdx;
+ Skipped = true;
+ }
+
+ // Set first operand
+ if (ImplicitRd)
+ // Special handling for implicit use (e.g. PC).
+ Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRd) << ARMII::RegRdShift);
+ else
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
+
+ // Set second operand
+ if (ImplicitRn)
+ // Special handling for implicit use (e.g. PC).
+ Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRn) << ARMII::RegRnShift);
+ else
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
+
+ // If this is a two-address operand, skip it. e.g. LDR_PRE.
+ if (!Skipped && MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
+ ++OpIdx;
+
+ const MachineOperand &MO2 = MI.getOperand(OpIdx);
+ unsigned AM2Opc = (ImplicitRn == ARM::PC)
+ ? 0 : MI.getOperand(OpIdx+1).getImm();
+
+ // Set bit U(23) according to sign of immed value (positive or negative).
+ Binary |= ((ARM_AM::getAM2Op(AM2Opc) == ARM_AM::add ? 1 : 0) <<
+ ARMII::U_BitShift);
+ if (!MO2.getReg()) { // is immediate
+ if (ARM_AM::getAM2Offset(AM2Opc))
+ // Set the value of offset_12 field
+ Binary |= ARM_AM::getAM2Offset(AM2Opc);
+ emitWordLE(Binary);
+ return;
+ }
+
+ // Set bit I(25), because this is not in immediate encoding.
+ Binary |= 1 << ARMII::I_BitShift;
+ assert(TargetRegisterInfo::isPhysicalRegister(MO2.getReg()));
+ // Set bit[3:0] to the corresponding Rm register
+ Binary |= II->getRegisterInfo().getEncodingValue(MO2.getReg());
+
+ // If this instr is in scaled register offset/index instruction, set
+ // shift_immed(bit[11:7]) and shift(bit[6:5]) fields.
+ if (unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc)) {
+ Binary |= getShiftOp(AM2Opc) << ARMII::ShiftImmShift; // shift
+ Binary |= ShImm << ARMII::ShiftShift; // shift_immed
+ }
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitMiscLoadStoreInstruction(const MachineInstr &MI,
+ unsigned ImplicitRn) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ unsigned Form = MCID.TSFlags & ARMII::FormMask;
+ bool IsPrePost = (MCID.TSFlags & ARMII::IndexModeMask) != 0;
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ unsigned OpIdx = 0;
+
+ // Operand 0 of a pre- and post-indexed store is the address base
+ // writeback. Skip it.
+ bool Skipped = false;
+ if (IsPrePost && Form == ARMII::StMiscFrm) {
+ ++OpIdx;
+ Skipped = true;
+ }
+
+ // Set first operand
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
+
+ // Skip LDRD and STRD's second operand.
+ if (MCID.Opcode == ARM::LDRD || MCID.Opcode == ARM::STRD)
+ ++OpIdx;
+
+ // Set second operand
+ if (ImplicitRn)
+ // Special handling for implicit use (e.g. PC).
+ Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRn) << ARMII::RegRnShift);
+ else
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
+
+ // If this is a two-address operand, skip it. e.g. LDRH_POST.
+ if (!Skipped && MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
+ ++OpIdx;
+
+ const MachineOperand &MO2 = MI.getOperand(OpIdx);
+ unsigned AM3Opc = (ImplicitRn == ARM::PC)
+ ? 0 : MI.getOperand(OpIdx+1).getImm();
+
+ // Set bit U(23) according to sign of immed value (positive or negative)
+ Binary |= ((ARM_AM::getAM3Op(AM3Opc) == ARM_AM::add ? 1 : 0) <<
+ ARMII::U_BitShift);
+
+ // If this instr is in register offset/index encoding, set bit[3:0]
+ // to the corresponding Rm register.
+ if (MO2.getReg()) {
+ Binary |= II->getRegisterInfo().getEncodingValue(MO2.getReg());
+ emitWordLE(Binary);
+ return;
+ }
+
+ // This instr is in immediate offset/index encoding, set bit 22 to 1.
+ Binary |= 1 << ARMII::AM3_I_BitShift;
+ if (unsigned ImmOffs = ARM_AM::getAM3Offset(AM3Opc)) {
+ // Set operands
+ Binary |= (ImmOffs >> 4) << ARMII::ImmHiShift; // immedH
+ Binary |= (ImmOffs & 0xF); // immedL
+ }
+
+ emitWordLE(Binary);
+}
+
+static unsigned getAddrModeUPBits(unsigned Mode) {
+ unsigned Binary = 0;
+
+ // Set addressing mode by modifying bits U(23) and P(24)
+ // IA - Increment after - bit U = 1 and bit P = 0
+ // IB - Increment before - bit U = 1 and bit P = 1
+ // DA - Decrement after - bit U = 0 and bit P = 0
+ // DB - Decrement before - bit U = 0 and bit P = 1
+ switch (Mode) {
+ default: llvm_unreachable("Unknown addressing sub-mode!");
+ case ARM_AM::da: break;
+ case ARM_AM::db: Binary |= 0x1 << ARMII::P_BitShift; break;
+ case ARM_AM::ia: Binary |= 0x1 << ARMII::U_BitShift; break;
+ case ARM_AM::ib: Binary |= 0x3 << ARMII::U_BitShift; break;
+ }
+
+ return Binary;
+}
+
+void ARMCodeEmitter::emitLoadStoreMultipleInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ bool IsUpdating = (MCID.TSFlags & ARMII::IndexModeMask) != 0;
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Skip operand 0 of an instruction with base register update.
+ unsigned OpIdx = 0;
+ if (IsUpdating)
+ ++OpIdx;
+
+ // Set base address operand
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
+
+ // Set addressing mode by modifying bits U(23) and P(24)
+ ARM_AM::AMSubMode Mode = ARM_AM::getLoadStoreMultipleSubMode(MI.getOpcode());
+ Binary |= getAddrModeUPBits(ARM_AM::getAM4SubMode(Mode));
+
+ // Set bit W(21)
+ if (IsUpdating)
+ Binary |= 0x1 << ARMII::W_BitShift;
+
+ // Set registers
+ for (unsigned i = OpIdx+2, e = MI.getNumOperands(); i != e; ++i) {
+ const MachineOperand &MO = MI.getOperand(i);
+ if (!MO.isReg() || MO.isImplicit())
+ break;
+ unsigned RegNum = II->getRegisterInfo().getEncodingValue(MO.getReg());
+ assert(TargetRegisterInfo::isPhysicalRegister(MO.getReg()) &&
+ RegNum < 16);
+ Binary |= 0x1 << RegNum;
+ }
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitMulFrmInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode S bit if MI modifies CPSR.
+ Binary |= getAddrModeSBit(MI, MCID);
+
+ // 32x32->64bit operations have two destination registers. The number
+ // of register definitions will tell us if that's what we're dealing with.
+ unsigned OpIdx = 0;
+ if (MCID.getNumDefs() == 2)
+ Binary |= getMachineOpValue (MI, OpIdx++) << ARMII::RegRdLoShift;
+
+ // Encode Rd
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdHiShift;
+
+ // Encode Rm
+ Binary |= getMachineOpValue(MI, OpIdx++);
+
+ // Encode Rs
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRsShift;
+
+ // Many multiple instructions (e.g. MLA) have three src operands. Encode
+ // it as Rn (for multiply, that's in the same offset as RdLo.
+ if (MCID.getNumOperands() > OpIdx &&
+ !MCID.OpInfo[OpIdx].isPredicate() &&
+ !MCID.OpInfo[OpIdx].isOptionalDef())
+ Binary |= getMachineOpValue(MI, OpIdx) << ARMII::RegRdLoShift;
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitExtendInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ unsigned OpIdx = 0;
+
+ // Encode Rd
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
+
+ const MachineOperand &MO1 = MI.getOperand(OpIdx++);
+ const MachineOperand &MO2 = MI.getOperand(OpIdx);
+ if (MO2.isReg()) {
+ // Two register operand form.
+ // Encode Rn.
+ Binary |= getMachineOpValue(MI, MO1) << ARMII::RegRnShift;
+
+ // Encode Rm.
+ Binary |= getMachineOpValue(MI, MO2);
+ ++OpIdx;
+ } else {
+ Binary |= getMachineOpValue(MI, MO1);
+ }
+
+ // Encode rot imm (0, 8, 16, or 24) if it has a rotate immediate operand.
+ if (MI.getOperand(OpIdx).isImm() &&
+ !MCID.OpInfo[OpIdx].isPredicate() &&
+ !MCID.OpInfo[OpIdx].isOptionalDef())
+ Binary |= (getMachineOpValue(MI, OpIdx) / 8) << ARMII::ExtRotImmShift;
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitMiscArithInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // PKH instructions are finished at this point
+ if (MCID.Opcode == ARM::PKHBT || MCID.Opcode == ARM::PKHTB) {
+ emitWordLE(Binary);
+ return;
+ }
+
+ unsigned OpIdx = 0;
+
+ // Encode Rd
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
+
+ const MachineOperand &MO = MI.getOperand(OpIdx++);
+ if (OpIdx == MCID.getNumOperands() ||
+ MCID.OpInfo[OpIdx].isPredicate() ||
+ MCID.OpInfo[OpIdx].isOptionalDef()) {
+ // Encode Rm and it's done.
+ Binary |= getMachineOpValue(MI, MO);
+ emitWordLE(Binary);
+ return;
+ }
+
+ // Encode Rn.
+ Binary |= getMachineOpValue(MI, MO) << ARMII::RegRnShift;
+
+ // Encode Rm.
+ Binary |= getMachineOpValue(MI, OpIdx++);
+
+ // Encode shift_imm.
+ unsigned ShiftAmt = MI.getOperand(OpIdx).getImm();
+ if (MCID.Opcode == ARM::PKHTB) {
+ assert(ShiftAmt != 0 && "PKHTB shift_imm is 0!");
+ if (ShiftAmt == 32)
+ ShiftAmt = 0;
+ }
+ assert(ShiftAmt < 32 && "shift_imm range is 0 to 31!");
+ Binary |= ShiftAmt << ARMII::ShiftShift;
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitSaturateInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Part of binary is determined by TableGen.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode Rd
+ Binary |= getMachineOpValue(MI, 0) << ARMII::RegRdShift;
+
+ // Encode saturate bit position.
+ unsigned Pos = MI.getOperand(1).getImm();
+ if (MCID.Opcode == ARM::SSAT || MCID.Opcode == ARM::SSAT16)
+ Pos -= 1;
+ assert((Pos < 16 || (Pos < 32 &&
+ MCID.Opcode != ARM::SSAT16 &&
+ MCID.Opcode != ARM::USAT16)) &&
+ "saturate bit position out of range");
+ Binary |= Pos << 16;
+
+ // Encode Rm
+ Binary |= getMachineOpValue(MI, 2);
+
+ // Encode shift_imm.
+ if (MCID.getNumOperands() == 4) {
+ unsigned ShiftOp = MI.getOperand(3).getImm();
+ ARM_AM::ShiftOpc Opc = ARM_AM::getSORegShOp(ShiftOp);
+ if (Opc == ARM_AM::asr)
+ Binary |= (1 << 6);
+ unsigned ShiftAmt = MI.getOperand(3).getImm();
+ if (ShiftAmt == 32 && Opc == ARM_AM::asr)
+ ShiftAmt = 0;
+ assert(ShiftAmt < 32 && "shift_imm range is 0 to 31!");
+ Binary |= ShiftAmt << ARMII::ShiftShift;
+ }
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitBranchInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ if (MCID.Opcode == ARM::TPsoft) {
+ llvm_unreachable("ARM::TPsoft FIXME"); // FIXME
+ }
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Set signed_immed_24 field
+ Binary |= getMachineOpValue(MI, 0);
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitInlineJumpTable(unsigned JTIndex) {
+ // Remember the base address of the inline jump table.
+ uintptr_t JTBase = MCE.getCurrentPCValue();
+ JTI->addJumpTableBaseAddr(JTIndex, JTBase);
+ DEBUG(errs() << " ** Jump Table #" << JTIndex << " @ " << (void*)JTBase
+ << '\n');
+
+ // Now emit the jump table entries.
+ const std::vector<MachineBasicBlock*> &MBBs = (*MJTEs)[JTIndex].MBBs;
+ for (unsigned i = 0, e = MBBs.size(); i != e; ++i) {
+ if (IsPIC)
+ // DestBB address - JT base.
+ emitMachineBasicBlock(MBBs[i], ARM::reloc_arm_pic_jt, JTBase);
+ else
+ // Absolute DestBB address.
+ emitMachineBasicBlock(MBBs[i], ARM::reloc_arm_absolute);
+ emitWordLE(0);
+ }
+}
+
+void ARMCodeEmitter::emitMiscBranchInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Handle jump tables.
+ if (MCID.Opcode == ARM::BR_JTr || MCID.Opcode == ARM::BR_JTadd) {
+ // First emit a ldr pc, [] instruction.
+ emitDataProcessingInstruction(MI, ARM::PC);
+
+ // Then emit the inline jump table.
+ unsigned JTIndex =
+ (MCID.Opcode == ARM::BR_JTr)
+ ? MI.getOperand(1).getIndex() : MI.getOperand(2).getIndex();
+ emitInlineJumpTable(JTIndex);
+ return;
+ } else if (MCID.Opcode == ARM::BR_JTm) {
+ // First emit a ldr pc, [] instruction.
+ emitLoadStoreInstruction(MI, ARM::PC);
+
+ // Then emit the inline jump table.
+ emitInlineJumpTable(MI.getOperand(3).getIndex());
+ return;
+ }
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ if (MCID.Opcode == ARM::BX_RET || MCID.Opcode == ARM::MOVPCLR)
+ // The return register is LR.
+ Binary |= II->getRegisterInfo().getEncodingValue(ARM::LR);
+ else
+ // otherwise, set the return register
+ Binary |= getMachineOpValue(MI, 0);
+
+ emitWordLE(Binary);
+}
+
+unsigned ARMCodeEmitter::encodeVFPRd(const MachineInstr &MI,
+ unsigned OpIdx) const {
+ unsigned RegD = MI.getOperand(OpIdx).getReg();
+ unsigned Binary = 0;
+ bool isSPVFP = ARM::SPRRegClass.contains(RegD);
+ RegD = II->getRegisterInfo().getEncodingValue(RegD);
+ if (!isSPVFP)
+ Binary |= RegD << ARMII::RegRdShift;
+ else {
+ Binary |= ((RegD & 0x1E) >> 1) << ARMII::RegRdShift;
+ Binary |= (RegD & 0x01) << ARMII::D_BitShift;
+ }
+ return Binary;
+}
+
+unsigned ARMCodeEmitter::encodeVFPRn(const MachineInstr &MI,
+ unsigned OpIdx) const {
+ unsigned RegN = MI.getOperand(OpIdx).getReg();
+ unsigned Binary = 0;
+ bool isSPVFP = ARM::SPRRegClass.contains(RegN);
+ RegN = II->getRegisterInfo().getEncodingValue(RegN);
+ if (!isSPVFP)
+ Binary |= RegN << ARMII::RegRnShift;
+ else {
+ Binary |= ((RegN & 0x1E) >> 1) << ARMII::RegRnShift;
+ Binary |= (RegN & 0x01) << ARMII::N_BitShift;
+ }
+ return Binary;
+}
+
+unsigned ARMCodeEmitter::encodeVFPRm(const MachineInstr &MI,
+ unsigned OpIdx) const {
+ unsigned RegM = MI.getOperand(OpIdx).getReg();
+ unsigned Binary = 0;
+ bool isSPVFP = ARM::SPRRegClass.contains(RegM);
+ RegM = II->getRegisterInfo().getEncodingValue(RegM);
+ if (!isSPVFP)
+ Binary |= RegM;
+ else {
+ Binary |= ((RegM & 0x1E) >> 1);
+ Binary |= (RegM & 0x01) << ARMII::M_BitShift;
+ }
+ return Binary;
+}
+
+void ARMCodeEmitter::emitVFPArithInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ unsigned OpIdx = 0;
+ assert((Binary & ARMII::D_BitShift) == 0 &&
+ (Binary & ARMII::N_BitShift) == 0 &&
+ (Binary & ARMII::M_BitShift) == 0 && "VFP encoding bug!");
+
+ // Encode Dd / Sd.
+ Binary |= encodeVFPRd(MI, OpIdx++);
+
+ // If this is a two-address operand, skip it, e.g. FMACD.
+ if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
+ ++OpIdx;
+
+ // Encode Dn / Sn.
+ if ((MCID.TSFlags & ARMII::FormMask) == ARMII::VFPBinaryFrm)
+ Binary |= encodeVFPRn(MI, OpIdx++);
+
+ if (OpIdx == MCID.getNumOperands() ||
+ MCID.OpInfo[OpIdx].isPredicate() ||
+ MCID.OpInfo[OpIdx].isOptionalDef()) {
+ // FCMPEZD etc. has only one operand.
+ emitWordLE(Binary);
+ return;
+ }
+
+ // Encode Dm / Sm.
+ Binary |= encodeVFPRm(MI, OpIdx);
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitVFPConversionInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ unsigned Form = MCID.TSFlags & ARMII::FormMask;
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ switch (Form) {
+ default: break;
+ case ARMII::VFPConv1Frm:
+ case ARMII::VFPConv2Frm:
+ case ARMII::VFPConv3Frm:
+ // Encode Dd / Sd.
+ Binary |= encodeVFPRd(MI, 0);
+ break;
+ case ARMII::VFPConv4Frm:
+ // Encode Dn / Sn.
+ Binary |= encodeVFPRn(MI, 0);
+ break;
+ case ARMII::VFPConv5Frm:
+ // Encode Dm / Sm.
+ Binary |= encodeVFPRm(MI, 0);
+ break;
+ }
+
+ switch (Form) {
+ default: break;
+ case ARMII::VFPConv1Frm:
+ // Encode Dm / Sm.
+ Binary |= encodeVFPRm(MI, 1);
+ break;
+ case ARMII::VFPConv2Frm:
+ case ARMII::VFPConv3Frm:
+ // Encode Dn / Sn.
+ Binary |= encodeVFPRn(MI, 1);
+ break;
+ case ARMII::VFPConv4Frm:
+ case ARMII::VFPConv5Frm:
+ // Encode Dd / Sd.
+ Binary |= encodeVFPRd(MI, 1);
+ break;
+ }
+
+ if (Form == ARMII::VFPConv5Frm)
+ // Encode Dn / Sn.
+ Binary |= encodeVFPRn(MI, 2);
+ else if (Form == ARMII::VFPConv3Frm)
+ // Encode Dm / Sm.
+ Binary |= encodeVFPRm(MI, 2);
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitVFPLoadStoreInstruction(const MachineInstr &MI) {
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ unsigned OpIdx = 0;
+
+ // Encode Dd / Sd.
+ Binary |= encodeVFPRd(MI, OpIdx++);
+
+ // Encode address base.
+ const MachineOperand &Base = MI.getOperand(OpIdx++);
+ Binary |= getMachineOpValue(MI, Base) << ARMII::RegRnShift;
+
+ // If there is a non-zero immediate offset, encode it.
+ if (Base.isReg()) {
+ const MachineOperand &Offset = MI.getOperand(OpIdx);
+ if (unsigned ImmOffs = ARM_AM::getAM5Offset(Offset.getImm())) {
+ if (ARM_AM::getAM5Op(Offset.getImm()) == ARM_AM::add)
+ Binary |= 1 << ARMII::U_BitShift;
+ Binary |= ImmOffs;
+ emitWordLE(Binary);
+ return;
+ }
+ }
+
+ // If immediate offset is omitted, default to +0.
+ Binary |= 1 << ARMII::U_BitShift;
+
+ emitWordLE(Binary);
+}
+
+void
+ARMCodeEmitter::emitVFPLoadStoreMultipleInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ bool IsUpdating = (MCID.TSFlags & ARMII::IndexModeMask) != 0;
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Skip operand 0 of an instruction with base register update.
+ unsigned OpIdx = 0;
+ if (IsUpdating)
+ ++OpIdx;
+
+ // Set base address operand
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
+
+ // Set addressing mode by modifying bits U(23) and P(24)
+ ARM_AM::AMSubMode Mode = ARM_AM::getLoadStoreMultipleSubMode(MI.getOpcode());
+ Binary |= getAddrModeUPBits(ARM_AM::getAM4SubMode(Mode));
+
+ // Set bit W(21)
+ if (IsUpdating)
+ Binary |= 0x1 << ARMII::W_BitShift;
+
+ // First register is encoded in Dd.
+ Binary |= encodeVFPRd(MI, OpIdx+2);
+
+ // Count the number of registers.
+ unsigned NumRegs = 1;
+ for (unsigned i = OpIdx+3, e = MI.getNumOperands(); i != e; ++i) {
+ const MachineOperand &MO = MI.getOperand(i);
+ if (!MO.isReg() || MO.isImplicit())
+ break;
+ ++NumRegs;
+ }
+ // Bit 8 will be set if <list> is consecutive 64-bit registers (e.g., D0)
+ // Otherwise, it will be 0, in the case of 32-bit registers.
+ if(Binary & 0x100)
+ Binary |= NumRegs * 2;
+ else
+ Binary |= NumRegs;
+
+ emitWordLE(Binary);
+}
+
+unsigned ARMCodeEmitter::encodeNEONRd(const MachineInstr &MI,
+ unsigned OpIdx) const {
+ unsigned RegD = MI.getOperand(OpIdx).getReg();
+ unsigned Binary = 0;
+ RegD = II->getRegisterInfo().getEncodingValue(RegD);
+ Binary |= (RegD & 0xf) << ARMII::RegRdShift;
+ Binary |= ((RegD >> 4) & 1) << ARMII::D_BitShift;
+ return Binary;
+}
+
+unsigned ARMCodeEmitter::encodeNEONRn(const MachineInstr &MI,
+ unsigned OpIdx) const {
+ unsigned RegN = MI.getOperand(OpIdx).getReg();
+ unsigned Binary = 0;
+ RegN = II->getRegisterInfo().getEncodingValue(RegN);
+ Binary |= (RegN & 0xf) << ARMII::RegRnShift;
+ Binary |= ((RegN >> 4) & 1) << ARMII::N_BitShift;
+ return Binary;
+}
+
+unsigned ARMCodeEmitter::encodeNEONRm(const MachineInstr &MI,
+ unsigned OpIdx) const {
+ unsigned RegM = MI.getOperand(OpIdx).getReg();
+ unsigned Binary = 0;
+ RegM = II->getRegisterInfo().getEncodingValue(RegM);
+ Binary |= (RegM & 0xf);
+ Binary |= ((RegM >> 4) & 1) << ARMII::M_BitShift;
+ return Binary;
+}
+
+/// convertNEONDataProcToThumb - Convert the ARM mode encoding for a NEON
+/// data-processing instruction to the corresponding Thumb encoding.
+static unsigned convertNEONDataProcToThumb(unsigned Binary) {
+ assert((Binary & 0xfe000000) == 0xf2000000 &&
+ "not an ARM NEON data-processing instruction");
+ unsigned UBit = (Binary >> 24) & 1;
+ return 0xef000000 | (UBit << 28) | (Binary & 0xffffff);
+}
+
+void ARMCodeEmitter::emitNEONLaneInstruction(const MachineInstr &MI) {
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ unsigned RegTOpIdx, RegNOpIdx, LnOpIdx;
+ const MCInstrDesc &MCID = MI.getDesc();
+ if ((MCID.TSFlags & ARMII::FormMask) == ARMII::NGetLnFrm) {
+ RegTOpIdx = 0;
+ RegNOpIdx = 1;
+ LnOpIdx = 2;
+ } else { // ARMII::NSetLnFrm
+ RegTOpIdx = 2;
+ RegNOpIdx = 0;
+ LnOpIdx = 3;
+ }
+
+ // Set the conditional execution predicate
+ Binary |= (IsThumb ? ARMCC::AL : II->getPredicate(&MI)) << ARMII::CondShift;
+
+ unsigned RegT = MI.getOperand(RegTOpIdx).getReg();
+ RegT = II->getRegisterInfo().getEncodingValue(RegT);
+ Binary |= (RegT << ARMII::RegRdShift);
+ Binary |= encodeNEONRn(MI, RegNOpIdx);
+
+ unsigned LaneShift;
+ if ((Binary & (1 << 22)) != 0)
+ LaneShift = 0; // 8-bit elements
+ else if ((Binary & (1 << 5)) != 0)
+ LaneShift = 1; // 16-bit elements
+ else
+ LaneShift = 2; // 32-bit elements
+
+ unsigned Lane = MI.getOperand(LnOpIdx).getImm() << LaneShift;
+ unsigned Opc1 = Lane >> 2;
+ unsigned Opc2 = Lane & 3;
+ assert((Opc1 & 3) == 0 && "out-of-range lane number operand");
+ Binary |= (Opc1 << 21);
+ Binary |= (Opc2 << 5);
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitNEONDupInstruction(const MachineInstr &MI) {
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= (IsThumb ? ARMCC::AL : II->getPredicate(&MI)) << ARMII::CondShift;
+
+ unsigned RegT = MI.getOperand(1).getReg();
+ RegT = II->getRegisterInfo().getEncodingValue(RegT);
+ Binary |= (RegT << ARMII::RegRdShift);
+ Binary |= encodeNEONRn(MI, 0);
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitNEON1RegModImmInstruction(const MachineInstr &MI) {
+ unsigned Binary = getBinaryCodeForInstr(MI);
+ // Destination register is encoded in Dd.
+ Binary |= encodeNEONRd(MI, 0);
+ // Immediate fields: Op, Cmode, I, Imm3, Imm4
+ unsigned Imm = MI.getOperand(1).getImm();
+ unsigned Op = (Imm >> 12) & 1;
+ unsigned Cmode = (Imm >> 8) & 0xf;
+ unsigned I = (Imm >> 7) & 1;
+ unsigned Imm3 = (Imm >> 4) & 0x7;
+ unsigned Imm4 = Imm & 0xf;
+ Binary |= (I << 24) | (Imm3 << 16) | (Cmode << 8) | (Op << 5) | Imm4;
+ if (IsThumb)
+ Binary = convertNEONDataProcToThumb(Binary);
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitNEON2RegInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ unsigned Binary = getBinaryCodeForInstr(MI);
+ // Destination register is encoded in Dd; source register in Dm.
+ unsigned OpIdx = 0;
+ Binary |= encodeNEONRd(MI, OpIdx++);
+ if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
+ ++OpIdx;
+ Binary |= encodeNEONRm(MI, OpIdx);
+ if (IsThumb)
+ Binary = convertNEONDataProcToThumb(Binary);
+ // FIXME: This does not handle VDUPfdf or VDUPfqf.
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitNEON3RegInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ unsigned Binary = getBinaryCodeForInstr(MI);
+ // Destination register is encoded in Dd; source registers in Dn and Dm.
+ unsigned OpIdx = 0;
+ Binary |= encodeNEONRd(MI, OpIdx++);
+ if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
+ ++OpIdx;
+ Binary |= encodeNEONRn(MI, OpIdx++);
+ if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
+ ++OpIdx;
+ Binary |= encodeNEONRm(MI, OpIdx);
+ if (IsThumb)
+ Binary = convertNEONDataProcToThumb(Binary);
+ // FIXME: This does not handle VMOVDneon or VMOVQ.
+ emitWordLE(Binary);
+}
+
+#include "ARMGenCodeEmitter.inc"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SelectionDAG.h"
--- /dev/null
+//===-- ARMJITInfo.cpp - Implement the JIT interfaces for the ARM target --===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the JIT interfaces for the ARM target.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ARMJITInfo.h"
+#include "ARMConstantPoolValue.h"
+#include "ARMMachineFunctionInfo.h"
+#include "ARMRelocations.h"
+#include "MCTargetDesc/ARMBaseInfo.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/IR/Function.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Memory.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cstdlib>
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+void ARMJITInfo::replaceMachineCodeForFunction(void *Old, void *New) {
+ report_fatal_error("ARMJITInfo::replaceMachineCodeForFunction");
+}
+
+/// JITCompilerFunction - This contains the address of the JIT function used to
+/// compile a function lazily.
+static TargetJITInfo::JITCompilerFn JITCompilerFunction;
+
+// Get the ASMPREFIX for the current host. This is often '_'.
+#ifndef __USER_LABEL_PREFIX__
+#define __USER_LABEL_PREFIX__
+#endif
+#define GETASMPREFIX2(X) #X
+#define GETASMPREFIX(X) GETASMPREFIX2(X)
+#define ASMPREFIX GETASMPREFIX(__USER_LABEL_PREFIX__)
+
+// CompilationCallback stub - We can't use a C function with inline assembly in
+// it, because the prolog/epilog inserted by GCC won't work for us. (We need
+// to preserve more context and manipulate the stack directly). Instead,
+// write our own wrapper, which does things our way, so we have complete
+// control over register saving and restoring.
+extern "C" {
+#if defined(__arm__)
+ void ARMCompilationCallback();
+ asm(
+ ".text\n"
+ ".align 2\n"
+ ".globl " ASMPREFIX "ARMCompilationCallback\n"
+ ASMPREFIX "ARMCompilationCallback:\n"
+ // Save caller saved registers since they may contain stuff
+ // for the real target function right now. We have to act as if this
+ // whole compilation callback doesn't exist as far as the caller is
+ // concerned, so we can't just preserve the callee saved regs.
+ "stmdb sp!, {r0, r1, r2, r3, lr}\n"
+#if (defined(__VFP_FP__) && !defined(__SOFTFP__))
+ "vstmdb sp!, {d0, d1, d2, d3, d4, d5, d6, d7}\n"
+#endif
+ // The LR contains the address of the stub function on entry.
+ // pass it as the argument to the C part of the callback
+ "mov r0, lr\n"
+ "sub sp, sp, #4\n"
+ // Call the C portion of the callback
+ "bl " ASMPREFIX "ARMCompilationCallbackC\n"
+ "add sp, sp, #4\n"
+ // Restoring the LR to the return address of the function that invoked
+ // the stub and de-allocating the stack space for it requires us to
+ // swap the two saved LR values on the stack, as they're backwards
+ // for what we need since the pop instruction has a pre-determined
+ // order for the registers.
+ // +--------+
+ // 0 | LR | Original return address
+ // +--------+
+ // 1 | LR | Stub address (start of stub)
+ // 2-5 | R3..R0 | Saved registers (we need to preserve all regs)
+ // 6-20 | D0..D7 | Saved VFP registers
+ // +--------+
+ //
+#if (defined(__VFP_FP__) && !defined(__SOFTFP__))
+ // Restore VFP caller-saved registers.
+ "vldmia sp!, {d0, d1, d2, d3, d4, d5, d6, d7}\n"
+#endif
+ //
+ // We need to exchange the values in slots 0 and 1 so we can
+ // return to the address in slot 1 with the address in slot 0
+ // restored to the LR.
+ "ldr r0, [sp,#20]\n"
+ "ldr r1, [sp,#16]\n"
+ "str r1, [sp,#20]\n"
+ "str r0, [sp,#16]\n"
+ // Return to the (newly modified) stub to invoke the real function.
+ // The above twiddling of the saved return addresses allows us to
+ // deallocate everything, including the LR the stub saved, with two
+ // updating load instructions.
+ "ldmia sp!, {r0, r1, r2, r3, lr}\n"
+ "ldr pc, [sp], #4\n"
+ );
+#else // Not an ARM host
+ void ARMCompilationCallback() {
+ llvm_unreachable("Cannot call ARMCompilationCallback() on a non-ARM arch!");
+ }
+#endif
+}
+
+/// ARMCompilationCallbackC - This is the target-specific function invoked
+/// by the function stub when we did not know the real target of a call.
+/// This function must locate the start of the stub or call site and pass
+/// it into the JIT compiler function.
+extern "C" void ARMCompilationCallbackC(intptr_t StubAddr) {
+ // Get the address of the compiled code for this function.
+ intptr_t NewVal = (intptr_t)JITCompilerFunction((void*)StubAddr);
+
+ // Rewrite the call target... so that we don't end up here every time we
+ // execute the call. We're replacing the first two instructions of the
+ // stub with:
+ // ldr pc, [pc,#-4]
+ // <addr>
+ if (!sys::Memory::setRangeWritable((void*)StubAddr, 8)) {
+ llvm_unreachable("ERROR: Unable to mark stub writable");
+ }
+ *(intptr_t *)StubAddr = 0xe51ff004; // ldr pc, [pc, #-4]
+ *(intptr_t *)(StubAddr+4) = NewVal;
+ if (!sys::Memory::setRangeExecutable((void*)StubAddr, 8)) {
+ llvm_unreachable("ERROR: Unable to mark stub executable");
+ }
+}
+
+TargetJITInfo::LazyResolverFn
+ARMJITInfo::getLazyResolverFunction(JITCompilerFn F) {
+ JITCompilerFunction = F;
+ return ARMCompilationCallback;
+}
+
+void *ARMJITInfo::emitGlobalValueIndirectSym(const GlobalValue *GV, void *Ptr,
+ JITCodeEmitter &JCE) {
+ uint8_t Buffer[4];
+ uint8_t *Cur = Buffer;
+ MachineCodeEmitter::emitWordLEInto(Cur, (intptr_t)Ptr);
+ void *PtrAddr = JCE.allocIndirectGV(
+ GV, Buffer, sizeof(Buffer), /*Alignment=*/4);
+ addIndirectSymAddr(Ptr, (intptr_t)PtrAddr);
+ return PtrAddr;
+}
+
+TargetJITInfo::StubLayout ARMJITInfo::getStubLayout() {
+ // The stub contains up to 3 4-byte instructions, aligned at 4 bytes, and a
+ // 4-byte address. See emitFunctionStub for details.
+ StubLayout Result = {16, 4};
+ return Result;
+}
+
+void *ARMJITInfo::emitFunctionStub(const Function* F, void *Fn,
+ JITCodeEmitter &JCE) {
+ void *Addr;
+ // If this is just a call to an external function, emit a branch instead of a
+ // call. The code is the same except for one bit of the last instruction.
+ if (Fn != (void*)(intptr_t)ARMCompilationCallback) {
+ // Branch to the corresponding function addr.
+ if (IsPIC) {
+ // The stub is 16-byte size and 4-aligned.
+ intptr_t LazyPtr = getIndirectSymAddr(Fn);
+ if (!LazyPtr) {
+ // In PIC mode, the function stub is loading a lazy-ptr.
+ LazyPtr= (intptr_t)emitGlobalValueIndirectSym((const GlobalValue*)F, Fn, JCE);
+ DEBUG(if (F)
+ errs() << "JIT: Indirect symbol emitted at [" << LazyPtr
+ << "] for GV '" << F->getName() << "'\n";
+ else
+ errs() << "JIT: Stub emitted at [" << LazyPtr
+ << "] for external function at '" << Fn << "'\n");
+ }
+ JCE.emitAlignment(4);
+ Addr = (void*)JCE.getCurrentPCValue();
+ if (!sys::Memory::setRangeWritable(Addr, 16)) {
+ llvm_unreachable("ERROR: Unable to mark stub writable");
+ }
+ JCE.emitWordLE(0xe59fc004); // ldr ip, [pc, #+4]
+ JCE.emitWordLE(0xe08fc00c); // L_func$scv: add ip, pc, ip
+ JCE.emitWordLE(0xe59cf000); // ldr pc, [ip]
+ JCE.emitWordLE(LazyPtr - (intptr_t(Addr)+4+8)); // func - (L_func$scv+8)
+ sys::Memory::InvalidateInstructionCache(Addr, 16);
+ if (!sys::Memory::setRangeExecutable(Addr, 16)) {
+ llvm_unreachable("ERROR: Unable to mark stub executable");
+ }
+ } else {
+ // The stub is 8-byte size and 4-aligned.
+ JCE.emitAlignment(4);
+ Addr = (void*)JCE.getCurrentPCValue();
+ if (!sys::Memory::setRangeWritable(Addr, 8)) {
+ llvm_unreachable("ERROR: Unable to mark stub writable");
+ }
+ JCE.emitWordLE(0xe51ff004); // ldr pc, [pc, #-4]
+ JCE.emitWordLE((intptr_t)Fn); // addr of function
+ sys::Memory::InvalidateInstructionCache(Addr, 8);
+ if (!sys::Memory::setRangeExecutable(Addr, 8)) {
+ llvm_unreachable("ERROR: Unable to mark stub executable");
+ }
+ }
+ } else {
+ // The compilation callback will overwrite the first two words of this
+ // stub with indirect branch instructions targeting the compiled code.
+ // This stub sets the return address to restart the stub, so that
+ // the new branch will be invoked when we come back.
+ //
+ // Branch and link to the compilation callback.
+ // The stub is 16-byte size and 4-byte aligned.
+ JCE.emitAlignment(4);
+ Addr = (void*)JCE.getCurrentPCValue();
+ if (!sys::Memory::setRangeWritable(Addr, 16)) {
+ llvm_unreachable("ERROR: Unable to mark stub writable");
+ }
+ // Save LR so the callback can determine which stub called it.
+ // The compilation callback is responsible for popping this prior
+ // to returning.
+ JCE.emitWordLE(0xe92d4000); // push {lr}
+ // Set the return address to go back to the start of this stub.
+ JCE.emitWordLE(0xe24fe00c); // sub lr, pc, #12
+ // Invoke the compilation callback.
+ JCE.emitWordLE(0xe51ff004); // ldr pc, [pc, #-4]
+ // The address of the compilation callback.
+ JCE.emitWordLE((intptr_t)ARMCompilationCallback);
+ sys::Memory::InvalidateInstructionCache(Addr, 16);
+ if (!sys::Memory::setRangeExecutable(Addr, 16)) {
+ llvm_unreachable("ERROR: Unable to mark stub executable");
+ }
+ }
+
+ return Addr;
+}
+
+intptr_t ARMJITInfo::resolveRelocDestAddr(MachineRelocation *MR) const {
+ ARM::RelocationType RT = (ARM::RelocationType)MR->getRelocationType();
+ switch (RT) {
+ default:
+ return (intptr_t)(MR->getResultPointer());
+ case ARM::reloc_arm_pic_jt:
+ // Destination address - jump table base.
+ return (intptr_t)(MR->getResultPointer()) - MR->getConstantVal();
+ case ARM::reloc_arm_jt_base:
+ // Jump table base address.
+ return getJumpTableBaseAddr(MR->getJumpTableIndex());
+ case ARM::reloc_arm_cp_entry:
+ case ARM::reloc_arm_vfp_cp_entry:
+ // Constant pool entry address.
+ return getConstantPoolEntryAddr(MR->getConstantPoolIndex());
+ case ARM::reloc_arm_machine_cp_entry: {
+ ARMConstantPoolValue *ACPV = (ARMConstantPoolValue*)MR->getConstantVal();
+ assert((!ACPV->hasModifier() && !ACPV->mustAddCurrentAddress()) &&
+ "Can't handle this machine constant pool entry yet!");
+ intptr_t Addr = (intptr_t)(MR->getResultPointer());
+ Addr -= getPCLabelAddr(ACPV->getLabelId()) + ACPV->getPCAdjustment();
+ return Addr;
+ }
+ }
+}
+
+/// relocate - Before the JIT can run a block of code that has been emitted,
+/// it must rewrite the code to contain the actual addresses of any
+/// referenced global symbols.
+void ARMJITInfo::relocate(void *Function, MachineRelocation *MR,
+ unsigned NumRelocs, unsigned char* GOTBase) {
+ for (unsigned i = 0; i != NumRelocs; ++i, ++MR) {
+ void *RelocPos = (char*)Function + MR->getMachineCodeOffset();
+ intptr_t ResultPtr = resolveRelocDestAddr(MR);
+ switch ((ARM::RelocationType)MR->getRelocationType()) {
+ case ARM::reloc_arm_cp_entry:
+ case ARM::reloc_arm_vfp_cp_entry:
+ case ARM::reloc_arm_relative: {
+ // It is necessary to calculate the correct PC relative value. We
+ // subtract the base addr from the target addr to form a byte offset.
+ ResultPtr = ResultPtr - (intptr_t)RelocPos - 8;
+ // If the result is positive, set bit U(23) to 1.
+ if (ResultPtr >= 0)
+ *((intptr_t*)RelocPos) |= 1 << ARMII::U_BitShift;
+ else {
+ // Otherwise, obtain the absolute value and set bit U(23) to 0.
+ *((intptr_t*)RelocPos) &= ~(1 << ARMII::U_BitShift);
+ ResultPtr = - ResultPtr;
+ }
+ // Set the immed value calculated.
+ // VFP immediate offset is multiplied by 4.
+ if (MR->getRelocationType() == ARM::reloc_arm_vfp_cp_entry)
+ ResultPtr = ResultPtr >> 2;
+ *((intptr_t*)RelocPos) |= ResultPtr;
+ // Set register Rn to PC (which is register 15 on all architectures).
+ // FIXME: This avoids the need for register info in the JIT class.
+ *((intptr_t*)RelocPos) |= 15 << ARMII::RegRnShift;
+ break;
+ }
+ case ARM::reloc_arm_pic_jt:
+ case ARM::reloc_arm_machine_cp_entry:
+ case ARM::reloc_arm_absolute: {
+ // These addresses have already been resolved.
+ *((intptr_t*)RelocPos) |= (intptr_t)ResultPtr;
+ break;
+ }
+ case ARM::reloc_arm_branch: {
+ // It is necessary to calculate the correct value of signed_immed_24
+ // field. We subtract the base addr from the target addr to form a
+ // byte offset, which must be inside the range -33554432 and +33554428.
+ // Then, we set the signed_immed_24 field of the instruction to bits
+ // [25:2] of the byte offset. More details ARM-ARM p. A4-11.
+ ResultPtr = ResultPtr - (intptr_t)RelocPos - 8;
+ ResultPtr = (ResultPtr & 0x03FFFFFC) >> 2;
+ assert(ResultPtr >= -33554432 && ResultPtr <= 33554428);
+ *((intptr_t*)RelocPos) |= ResultPtr;
+ break;
+ }
+ case ARM::reloc_arm_jt_base: {
+ // JT base - (instruction addr + 8)
+ ResultPtr = ResultPtr - (intptr_t)RelocPos - 8;
+ *((intptr_t*)RelocPos) |= ResultPtr;
+ break;
+ }
+ case ARM::reloc_arm_movw: {
+ ResultPtr = ResultPtr & 0xFFFF;
+ *((intptr_t*)RelocPos) |= ResultPtr & 0xFFF;
+ *((intptr_t*)RelocPos) |= ((ResultPtr >> 12) & 0xF) << 16;
+ break;
+ }
+ case ARM::reloc_arm_movt: {
+ ResultPtr = (ResultPtr >> 16) & 0xFFFF;
+ *((intptr_t*)RelocPos) |= ResultPtr & 0xFFF;
+ *((intptr_t*)RelocPos) |= ((ResultPtr >> 12) & 0xF) << 16;
+ break;
+ }
+ }
+ }
+}
+
+void ARMJITInfo::Initialize(const MachineFunction &MF, bool isPIC) {
+ const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+ ConstPoolId2AddrMap.resize(AFI->getNumPICLabels());
+ JumpTableId2AddrMap.resize(AFI->getNumJumpTables());
+ IsPIC = isPIC;
+}
--- /dev/null
+//===-- ARMJITInfo.h - ARM implementation of the JIT interface -*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the declaration of the ARMJITInfo class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ARMJITINFO_H
+#define ARMJITINFO_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/Target/TargetJITInfo.h"
+
+namespace llvm {
+ class ARMTargetMachine;
+
+ class ARMJITInfo : public TargetJITInfo {
+ // ConstPoolId2AddrMap - A map from constant pool ids to the corresponding
+ // CONSTPOOL_ENTRY addresses.
+ SmallVector<intptr_t, 16> ConstPoolId2AddrMap;
+
+ // JumpTableId2AddrMap - A map from inline jumptable ids to the
+ // corresponding inline jump table bases.
+ SmallVector<intptr_t, 16> JumpTableId2AddrMap;
+
+ // PCLabelMap - A map from PC labels to addresses.
+ DenseMap<unsigned, intptr_t> PCLabelMap;
+
+ // Sym2IndirectSymMap - A map from symbol (GlobalValue and ExternalSymbol)
+ // addresses to their indirect symbol addresses.
+ DenseMap<void*, intptr_t> Sym2IndirectSymMap;
+
+ // IsPIC - True if the relocation model is PIC. This is used to determine
+ // how to codegen function stubs.
+ bool IsPIC;
+
+ public:
+ explicit ARMJITInfo() : IsPIC(false) { useGOT = false; }
+
+ /// replaceMachineCodeForFunction - Make it so that calling the function
+ /// whose machine code is at OLD turns into a call to NEW, perhaps by
+ /// overwriting OLD with a branch to NEW. This is used for self-modifying
+ /// code.
+ ///
+ void replaceMachineCodeForFunction(void *Old, void *New) override;
+
+ /// emitGlobalValueIndirectSym - Use the specified JITCodeEmitter object
+ /// to emit an indirect symbol which contains the address of the specified
+ /// ptr.
+ void *emitGlobalValueIndirectSym(const GlobalValue* GV, void *ptr,
+ JITCodeEmitter &JCE) override;
+
+ // getStubLayout - Returns the size and alignment of the largest call stub
+ // on ARM.
+ StubLayout getStubLayout() override;
+
+ /// emitFunctionStub - Use the specified JITCodeEmitter object to emit a
+ /// small native function that simply calls the function at the specified
+ /// address.
+ void *emitFunctionStub(const Function* F, void *Fn,
+ JITCodeEmitter &JCE) override;
+
+ /// getLazyResolverFunction - Expose the lazy resolver to the JIT.
+ LazyResolverFn getLazyResolverFunction(JITCompilerFn) override;
+
+ /// relocate - Before the JIT can run a block of code that has been emitted,
+ /// it must rewrite the code to contain the actual addresses of any
+ /// referenced global symbols.
+ void relocate(void *Function, MachineRelocation *MR,
+ unsigned NumRelocs, unsigned char* GOTBase) override;
+
+ /// hasCustomConstantPool - Allows a target to specify that constant
+ /// pool address resolution is handled by the target.
+ bool hasCustomConstantPool() const override { return true; }
+
+ /// hasCustomJumpTables - Allows a target to specify that jumptables
+ /// are emitted by the target.
+ bool hasCustomJumpTables() const override { return true; }
+
+ /// allocateSeparateGVMemory - If true, globals should be placed in
+ /// separately allocated heap memory rather than in the same
+ /// code memory allocated by JITCodeEmitter.
+ bool allocateSeparateGVMemory() const override {
+#ifdef __APPLE__
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ /// Initialize - Initialize internal stage for the function being JITted.
+ /// Resize constant pool ids to CONSTPOOL_ENTRY addresses map; resize
+ /// jump table ids to jump table bases map; remember if codegen relocation
+ /// model is PIC.
+ void Initialize(const MachineFunction &MF, bool isPIC);
+
+ /// getConstantPoolEntryAddr - The ARM target puts all constant
+ /// pool entries into constant islands. This returns the address of the
+ /// constant pool entry of the specified index.
+ intptr_t getConstantPoolEntryAddr(unsigned CPI) const {
+ assert(CPI < ConstPoolId2AddrMap.size());
+ return ConstPoolId2AddrMap[CPI];
+ }
+
+ /// addConstantPoolEntryAddr - Map a Constant Pool Index to the address
+ /// where its associated value is stored. When relocations are processed,
+ /// this value will be used to resolve references to the constant.
+ void addConstantPoolEntryAddr(unsigned CPI, intptr_t Addr) {
+ assert(CPI < ConstPoolId2AddrMap.size());
+ ConstPoolId2AddrMap[CPI] = Addr;
+ }
+
+ /// getJumpTableBaseAddr - The ARM target inline all jump tables within
+ /// text section of the function. This returns the address of the base of
+ /// the jump table of the specified index.
+ intptr_t getJumpTableBaseAddr(unsigned JTI) const {
+ assert(JTI < JumpTableId2AddrMap.size());
+ return JumpTableId2AddrMap[JTI];
+ }
+
+ /// addJumpTableBaseAddr - Map a jump table index to the address where
+ /// the corresponding inline jump table is emitted. When relocations are
+ /// processed, this value will be used to resolve references to the
+ /// jump table.
+ void addJumpTableBaseAddr(unsigned JTI, intptr_t Addr) {
+ assert(JTI < JumpTableId2AddrMap.size());
+ JumpTableId2AddrMap[JTI] = Addr;
+ }
+
+ /// getPCLabelAddr - Retrieve the address of the PC label of the
+ /// specified id.
+ intptr_t getPCLabelAddr(unsigned Id) const {
+ DenseMap<unsigned, intptr_t>::const_iterator I = PCLabelMap.find(Id);
+ assert(I != PCLabelMap.end());
+ return I->second;
+ }
+
+ /// addPCLabelAddr - Remember the address of the specified PC label.
+ void addPCLabelAddr(unsigned Id, intptr_t Addr) {
+ PCLabelMap.insert(std::make_pair(Id, Addr));
+ }
+
+ /// getIndirectSymAddr - Retrieve the address of the indirect symbol of the
+ /// specified symbol located at address. Returns 0 if the indirect symbol
+ /// has not been emitted.
+ intptr_t getIndirectSymAddr(void *Addr) const {
+ DenseMap<void*,intptr_t>::const_iterator I= Sym2IndirectSymMap.find(Addr);
+ if (I != Sym2IndirectSymMap.end())
+ return I->second;
+ return 0;
+ }
+
+ /// addIndirectSymAddr - Add a mapping from address of an emitted symbol to
+ /// its indirect symbol address.
+ void addIndirectSymAddr(void *SymAddr, intptr_t IndSymAddr) {
+ Sym2IndirectSymMap.insert(std::make_pair(SymAddr, IndSymAddr));
+ }
+
+ private:
+ /// resolveRelocDestAddr - Resolve the resulting address of the relocation
+ /// if it's not already solved. Constantpool entries must be resolved by
+ /// ARM target.
+ intptr_t resolveRelocDestAddr(MachineRelocation *MR) const;
+ };
+}
+
+#endif
#include "ARMFrameLowering.h"
#include "ARMISelLowering.h"
#include "ARMInstrInfo.h"
+#include "ARMJITInfo.h"
#include "ARMSelectionDAGInfo.h"
#include "ARMSubtarget.h"
#include "ARMMachineFunctionInfo.h"
ARMProcClass(None), stackAlignment(4), CPUString(CPU), IsLittle(IsLittle),
TargetTriple(TT), Options(Options), TargetABI(ARM_ABI_UNKNOWN),
DL(computeDataLayout(initializeSubtargetDependencies(CPU, FS))),
- TSInfo(DL),
+ TSInfo(DL), JITInfo(),
InstrInfo(isThumb1Only()
? (ARMBaseInstrInfo *)new Thumb1InstrInfo(*this)
: !isThumb()
#include "ARMFrameLowering.h"
#include "ARMISelLowering.h"
#include "ARMInstrInfo.h"
+#include "ARMJITInfo.h"
#include "ARMSelectionDAGInfo.h"
#include "ARMSubtarget.h"
#include "Thumb1FrameLowering.h"
#include "Thumb1InstrInfo.h"
#include "Thumb2InstrInfo.h"
+#include "ARMJITInfo.h"
#include "MCTargetDesc/ARMMCTargetDesc.h"
#include "llvm/ADT/Triple.h"
#include "llvm/IR/DataLayout.h"
const ARMSelectionDAGInfo *getSelectionDAGInfo() const override {
return &TSInfo;
}
+ ARMJITInfo *getJITInfo() override { return &JITInfo; }
const ARMBaseInstrInfo *getInstrInfo() const override {
return InstrInfo.get();
}
private:
const DataLayout DL;
ARMSelectionDAGInfo TSInfo;
+ ARMJITInfo JITInfo;
// Either Thumb1InstrInfo or Thumb2InstrInfo.
std::unique_ptr<ARMBaseInstrInfo> InstrInfo;
ARMTargetLowering TLInfo;
return true;
}
+
+bool ARMBaseTargetMachine::addCodeEmitter(PassManagerBase &PM,
+ JITCodeEmitter &JCE) {
+ // Machine code emitter pass for ARM.
+ PM.add(createARMJITCodeEmitterPass(*this, JCE));
+ return false;
+}
// Pass Pipeline Configuration
TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
+
+ bool addCodeEmitter(PassManagerBase &PM, JITCodeEmitter &MCE) override;
};
/// ARMTargetMachine - ARM target machine.
tablegen(LLVM ARMGenRegisterInfo.inc -gen-register-info)
tablegen(LLVM ARMGenInstrInfo.inc -gen-instr-info)
-tablegen(LLVM ARMGenMCCodeEmitter.inc -gen-emitter)
+tablegen(LLVM ARMGenCodeEmitter.inc -gen-emitter)
+tablegen(LLVM ARMGenMCCodeEmitter.inc -gen-emitter -mc-emitter)
tablegen(LLVM ARMGenMCPseudoLowering.inc -gen-pseudo-lowering)
tablegen(LLVM ARMGenAsmWriter.inc -gen-asm-writer)
tablegen(LLVM ARMGenAsmMatcher.inc -gen-asm-matcher)
ARMAsmPrinter.cpp
ARMBaseInstrInfo.cpp
ARMBaseRegisterInfo.cpp
+ ARMCodeEmitter.cpp
ARMConstantIslandPass.cpp
ARMConstantPoolValue.cpp
ARMExpandPseudoInsts.cpp
ARMISelDAGToDAG.cpp
ARMISelLowering.cpp
ARMInstrInfo.cpp
+ ARMJITInfo.cpp
ARMLoadStoreOptimizer.cpp
ARMMCInstLower.cpp
ARMMachineFunctionInfo.cpp
BUILT_SOURCES = ARMGenRegisterInfo.inc ARMGenInstrInfo.inc \
ARMGenAsmWriter.inc ARMGenAsmMatcher.inc \
ARMGenDAGISel.inc ARMGenSubtargetInfo.inc \
- ARMGenCallingConv.inc \
+ ARMGenCodeEmitter.inc ARMGenCallingConv.inc \
ARMGenFastISel.inc ARMGenMCCodeEmitter.inc \
ARMGenMCPseudoLowering.inc ARMGenDisassemblerTables.inc
add_llvm_library(LLVMTarget
Target.cpp
TargetIntrinsicInfo.cpp
+ TargetJITInfo.cpp
TargetLibraryInfo.cpp
TargetLoweringObjectFile.cpp
TargetMachine.cpp
tablegen(LLVM MipsGenRegisterInfo.inc -gen-register-info)
tablegen(LLVM MipsGenInstrInfo.inc -gen-instr-info)
tablegen(LLVM MipsGenDisassemblerTables.inc -gen-disassembler)
-tablegen(LLVM MipsGenMCCodeEmitter.inc -gen-emitter)
+tablegen(LLVM MipsGenCodeEmitter.inc -gen-emitter)
+tablegen(LLVM MipsGenMCCodeEmitter.inc -gen-emitter -mc-emitter)
tablegen(LLVM MipsGenAsmWriter.inc -gen-asm-writer)
tablegen(LLVM MipsGenDAGISel.inc -gen-dag-isel)
tablegen(LLVM MipsGenFastISel.inc -gen-fast-isel)
Mips16RegisterInfo.cpp
MipsAnalyzeImmediate.cpp
MipsAsmPrinter.cpp
+ MipsCodeEmitter.cpp
MipsConstantIslandPass.cpp
MipsDelaySlotFiller.cpp
MipsFastISel.cpp
+ MipsJITInfo.cpp
MipsInstrInfo.cpp
MipsISelDAGToDAG.cpp
MipsISelLowering.cpp
# Make sure that tblgen is run, first thing.
BUILT_SOURCES = MipsGenRegisterInfo.inc MipsGenInstrInfo.inc \
- MipsGenAsmWriter.inc MipsGenFastISel.inc \
+ MipsGenAsmWriter.inc MipsGenFastISel.inc MipsGenCodeEmitter.inc \
MipsGenDAGISel.inc MipsGenCallingConv.inc \
MipsGenSubtargetInfo.inc MipsGenMCCodeEmitter.inc \
MipsGenDisassemblerTables.inc \
FunctionPass *createMipsOptimizePICCallPass(MipsTargetMachine &TM);
FunctionPass *createMipsDelaySlotFillerPass(MipsTargetMachine &TM);
FunctionPass *createMipsLongBranchPass(MipsTargetMachine &TM);
+ FunctionPass *createMipsJITCodeEmitterPass(MipsTargetMachine &TM,
+ JITCodeEmitter &JCE);
FunctionPass *createMipsConstantIslandPass(MipsTargetMachine &tm);
} // end namespace llvm;
//===----------------------------------------------------------------------===//
#include "Mips16ISelLowering.h"
#include "MCTargetDesc/MipsBaseInfo.h"
-#include "Mips16HardFloatInfo.h"
-#include "MipsMachineFunction.h"
#include "MipsRegisterInfo.h"
#include "MipsTargetMachine.h"
#include "llvm/ADT/StringRef.h"
--- /dev/null
+//===-- Mips/MipsCodeEmitter.cpp - Convert Mips Code to Machine Code ------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file contains the pass that transforms the Mips machine instructions
+// into relocatable machine code.
+//
+//===---------------------------------------------------------------------===//
+
+#include "Mips.h"
+#include "MCTargetDesc/MipsBaseInfo.h"
+#include "MipsInstrInfo.h"
+#include "MipsRelocations.h"
+#include "MipsSubtarget.h"
+#include "MipsTargetMachine.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/PassManager.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#ifndef NDEBUG
+#include <iomanip>
+#endif
+
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+STATISTIC(NumEmitted, "Number of machine instructions emitted");
+
+namespace {
+
+class MipsCodeEmitter : public MachineFunctionPass {
+ MipsJITInfo *JTI;
+ const MipsInstrInfo *II;
+ const DataLayout *TD;
+ const MipsSubtarget *Subtarget;
+ TargetMachine &TM;
+ JITCodeEmitter &MCE;
+ const std::vector<MachineConstantPoolEntry> *MCPEs;
+ const std::vector<MachineJumpTableEntry> *MJTEs;
+ bool IsPIC;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<MachineModuleInfo> ();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ static char ID;
+
+public:
+ MipsCodeEmitter(TargetMachine &tm, JITCodeEmitter &mce)
+ : MachineFunctionPass(ID), JTI(nullptr), II(nullptr), TD(nullptr),
+ TM(tm), MCE(mce), MCPEs(nullptr), MJTEs(nullptr),
+ IsPIC(TM.getRelocationModel() == Reloc::PIC_) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ const char *getPassName() const override {
+ return "Mips Machine Code Emitter";
+ }
+
+ /// getBinaryCodeForInstr - This function, generated by the
+ /// CodeEmitterGenerator using TableGen, produces the binary encoding for
+ /// machine instructions.
+ uint64_t getBinaryCodeForInstr(const MachineInstr &MI) const;
+
+ void emitInstruction(MachineBasicBlock::instr_iterator MI,
+ MachineBasicBlock &MBB);
+
+private:
+
+ void emitWord(unsigned Word);
+
+ /// Routines that handle operands which add machine relocations which are
+ /// fixed up by the relocation stage.
+ void emitGlobalAddress(const GlobalValue *GV, unsigned Reloc,
+ bool MayNeedFarStub) const;
+ void emitExternalSymbolAddress(const char *ES, unsigned Reloc) const;
+ void emitConstPoolAddress(unsigned CPI, unsigned Reloc) const;
+ void emitJumpTableAddress(unsigned JTIndex, unsigned Reloc) const;
+ void emitMachineBasicBlock(MachineBasicBlock *BB, unsigned Reloc) const;
+
+ /// getMachineOpValue - Return binary encoding of operand. If the machine
+ /// operand requires relocation, record the relocation and return zero.
+ unsigned getMachineOpValue(const MachineInstr &MI,
+ const MachineOperand &MO) const;
+
+ unsigned getRelocation(const MachineInstr &MI,
+ const MachineOperand &MO) const;
+
+ unsigned getJumpTargetOpValue(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getJumpTargetOpValueMM(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getBranchTargetOpValueMM(const MachineInstr &MI,
+ unsigned OpNo) const;
+
+ unsigned getBranchTarget21OpValue(const MachineInstr &MI,
+ unsigned OpNo) const;
+ unsigned getBranchTarget26OpValue(const MachineInstr &MI,
+ unsigned OpNo) const;
+ unsigned getJumpOffset16OpValue(const MachineInstr &MI, unsigned OpNo) const;
+
+ unsigned getBranchTargetOpValue(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getMemEncoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getMemEncodingMMImm12(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getMSAMemEncoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getSizeExtEncoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getSizeInsEncoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getLSAImmEncoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getSimm19Lsl2Encoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getSimm18Lsl3Encoding(const MachineInstr &MI, unsigned OpNo) const;
+
+ /// Expand pseudo instructions with accumulator register operands.
+ void expandACCInstr(MachineBasicBlock::instr_iterator MI,
+ MachineBasicBlock &MBB, unsigned Opc) const;
+
+ void expandPseudoIndirectBranch(MachineBasicBlock::instr_iterator MI,
+ MachineBasicBlock &MBB) const;
+
+ /// \brief Expand pseudo instruction. Return true if MI was expanded.
+ bool expandPseudos(MachineBasicBlock::instr_iterator &MI,
+ MachineBasicBlock &MBB) const;
+};
+}
+
+char MipsCodeEmitter::ID = 0;
+
+bool MipsCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
+ MipsTargetMachine &Target = static_cast<MipsTargetMachine &>(
+ const_cast<TargetMachine &>(MF.getTarget()));
+ // Initialize the subtarget so that we can grab the subtarget dependent
+ // variables from it.
+ Subtarget = &TM.getSubtarget<MipsSubtarget>();
+ JTI = Target.getSubtargetImpl()->getJITInfo();
+ II = Subtarget->getInstrInfo();
+ TD = Subtarget->getDataLayout();
+ MCPEs = &MF.getConstantPool()->getConstants();
+ MJTEs = nullptr;
+ if (MF.getJumpTableInfo()) MJTEs = &MF.getJumpTableInfo()->getJumpTables();
+ JTI->Initialize(MF, IsPIC, Subtarget->isLittle());
+ MCE.setModuleInfo(&getAnalysis<MachineModuleInfo> ());
+
+ do {
+ DEBUG(errs() << "JITTing function '"
+ << MF.getName() << "'\n");
+ MCE.startFunction(MF);
+
+ for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
+ MBB != E; ++MBB){
+ MCE.StartMachineBasicBlock(MBB);
+ for (MachineBasicBlock::instr_iterator I = MBB->instr_begin(),
+ E = MBB->instr_end(); I != E;)
+ emitInstruction(*I++, *MBB);
+ }
+ } while (MCE.finishFunction(MF));
+
+ return false;
+}
+
+unsigned MipsCodeEmitter::getRelocation(const MachineInstr &MI,
+ const MachineOperand &MO) const {
+ // NOTE: This relocations are for static.
+ uint64_t TSFlags = MI.getDesc().TSFlags;
+ uint64_t Form = TSFlags & MipsII::FormMask;
+ if (Form == MipsII::FrmJ)
+ return Mips::reloc_mips_26;
+ if ((Form == MipsII::FrmI || Form == MipsII::FrmFI)
+ && MI.isBranch())
+ return Mips::reloc_mips_pc16;
+ if (Form == MipsII::FrmI && MI.getOpcode() == Mips::LUi)
+ return Mips::reloc_mips_hi;
+ return Mips::reloc_mips_lo;
+}
+
+unsigned MipsCodeEmitter::getJumpTargetOpValue(const MachineInstr &MI,
+ unsigned OpNo) const {
+ MachineOperand MO = MI.getOperand(OpNo);
+ if (MO.isGlobal())
+ emitGlobalAddress(MO.getGlobal(), getRelocation(MI, MO), true);
+ else if (MO.isSymbol())
+ emitExternalSymbolAddress(MO.getSymbolName(), getRelocation(MI, MO));
+ else if (MO.isMBB())
+ emitMachineBasicBlock(MO.getMBB(), getRelocation(MI, MO));
+ else
+ llvm_unreachable("Unexpected jump target operand kind.");
+ return 0;
+}
+
+unsigned MipsCodeEmitter::getJumpTargetOpValueMM(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("Unimplemented function.");
+ return 0;
+}
+
+unsigned MipsCodeEmitter::getBranchTargetOpValueMM(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("Unimplemented function.");
+ return 0;
+}
+
+unsigned MipsCodeEmitter::getBranchTarget21OpValue(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("Unimplemented function.");
+ return 0;
+}
+
+unsigned MipsCodeEmitter::getBranchTarget26OpValue(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("Unimplemented function.");
+ return 0;
+}
+
+unsigned MipsCodeEmitter::getJumpOffset16OpValue(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("Unimplemented function.");
+ return 0;
+}
+
+unsigned MipsCodeEmitter::getBranchTargetOpValue(const MachineInstr &MI,
+ unsigned OpNo) const {
+ MachineOperand MO = MI.getOperand(OpNo);
+ emitMachineBasicBlock(MO.getMBB(), getRelocation(MI, MO));
+ return 0;
+}
+
+unsigned MipsCodeEmitter::getMemEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ // Base register is encoded in bits 20-16, offset is encoded in bits 15-0.
+ assert(MI.getOperand(OpNo).isReg());
+ unsigned RegBits = getMachineOpValue(MI, MI.getOperand(OpNo)) << 16;
+ return (getMachineOpValue(MI, MI.getOperand(OpNo+1)) & 0xFFFF) | RegBits;
+}
+
+unsigned MipsCodeEmitter::getMemEncodingMMImm12(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("Unimplemented function.");
+ return 0;
+}
+
+unsigned MipsCodeEmitter::getMSAMemEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("Unimplemented function.");
+ return 0;
+}
+
+unsigned MipsCodeEmitter::getSizeExtEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ // size is encoded as size-1.
+ return getMachineOpValue(MI, MI.getOperand(OpNo)) - 1;
+}
+
+unsigned MipsCodeEmitter::getSizeInsEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ // size is encoded as pos+size-1.
+ return getMachineOpValue(MI, MI.getOperand(OpNo-1)) +
+ getMachineOpValue(MI, MI.getOperand(OpNo)) - 1;
+}
+
+unsigned MipsCodeEmitter::getLSAImmEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("Unimplemented function.");
+ return 0;
+}
+
+unsigned MipsCodeEmitter::getSimm18Lsl3Encoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("Unimplemented function.");
+ return 0;
+}
+
+unsigned MipsCodeEmitter::getSimm19Lsl2Encoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("Unimplemented function.");
+ return 0;
+}
+
+/// getMachineOpValue - Return binary encoding of operand. If the machine
+/// operand requires relocation, record the relocation and return zero.
+unsigned MipsCodeEmitter::getMachineOpValue(const MachineInstr &MI,
+ const MachineOperand &MO) const {
+ if (MO.isReg())
+ return TM.getSubtargetImpl()->getRegisterInfo()->getEncodingValue(
+ MO.getReg());
+ else if (MO.isImm())
+ return static_cast<unsigned>(MO.getImm());
+ else if (MO.isGlobal())
+ emitGlobalAddress(MO.getGlobal(), getRelocation(MI, MO), true);
+ else if (MO.isSymbol())
+ emitExternalSymbolAddress(MO.getSymbolName(), getRelocation(MI, MO));
+ else if (MO.isCPI())
+ emitConstPoolAddress(MO.getIndex(), getRelocation(MI, MO));
+ else if (MO.isJTI())
+ emitJumpTableAddress(MO.getIndex(), getRelocation(MI, MO));
+ else if (MO.isMBB())
+ emitMachineBasicBlock(MO.getMBB(), getRelocation(MI, MO));
+ else
+ llvm_unreachable("Unable to encode MachineOperand!");
+ return 0;
+}
+
+void MipsCodeEmitter::emitGlobalAddress(const GlobalValue *GV, unsigned Reloc,
+ bool MayNeedFarStub) const {
+ MCE.addRelocation(MachineRelocation::getGV(MCE.getCurrentPCOffset(), Reloc,
+ const_cast<GlobalValue *>(GV), 0,
+ MayNeedFarStub));
+}
+
+void MipsCodeEmitter::
+emitExternalSymbolAddress(const char *ES, unsigned Reloc) const {
+ MCE.addRelocation(MachineRelocation::getExtSym(MCE.getCurrentPCOffset(),
+ Reloc, ES, 0, 0));
+}
+
+void MipsCodeEmitter::emitConstPoolAddress(unsigned CPI, unsigned Reloc) const {
+ MCE.addRelocation(MachineRelocation::getConstPool(MCE.getCurrentPCOffset(),
+ Reloc, CPI, 0, false));
+}
+
+void MipsCodeEmitter::
+emitJumpTableAddress(unsigned JTIndex, unsigned Reloc) const {
+ MCE.addRelocation(MachineRelocation::getJumpTable(MCE.getCurrentPCOffset(),
+ Reloc, JTIndex, 0, false));
+}
+
+void MipsCodeEmitter::emitMachineBasicBlock(MachineBasicBlock *BB,
+ unsigned Reloc) const {
+ MCE.addRelocation(MachineRelocation::getBB(MCE.getCurrentPCOffset(),
+ Reloc, BB));
+}
+
+void MipsCodeEmitter::emitInstruction(MachineBasicBlock::instr_iterator MI,
+ MachineBasicBlock &MBB) {
+ DEBUG(errs() << "JIT: " << (void*)MCE.getCurrentPCValue() << ":\t" << *MI);
+
+ // Expand pseudo instruction. Skip if MI was not expanded.
+ if (((MI->getDesc().TSFlags & MipsII::FormMask) == MipsII::Pseudo) &&
+ !expandPseudos(MI, MBB))
+ return;
+
+ MCE.processDebugLoc(MI->getDebugLoc(), true);
+
+ emitWord(getBinaryCodeForInstr(*MI));
+ ++NumEmitted; // Keep track of the # of mi's emitted
+
+ MCE.processDebugLoc(MI->getDebugLoc(), false);
+}
+
+void MipsCodeEmitter::emitWord(unsigned Word) {
+ DEBUG(errs() << " 0x";
+ errs().write_hex(Word) << "\n");
+ if (Subtarget->isLittle())
+ MCE.emitWordLE(Word);
+ else
+ MCE.emitWordBE(Word);
+}
+
+void MipsCodeEmitter::expandACCInstr(MachineBasicBlock::instr_iterator MI,
+ MachineBasicBlock &MBB,
+ unsigned Opc) const {
+ // Expand "pseudomult $ac0, $t0, $t1" to "mult $t0, $t1".
+ BuildMI(MBB, &*MI, MI->getDebugLoc(), II->get(Opc))
+ .addReg(MI->getOperand(1).getReg()).addReg(MI->getOperand(2).getReg());
+}
+
+void MipsCodeEmitter::expandPseudoIndirectBranch(
+ MachineBasicBlock::instr_iterator MI, MachineBasicBlock &MBB) const {
+ // This logic is duplicated from MipsAsmPrinter::emitPseudoIndirectBranch()
+ bool HasLinkReg = false;
+ unsigned Opcode = 0;
+
+ if (Subtarget->hasMips64r6()) {
+ // MIPS64r6 should use (JALR64 ZERO_64, $rs)
+ Opcode = Mips::JALR64;
+ HasLinkReg = true;
+ } else if (Subtarget->hasMips32r6()) {
+ // MIPS32r6 should use (JALR ZERO, $rs)
+ Opcode = Mips::JALR;
+ HasLinkReg = true;
+ } else if (Subtarget->inMicroMipsMode())
+ // microMIPS should use (JR_MM $rs)
+ Opcode = Mips::JR_MM;
+ else {
+ // Everything else should use (JR $rs)
+ Opcode = Mips::JR;
+ }
+
+ auto MIB = BuildMI(MBB, &*MI, MI->getDebugLoc(), II->get(Opcode));
+
+ if (HasLinkReg) {
+ unsigned ZeroReg = Subtarget->isGP64bit() ? Mips::ZERO_64 : Mips::ZERO;
+ MIB.addReg(ZeroReg);
+ }
+
+ MIB.addReg(MI->getOperand(0).getReg());
+}
+
+bool MipsCodeEmitter::expandPseudos(MachineBasicBlock::instr_iterator &MI,
+ MachineBasicBlock &MBB) const {
+ switch (MI->getOpcode()) {
+ default:
+ llvm_unreachable("Unhandled pseudo");
+ return false;
+ case Mips::NOP:
+ BuildMI(MBB, &*MI, MI->getDebugLoc(), II->get(Mips::SLL), Mips::ZERO)
+ .addReg(Mips::ZERO).addImm(0);
+ break;
+ case Mips::B:
+ BuildMI(MBB, &*MI, MI->getDebugLoc(), II->get(Mips::BEQ)).addReg(Mips::ZERO)
+ .addReg(Mips::ZERO).addOperand(MI->getOperand(0));
+ break;
+ case Mips::TRAP:
+ BuildMI(MBB, &*MI, MI->getDebugLoc(), II->get(Mips::BREAK)).addImm(0)
+ .addImm(0);
+ break;
+ case Mips::JALRPseudo:
+ BuildMI(MBB, &*MI, MI->getDebugLoc(), II->get(Mips::JALR), Mips::RA)
+ .addReg(MI->getOperand(0).getReg());
+ break;
+ case Mips::PseudoMULT:
+ expandACCInstr(MI, MBB, Mips::MULT);
+ break;
+ case Mips::PseudoMULTu:
+ expandACCInstr(MI, MBB, Mips::MULTu);
+ break;
+ case Mips::PseudoSDIV:
+ expandACCInstr(MI, MBB, Mips::SDIV);
+ break;
+ case Mips::PseudoUDIV:
+ expandACCInstr(MI, MBB, Mips::UDIV);
+ break;
+ case Mips::PseudoMADD:
+ expandACCInstr(MI, MBB, Mips::MADD);
+ break;
+ case Mips::PseudoMADDU:
+ expandACCInstr(MI, MBB, Mips::MADDU);
+ break;
+ case Mips::PseudoMSUB:
+ expandACCInstr(MI, MBB, Mips::MSUB);
+ break;
+ case Mips::PseudoMSUBU:
+ expandACCInstr(MI, MBB, Mips::MSUBU);
+ break;
+ case Mips::PseudoReturn:
+ case Mips::PseudoReturn64:
+ case Mips::PseudoIndirectBranch:
+ case Mips::PseudoIndirectBranch64:
+ expandPseudoIndirectBranch(MI, MBB);
+ break;
+ case TargetOpcode::CFI_INSTRUCTION:
+ case TargetOpcode::IMPLICIT_DEF:
+ case TargetOpcode::KILL:
+ // Do nothing
+ return false;
+ }
+
+ (MI--)->eraseFromBundle();
+ return true;
+}
+
+/// createMipsJITCodeEmitterPass - Return a pass that emits the collected Mips
+/// code to the specified MCE object.
+FunctionPass *llvm::createMipsJITCodeEmitterPass(MipsTargetMachine &TM,
+ JITCodeEmitter &JCE) {
+ return new MipsCodeEmitter(TM, JCE);
+}
+
+#include "MipsGenCodeEmitter.inc"
#include "MipsTargetMachine.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
-#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SelectionDAGISel.h"
#include "llvm/CodeGen/ValueTypes.h"
--- /dev/null
+//===-- MipsJITInfo.cpp - Implement the Mips JIT Interface ----------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the JIT interfaces for the Mips target.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MipsJITInfo.h"
+#include "MipsInstrInfo.h"
+#include "MipsRelocations.h"
+#include "MipsSubtarget.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/IR/Function.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Memory.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cstdlib>
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+
+void MipsJITInfo::replaceMachineCodeForFunction(void *Old, void *New) {
+ unsigned NewAddr = (intptr_t)New;
+ unsigned OldAddr = (intptr_t)Old;
+ const unsigned NopInstr = 0x0;
+
+ // If the functions are in the same memory segment, insert PC-region branch.
+ if ((NewAddr & 0xF0000000) == ((OldAddr + 4) & 0xF0000000)) {
+ unsigned *OldInstruction = (unsigned *)Old;
+ *OldInstruction = 0x08000000;
+ unsigned JTargetAddr = NewAddr & 0x0FFFFFFC;
+
+ JTargetAddr >>= 2;
+ *OldInstruction |= JTargetAddr;
+
+ // Insert a NOP.
+ OldInstruction++;
+ *OldInstruction = NopInstr;
+
+ sys::Memory::InvalidateInstructionCache(Old, 2 * 4);
+ } else {
+ // We need to clear hint bits from the instruction, in case it is 'jr ra'.
+ const unsigned HintMask = 0xFFFFF83F, ReturnSequence = 0x03e00008;
+ unsigned* CurrentInstr = (unsigned*)Old;
+ unsigned CurrInstrHintClear = (*CurrentInstr) & HintMask;
+ unsigned* NextInstr = CurrentInstr + 1;
+ unsigned NextInstrHintClear = (*NextInstr) & HintMask;
+
+ // Do absolute jump if there are 2 or more instructions before return from
+ // the old function.
+ if ((CurrInstrHintClear != ReturnSequence) &&
+ (NextInstrHintClear != ReturnSequence)) {
+ const unsigned LuiT0Instr = 0x3c080000, AddiuT0Instr = 0x25080000;
+ const unsigned JrT0Instr = 0x01000008;
+ // lui t0, high 16 bit of the NewAddr
+ (*(CurrentInstr++)) = LuiT0Instr | ((NewAddr & 0xffff0000) >> 16);
+ // addiu t0, t0, low 16 bit of the NewAddr
+ (*(CurrentInstr++)) = AddiuT0Instr | (NewAddr & 0x0000ffff);
+ // jr t0
+ (*(CurrentInstr++)) = JrT0Instr;
+ (*CurrentInstr) = NopInstr;
+
+ sys::Memory::InvalidateInstructionCache(Old, 4 * 4);
+ } else {
+ // Unsupported case
+ report_fatal_error("MipsJITInfo::replaceMachineCodeForFunction");
+ }
+ }
+}
+
+/// JITCompilerFunction - This contains the address of the JIT function used to
+/// compile a function lazily.
+static TargetJITInfo::JITCompilerFn JITCompilerFunction;
+
+// Get the ASMPREFIX for the current host. This is often '_'.
+#ifndef __USER_LABEL_PREFIX__
+#define __USER_LABEL_PREFIX__
+#endif
+#define GETASMPREFIX2(X) #X
+#define GETASMPREFIX(X) GETASMPREFIX2(X)
+#define ASMPREFIX GETASMPREFIX(__USER_LABEL_PREFIX__)
+
+// CompilationCallback stub - We can't use a C function with inline assembly in
+// it, because the prolog/epilog inserted by GCC won't work for us. Instead,
+// write our own wrapper, which does things our way, so we have complete control
+// over register saving and restoring. This code saves registers, calls
+// MipsCompilationCallbackC and restores registers.
+extern "C" {
+#if defined (__mips__)
+void MipsCompilationCallback();
+
+ asm(
+ ".text\n"
+ ".align 2\n"
+ ".globl " ASMPREFIX "MipsCompilationCallback\n"
+ ASMPREFIX "MipsCompilationCallback:\n"
+ ".ent " ASMPREFIX "MipsCompilationCallback\n"
+ ".frame $sp, 32, $ra\n"
+ ".set noreorder\n"
+ ".cpload $t9\n"
+
+ "addiu $sp, $sp, -64\n"
+ ".cprestore 16\n"
+
+ // Save argument registers a0, a1, a2, a3, f12, f14 since they may contain
+ // stuff for the real target function right now. We have to act as if this
+ // whole compilation callback doesn't exist as far as the caller is
+ // concerned. We also need to save the ra register since it contains the
+ // original return address, and t8 register since it contains the address
+ // of the end of function stub.
+ "sw $a0, 20($sp)\n"
+ "sw $a1, 24($sp)\n"
+ "sw $a2, 28($sp)\n"
+ "sw $a3, 32($sp)\n"
+ "sw $ra, 36($sp)\n"
+ "sw $t8, 40($sp)\n"
+ "sdc1 $f12, 48($sp)\n"
+ "sdc1 $f14, 56($sp)\n"
+
+ // t8 points at the end of function stub. Pass the beginning of the stub
+ // to the MipsCompilationCallbackC.
+ "addiu $a0, $t8, -16\n"
+ "jal " ASMPREFIX "MipsCompilationCallbackC\n"
+ "nop\n"
+
+ // Restore registers.
+ "lw $a0, 20($sp)\n"
+ "lw $a1, 24($sp)\n"
+ "lw $a2, 28($sp)\n"
+ "lw $a3, 32($sp)\n"
+ "lw $ra, 36($sp)\n"
+ "lw $t8, 40($sp)\n"
+ "ldc1 $f12, 48($sp)\n"
+ "ldc1 $f14, 56($sp)\n"
+ "addiu $sp, $sp, 64\n"
+
+ // Jump to the (newly modified) stub to invoke the real function.
+ "addiu $t8, $t8, -16\n"
+ "jr $t8\n"
+ "nop\n"
+
+ ".set reorder\n"
+ ".end " ASMPREFIX "MipsCompilationCallback\n"
+ );
+#else // host != Mips
+ void MipsCompilationCallback() {
+ llvm_unreachable(
+ "Cannot call MipsCompilationCallback() on a non-Mips arch!");
+ }
+#endif
+}
+
+/// MipsCompilationCallbackC - This is the target-specific function invoked
+/// by the function stub when we did not know the real target of a call.
+/// This function must locate the start of the stub or call site and pass
+/// it into the JIT compiler function.
+extern "C" void MipsCompilationCallbackC(intptr_t StubAddr) {
+ // Get the address of the compiled code for this function.
+ intptr_t NewVal = (intptr_t) JITCompilerFunction((void*) StubAddr);
+
+ // Rewrite the function stub so that we don't end up here every time we
+ // execute the call. We're replacing the first four instructions of the
+ // stub with code that jumps to the compiled function:
+ // lui $t9, %hi(NewVal)
+ // addiu $t9, $t9, %lo(NewVal)
+ // jr $t9
+ // nop
+
+ int Hi = ((unsigned)NewVal & 0xffff0000) >> 16;
+ if ((NewVal & 0x8000) != 0)
+ Hi++;
+ int Lo = (int)(NewVal & 0xffff);
+
+ *(intptr_t *)(StubAddr) = 0xf << 26 | 25 << 16 | Hi;
+ *(intptr_t *)(StubAddr + 4) = 9 << 26 | 25 << 21 | 25 << 16 | Lo;
+ *(intptr_t *)(StubAddr + 8) = 25 << 21 | 8;
+ *(intptr_t *)(StubAddr + 12) = 0;
+
+ sys::Memory::InvalidateInstructionCache((void*) StubAddr, 16);
+}
+
+TargetJITInfo::LazyResolverFn MipsJITInfo::getLazyResolverFunction(
+ JITCompilerFn F) {
+ JITCompilerFunction = F;
+ return MipsCompilationCallback;
+}
+
+TargetJITInfo::StubLayout MipsJITInfo::getStubLayout() {
+ // The stub contains 4 4-byte instructions, aligned at 4 bytes. See
+ // emitFunctionStub for details.
+ StubLayout Result = { 4*4, 4 };
+ return Result;
+}
+
+void *MipsJITInfo::emitFunctionStub(const Function *F, void *Fn,
+ JITCodeEmitter &JCE) {
+ JCE.emitAlignment(4);
+ void *Addr = (void*) (JCE.getCurrentPCValue());
+ if (!sys::Memory::setRangeWritable(Addr, 16))
+ llvm_unreachable("ERROR: Unable to mark stub writable.");
+
+ intptr_t EmittedAddr;
+ if (Fn != (void*)(intptr_t)MipsCompilationCallback)
+ EmittedAddr = (intptr_t)Fn;
+ else
+ EmittedAddr = (intptr_t)MipsCompilationCallback;
+
+
+ int Hi = ((unsigned)EmittedAddr & 0xffff0000) >> 16;
+ if ((EmittedAddr & 0x8000) != 0)
+ Hi++;
+ int Lo = (int)(EmittedAddr & 0xffff);
+
+ // lui $t9, %hi(EmittedAddr)
+ // addiu $t9, $t9, %lo(EmittedAddr)
+ // jalr $t8, $t9
+ // nop
+ if (IsLittleEndian) {
+ JCE.emitWordLE(0xf << 26 | 25 << 16 | Hi);
+ JCE.emitWordLE(9 << 26 | 25 << 21 | 25 << 16 | Lo);
+ JCE.emitWordLE(25 << 21 | 24 << 11 | 9);
+ JCE.emitWordLE(0);
+ } else {
+ JCE.emitWordBE(0xf << 26 | 25 << 16 | Hi);
+ JCE.emitWordBE(9 << 26 | 25 << 21 | 25 << 16 | Lo);
+ JCE.emitWordBE(25 << 21 | 24 << 11 | 9);
+ JCE.emitWordBE(0);
+ }
+
+ sys::Memory::InvalidateInstructionCache(Addr, 16);
+ if (!sys::Memory::setRangeExecutable(Addr, 16))
+ llvm_unreachable("ERROR: Unable to mark stub executable.");
+
+ return Addr;
+}
+
+/// relocate - Before the JIT can run a block of code that has been emitted,
+/// it must rewrite the code to contain the actual addresses of any
+/// referenced global symbols.
+void MipsJITInfo::relocate(void *Function, MachineRelocation *MR,
+ unsigned NumRelocs, unsigned char *GOTBase) {
+ for (unsigned i = 0; i != NumRelocs; ++i, ++MR) {
+
+ void *RelocPos = (char*) Function + MR->getMachineCodeOffset();
+ intptr_t ResultPtr = (intptr_t) MR->getResultPointer();
+
+ switch ((Mips::RelocationType) MR->getRelocationType()) {
+ case Mips::reloc_mips_pc16:
+ ResultPtr = (((ResultPtr - (intptr_t) RelocPos) - 4) >> 2) & 0xffff;
+ *((unsigned*) RelocPos) |= (unsigned) ResultPtr;
+ break;
+
+ case Mips::reloc_mips_26:
+ ResultPtr = (ResultPtr & 0x0fffffff) >> 2;
+ *((unsigned*) RelocPos) |= (unsigned) ResultPtr;
+ break;
+
+ case Mips::reloc_mips_hi:
+ ResultPtr = ResultPtr >> 16;
+ if ((((intptr_t) (MR->getResultPointer()) & 0xffff) >> 15) == 1) {
+ ResultPtr += 1;
+ }
+ *((unsigned*) RelocPos) |= (unsigned) ResultPtr;
+ break;
+
+ case Mips::reloc_mips_lo: {
+ // Addend is needed for unaligned load/store instructions, where offset
+ // for the second load/store in the expanded instruction sequence must
+ // be modified by +1 or +3. Otherwise, Addend is 0.
+ int Addend = *((unsigned*) RelocPos) & 0xffff;
+ ResultPtr = (ResultPtr + Addend) & 0xffff;
+ *((unsigned*) RelocPos) &= 0xffff0000;
+ *((unsigned*) RelocPos) |= (unsigned) ResultPtr;
+ break;
+ }
+ }
+ }
+}
--- /dev/null
+//===- MipsJITInfo.h - Mips Implementation of the JIT Interface -*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the declaration of the MipsJITInfo class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MIPSJITINFO_H
+#define MIPSJITINFO_H
+
+#include "MipsMachineFunction.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/Target/TargetJITInfo.h"
+
+namespace llvm {
+class MipsTargetMachine;
+
+class MipsJITInfo : public TargetJITInfo {
+
+ bool IsPIC;
+ bool IsLittleEndian;
+
+ public:
+ explicit MipsJITInfo() :
+ IsPIC(false), IsLittleEndian(true) {}
+
+ /// replaceMachineCodeForFunction - Make it so that calling the function
+ /// whose machine code is at OLD turns into a call to NEW, perhaps by
+ /// overwriting OLD with a branch to NEW. This is used for self-modifying
+ /// code.
+ ///
+ void replaceMachineCodeForFunction(void *Old, void *New) override;
+
+ // getStubLayout - Returns the size and alignment of the largest call stub
+ // on Mips.
+ StubLayout getStubLayout() override;
+
+ /// emitFunctionStub - Use the specified JITCodeEmitter object to emit a
+ /// small native function that simply calls the function at the specified
+ /// address.
+ void *emitFunctionStub(const Function *F, void *Fn,
+ JITCodeEmitter &JCE) override;
+
+ /// getLazyResolverFunction - Expose the lazy resolver to the JIT.
+ LazyResolverFn getLazyResolverFunction(JITCompilerFn) override;
+
+ /// relocate - Before the JIT can run a block of code that has been emitted,
+ /// it must rewrite the code to contain the actual addresses of any
+ /// referenced global symbols.
+ void relocate(void *Function, MachineRelocation *MR,
+ unsigned NumRelocs, unsigned char *GOTBase) override;
+
+ /// Initialize - Initialize internal stage for the function being JITted.
+ void Initialize(const MachineFunction &MF, bool isPIC,
+ bool isLittleEndian) {
+ IsPIC = isPIC;
+ IsLittleEndian = isLittleEndian;
+ }
+
+};
+}
+
+#endif
#include "Mips.h"
#include "MCTargetDesc/MipsBaseInfo.h"
#include "MCTargetDesc/MipsMCNaCl.h"
-#include "MipsMachineFunction.h"
#include "MipsTargetMachine.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
//
//===----------------------------------------------------------------------===//
#include "MipsSEISelLowering.h"
-#include "MipsMachineFunction.h"
#include "MipsRegisterInfo.h"
#include "MipsTargetMachine.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
HasDSPR2(false), AllowMixed16_32(Mixed16_32 | Mips_Os16), Os16(Mips_Os16),
HasMSA(false), TM(_TM), TargetTriple(TT),
DL(computeDataLayout(initializeSubtargetDependencies(CPU, FS, TM))),
- TSInfo(DL), InstrInfo(MipsInstrInfo::create(*this)),
+ TSInfo(DL), JITInfo(), InstrInfo(MipsInstrInfo::create(*this)),
FrameLowering(MipsFrameLowering::create(*this)),
TLInfo(MipsTargetLowering::create(*TM, *this)) {
#include "MipsFrameLowering.h"
#include "MipsISelLowering.h"
#include "MipsInstrInfo.h"
+#include "MipsJITInfo.h"
#include "MipsSelectionDAGInfo.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/MC/MCInstrItineraries.h"
const DataLayout DL; // Calculates type size & alignment
const MipsSelectionDAGInfo TSInfo;
+ MipsJITInfo JITInfo;
std::unique_ptr<const MipsInstrInfo> InstrInfo;
std::unique_ptr<const MipsFrameLowering> FrameLowering;
std::unique_ptr<const MipsTargetLowering> TLInfo;
void setHelperClassesMips16();
void setHelperClassesMipsSE();
+ MipsJITInfo *getJITInfo() override { return &JITInfo; }
const MipsSelectionDAGInfo *getSelectionDAGInfo() const override {
return &TSInfo;
}
addPass(createMipsConstantIslandPass(TM));
return true;
}
+
+bool MipsTargetMachine::addCodeEmitter(PassManagerBase &PM,
+ JITCodeEmitter &JCE) {
+ // Machine code emitter pass for Mips.
+ PM.add(createMipsJITCodeEmitterPass(*this, JCE));
+ return false;
+}
return Subtarget;
return &DefaultSubtarget;
}
+ MipsSubtarget *getSubtargetImpl() {
+ return static_cast<MipsSubtarget *>(TargetMachine::getSubtargetImpl());
+ }
/// \brief Reset the subtarget for the Mips target.
void resetSubtarget(MachineFunction *MF);
// Pass Pipeline Configuration
TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
+ bool addCodeEmitter(PassManagerBase &PM, JITCodeEmitter &JCE) override;
};
/// MipsebTargetMachine - Mips32/64 big endian target machine.
TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
+ // Emission of machine code through JITCodeEmitter is not supported.
+ bool addPassesToEmitMachineCode(PassManagerBase &, JITCodeEmitter &,
+ bool = true) override {
+ return true;
+ }
+
// Emission of machine code through MCJIT is not supported.
bool addPassesToEmitMC(PassManagerBase &, MCContext *&, raw_ostream &,
bool = true) override {
tablegen(LLVM PPCGenAsmWriter.inc -gen-asm-writer)
tablegen(LLVM PPCGenAsmMatcher.inc -gen-asm-matcher)
+tablegen(LLVM PPCGenCodeEmitter.inc -gen-emitter)
tablegen(LLVM PPCGenDisassemblerTables.inc -gen-disassembler)
-tablegen(LLVM PPCGenMCCodeEmitter.inc -gen-emitter)
+tablegen(LLVM PPCGenMCCodeEmitter.inc -gen-emitter -mc-emitter)
tablegen(LLVM PPCGenRegisterInfo.inc -gen-register-info)
tablegen(LLVM PPCGenInstrInfo.inc -gen-instr-info)
tablegen(LLVM PPCGenDAGISel.inc -gen-dag-isel)
add_llvm_target(PowerPCCodeGen
PPCAsmPrinter.cpp
PPCBranchSelector.cpp
+ PPCCodeEmitter.cpp
PPCCTRLoops.cpp
PPCHazardRecognizers.cpp
PPCInstrInfo.cpp
PPCISelLowering.cpp
PPCFastISel.cpp
PPCFrameLowering.cpp
+ PPCJITInfo.cpp
PPCMCInstLower.cpp
PPCMachineFunctionInfo.cpp
PPCRegisterInfo.cpp
# Make sure that tblgen is run, first thing.
BUILT_SOURCES = PPCGenRegisterInfo.inc PPCGenAsmMatcher.inc \
- PPCGenAsmWriter.inc \
+ PPCGenAsmWriter.inc PPCGenCodeEmitter.inc \
PPCGenInstrInfo.inc PPCGenDAGISel.inc \
PPCGenSubtargetInfo.inc PPCGenCallingConv.inc \
PPCGenMCCodeEmitter.inc PPCGenFastISel.inc \
class PassRegistry;
class FunctionPass;
class ImmutablePass;
+ class JITCodeEmitter;
class MachineInstr;
class AsmPrinter;
class MCInst;
FunctionPass *createPPCVSXFMAMutatePass();
FunctionPass *createPPCBranchSelectionPass();
FunctionPass *createPPCISelDag(PPCTargetMachine &TM);
+ FunctionPass *createPPCJITCodeEmitterPass(PPCTargetMachine &TM,
+ JITCodeEmitter &MCE);
void LowerPPCMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
AsmPrinter &AP, bool isDarwin);
return true;
const TargetLowering *TLI = TM->getSubtargetImpl()->getTargetLowering();
- if (SI->getNumCases() + 1 >= (unsigned)TLI->getMinimumJumpTableEntries())
+ if (TLI->supportJumpTables() &&
+ SI->getNumCases()+1 >= (unsigned) TLI->getMinimumJumpTableEntries())
return true;
}
}
--- /dev/null
+//===-- PPCCodeEmitter.cpp - JIT Code Emitter for PowerPC -----------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the PowerPC 32-bit CodeEmitter and associated machinery to
+// JIT-compile bitcode to native PowerPC.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PPC.h"
+#include "PPCRelocations.h"
+#include "PPCTargetMachine.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/IR/Module.h"
+#include "llvm/PassManager.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetOptions.h"
+using namespace llvm;
+
+namespace {
+ class PPCCodeEmitter : public MachineFunctionPass {
+ TargetMachine &TM;
+ JITCodeEmitter &MCE;
+ MachineModuleInfo *MMI;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<MachineModuleInfo>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ static char ID;
+
+ /// MovePCtoLROffset - When/if we see a MovePCtoLR instruction, we record
+ /// its address in the function into this pointer.
+ void *MovePCtoLROffset;
+ public:
+
+ PPCCodeEmitter(TargetMachine &tm, JITCodeEmitter &mce)
+ : MachineFunctionPass(ID), TM(tm), MCE(mce) {}
+
+ /// getBinaryCodeForInstr - This function, generated by the
+ /// CodeEmitterGenerator using TableGen, produces the binary encoding for
+ /// machine instructions.
+ uint64_t getBinaryCodeForInstr(const MachineInstr &MI) const;
+
+
+ MachineRelocation GetRelocation(const MachineOperand &MO,
+ unsigned RelocID) const;
+
+ /// getMachineOpValue - evaluates the MachineOperand of a given MachineInstr
+ unsigned getMachineOpValue(const MachineInstr &MI,
+ const MachineOperand &MO) const;
+
+ unsigned get_crbitm_encoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getDirectBrEncoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getCondBrEncoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getAbsDirectBrEncoding(const MachineInstr &MI,
+ unsigned OpNo) const;
+ unsigned getAbsCondBrEncoding(const MachineInstr &MI, unsigned OpNo) const;
+
+ unsigned getImm16Encoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getMemRIEncoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getMemRIXEncoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getTLSRegEncoding(const MachineInstr &MI, unsigned OpNo) const;
+ unsigned getTLSCallEncoding(const MachineInstr &MI, unsigned OpNo) const;
+
+ const char *getPassName() const override {
+ return "PowerPC Machine Code Emitter";
+ }
+
+ /// runOnMachineFunction - emits the given MachineFunction to memory
+ ///
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ /// emitBasicBlock - emits the given MachineBasicBlock to memory
+ ///
+ void emitBasicBlock(MachineBasicBlock &MBB);
+ };
+}
+
+char PPCCodeEmitter::ID = 0;
+
+/// createPPCCodeEmitterPass - Return a pass that emits the collected PPC code
+/// to the specified MCE object.
+FunctionPass *llvm::createPPCJITCodeEmitterPass(PPCTargetMachine &TM,
+ JITCodeEmitter &JCE) {
+ return new PPCCodeEmitter(TM, JCE);
+}
+
+bool PPCCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
+ assert((MF.getTarget().getRelocationModel() != Reloc::Default ||
+ MF.getTarget().getRelocationModel() != Reloc::Static) &&
+ "JIT relocation model must be set to static or default!");
+
+ MMI = &getAnalysis<MachineModuleInfo>();
+ MCE.setModuleInfo(MMI);
+ do {
+ MovePCtoLROffset = nullptr;
+ MCE.startFunction(MF);
+ for (MachineFunction::iterator BB = MF.begin(), E = MF.end(); BB != E; ++BB)
+ emitBasicBlock(*BB);
+ } while (MCE.finishFunction(MF));
+
+ return false;
+}
+
+void PPCCodeEmitter::emitBasicBlock(MachineBasicBlock &MBB) {
+ MCE.StartMachineBasicBlock(&MBB);
+
+ for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E; ++I){
+ const MachineInstr &MI = *I;
+ MCE.processDebugLoc(MI.getDebugLoc(), true);
+ switch (MI.getOpcode()) {
+ default:
+ MCE.emitWordBE(getBinaryCodeForInstr(MI));
+ break;
+ case TargetOpcode::CFI_INSTRUCTION:
+ break;
+ case TargetOpcode::EH_LABEL:
+ MCE.emitLabel(MI.getOperand(0).getMCSymbol());
+ break;
+ case TargetOpcode::IMPLICIT_DEF:
+ case TargetOpcode::KILL:
+ break; // pseudo opcode, no side effects
+ case PPC::MovePCtoLR:
+ case PPC::MovePCtoLR8:
+ assert(TM.getRelocationModel() == Reloc::PIC_);
+ MovePCtoLROffset = (void*)MCE.getCurrentPCValue();
+ MCE.emitWordBE(0x48000005); // bl 1
+ break;
+ }
+ MCE.processDebugLoc(MI.getDebugLoc(), false);
+ }
+}
+
+unsigned PPCCodeEmitter::get_crbitm_encoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ const MachineOperand &MO = MI.getOperand(OpNo);
+ assert((MI.getOpcode() == PPC::MTOCRF || MI.getOpcode() == PPC::MTOCRF8 ||
+ MI.getOpcode() == PPC::MFOCRF || MI.getOpcode() == PPC::MFOCRF8) &&
+ (MO.getReg() >= PPC::CR0 && MO.getReg() <= PPC::CR7));
+ return 0x80 >> TM.getSubtargetImpl()->getRegisterInfo()->getEncodingValue(
+ MO.getReg());
+}
+
+MachineRelocation PPCCodeEmitter::GetRelocation(const MachineOperand &MO,
+ unsigned RelocID) const {
+ // If in PIC mode, we need to encode the negated address of the
+ // 'movepctolr' into the unrelocated field. After relocation, we'll have
+ // &gv-&movepctolr-4 in the imm field. Once &movepctolr is added to the imm
+ // field, we get &gv. This doesn't happen for branch relocations, which are
+ // always implicitly pc relative.
+ intptr_t Cst = 0;
+ if (TM.getRelocationModel() == Reloc::PIC_) {
+ assert(MovePCtoLROffset && "MovePCtoLR not seen yet?");
+ Cst = -(intptr_t)MovePCtoLROffset - 4;
+ }
+
+ if (MO.isGlobal())
+ return MachineRelocation::getGV(MCE.getCurrentPCOffset(), RelocID,
+ const_cast<GlobalValue *>(MO.getGlobal()),
+ Cst, isa<Function>(MO.getGlobal()));
+ if (MO.isSymbol())
+ return MachineRelocation::getExtSym(MCE.getCurrentPCOffset(),
+ RelocID, MO.getSymbolName(), Cst);
+ if (MO.isCPI())
+ return MachineRelocation::getConstPool(MCE.getCurrentPCOffset(),
+ RelocID, MO.getIndex(), Cst);
+
+ if (MO.isMBB())
+ return MachineRelocation::getBB(MCE.getCurrentPCOffset(),
+ RelocID, MO.getMBB());
+
+ assert(MO.isJTI());
+ return MachineRelocation::getJumpTable(MCE.getCurrentPCOffset(),
+ RelocID, MO.getIndex(), Cst);
+}
+
+unsigned PPCCodeEmitter::getDirectBrEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ const MachineOperand &MO = MI.getOperand(OpNo);
+ if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO);
+
+ MCE.addRelocation(GetRelocation(MO, PPC::reloc_pcrel_bx));
+ return 0;
+}
+
+unsigned PPCCodeEmitter::getCondBrEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ const MachineOperand &MO = MI.getOperand(OpNo);
+ MCE.addRelocation(GetRelocation(MO, PPC::reloc_pcrel_bcx));
+ return 0;
+}
+
+unsigned PPCCodeEmitter::getAbsDirectBrEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ const MachineOperand &MO = MI.getOperand(OpNo);
+ if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO);
+
+ llvm_unreachable("Absolute branch relocations unsupported on the old JIT.");
+}
+
+unsigned PPCCodeEmitter::getAbsCondBrEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("Absolute branch relocations unsupported on the old JIT.");
+}
+
+unsigned PPCCodeEmitter::getImm16Encoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ const MachineOperand &MO = MI.getOperand(OpNo);
+ if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO);
+
+ unsigned RelocID;
+ switch (MO.getTargetFlags() & PPCII::MO_ACCESS_MASK) {
+ default: llvm_unreachable("Unsupported target operand flags!");
+ case PPCII::MO_LO: RelocID = PPC::reloc_absolute_low; break;
+ case PPCII::MO_HA: RelocID = PPC::reloc_absolute_high; break;
+ }
+
+ MCE.addRelocation(GetRelocation(MO, RelocID));
+ return 0;
+}
+
+unsigned PPCCodeEmitter::getMemRIEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ // Encode (imm, reg) as a memri, which has the low 16-bits as the
+ // displacement and the next 5 bits as the register #.
+ assert(MI.getOperand(OpNo+1).isReg());
+ unsigned RegBits = getMachineOpValue(MI, MI.getOperand(OpNo+1)) << 16;
+
+ const MachineOperand &MO = MI.getOperand(OpNo);
+ if (MO.isImm())
+ return (getMachineOpValue(MI, MO) & 0xFFFF) | RegBits;
+
+ // Add a fixup for the displacement field.
+ MCE.addRelocation(GetRelocation(MO, PPC::reloc_absolute_low));
+ return RegBits;
+}
+
+unsigned PPCCodeEmitter::getMemRIXEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ // Encode (imm, reg) as a memrix, which has the low 14-bits as the
+ // displacement and the next 5 bits as the register #.
+ assert(MI.getOperand(OpNo+1).isReg());
+ unsigned RegBits = getMachineOpValue(MI, MI.getOperand(OpNo+1)) << 14;
+
+ const MachineOperand &MO = MI.getOperand(OpNo);
+ if (MO.isImm())
+ return ((getMachineOpValue(MI, MO) >> 2) & 0x3FFF) | RegBits;
+
+ MCE.addRelocation(GetRelocation(MO, PPC::reloc_absolute_low_ix));
+ return RegBits;
+}
+
+
+unsigned PPCCodeEmitter::getTLSRegEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("TLS not supported on the old JIT.");
+ return 0;
+}
+
+unsigned PPCCodeEmitter::getTLSCallEncoding(const MachineInstr &MI,
+ unsigned OpNo) const {
+ llvm_unreachable("TLS not supported on the old JIT.");
+ return 0;
+}
+
+unsigned PPCCodeEmitter::getMachineOpValue(const MachineInstr &MI,
+ const MachineOperand &MO) const {
+
+ if (MO.isReg()) {
+ // MTOCRF/MFOCRF should go through get_crbitm_encoding for the CR operand.
+ // The GPR operand should come through here though.
+ assert((MI.getOpcode() != PPC::MTOCRF && MI.getOpcode() != PPC::MTOCRF8 &&
+ MI.getOpcode() != PPC::MFOCRF && MI.getOpcode() != PPC::MFOCRF8) ||
+ MO.getReg() < PPC::CR0 || MO.getReg() > PPC::CR7);
+ return TM.getSubtargetImpl()->getRegisterInfo()->getEncodingValue(
+ MO.getReg());
+ }
+
+ assert(MO.isImm() &&
+ "Relocation required in an instruction that we cannot encode!");
+ return MO.getImm();
+}
+
+#include "PPCGenCodeEmitter.inc"
if (Subtarget.isDarwin())
setPrefFunctionAlignment(4);
+ if (isPPC64 && Subtarget.isJITCodeModel())
+ // Temporary workaround for the inability of PPC64 JIT to handle jump
+ // tables.
+ setSupportJumpTables(false);
+
setInsertFencesForAtomic(true);
if (Subtarget.enableMachineScheduler())
}
if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
- unsigned OpFlags = 0;
- if ((DAG.getTarget().getRelocationModel() != Reloc::Static &&
- (Subtarget.getTargetTriple().isMacOSX() &&
- Subtarget.getTargetTriple().isMacOSXVersionLT(10, 5)) &&
- (G->getGlobal()->isDeclaration() ||
- G->getGlobal()->isWeakForLinker())) ||
- (Subtarget.isTargetELF() && !isPPC64 &&
- !G->getGlobal()->hasLocalLinkage() &&
- DAG.getTarget().getRelocationModel() == Reloc::PIC_)) {
- // PC-relative references to external symbols should go through $stub,
- // unless we're building with the leopard linker or later, which
- // automatically synthesizes these stubs.
- OpFlags = PPCII::MO_PLT_OR_STUB;
- }
+ // XXX Work around for http://llvm.org/bugs/show_bug.cgi?id=5201
+ // Use indirect calls for ALL functions calls in JIT mode, since the
+ // far-call stubs may be outside relocation limits for a BL instruction.
+ if (!DAG.getTarget().getSubtarget<PPCSubtarget>().isJITCodeModel()) {
+ unsigned OpFlags = 0;
+ if ((DAG.getTarget().getRelocationModel() != Reloc::Static &&
+ (Subtarget.getTargetTriple().isMacOSX() &&
+ Subtarget.getTargetTriple().isMacOSXVersionLT(10, 5)) &&
+ (G->getGlobal()->isDeclaration() ||
+ G->getGlobal()->isWeakForLinker())) ||
+ (Subtarget.isTargetELF() && !isPPC64 &&
+ !G->getGlobal()->hasLocalLinkage() &&
+ DAG.getTarget().getRelocationModel() == Reloc::PIC_)) {
+ // PC-relative references to external symbols should go through $stub,
+ // unless we're building with the leopard linker or later, which
+ // automatically synthesizes these stubs.
+ OpFlags = PPCII::MO_PLT_OR_STUB;
+ }
- // If the callee is a GlobalAddress/ExternalSymbol node (quite common,
- // every direct call is) turn it into a TargetGlobalAddress /
- // TargetExternalSymbol node so that legalize doesn't hack it.
- Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl,
- Callee.getValueType(), 0, OpFlags);
- needIndirectCall = false;
+ // If the callee is a GlobalAddress/ExternalSymbol node (quite common,
+ // every direct call is) turn it into a TargetGlobalAddress /
+ // TargetExternalSymbol node so that legalize doesn't hack it.
+ Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl,
+ Callee.getValueType(),
+ 0, OpFlags);
+ needIndirectCall = false;
+ }
}
if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
--- /dev/null
+//===-- PPCJITInfo.cpp - Implement the JIT interfaces for the PowerPC -----===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the JIT interfaces for the 32-bit PowerPC target.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PPCJITInfo.h"
+#include "PPCRelocations.h"
+#include "PPCSubtarget.h"
+#include "llvm/IR/Function.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Memory.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+static TargetJITInfo::JITCompilerFn JITCompilerFunction;
+
+PPCJITInfo::PPCJITInfo(PPCSubtarget &STI)
+ : Subtarget(STI), is64Bit(STI.isPPC64()) {
+ useGOT = 0;
+}
+
+#define BUILD_ADDIS(RD,RS,IMM16) \
+ ((15 << 26) | ((RD) << 21) | ((RS) << 16) | ((IMM16) & 65535))
+#define BUILD_ORI(RD,RS,UIMM16) \
+ ((24 << 26) | ((RS) << 21) | ((RD) << 16) | ((UIMM16) & 65535))
+#define BUILD_ORIS(RD,RS,UIMM16) \
+ ((25 << 26) | ((RS) << 21) | ((RD) << 16) | ((UIMM16) & 65535))
+#define BUILD_RLDICR(RD,RS,SH,ME) \
+ ((30 << 26) | ((RS) << 21) | ((RD) << 16) | (((SH) & 31) << 11) | \
+ (((ME) & 63) << 6) | (1 << 2) | ((((SH) >> 5) & 1) << 1))
+#define BUILD_MTSPR(RS,SPR) \
+ ((31 << 26) | ((RS) << 21) | ((SPR) << 16) | (467 << 1))
+#define BUILD_BCCTRx(BO,BI,LINK) \
+ ((19 << 26) | ((BO) << 21) | ((BI) << 16) | (528 << 1) | ((LINK) & 1))
+#define BUILD_B(TARGET, LINK) \
+ ((18 << 26) | (((TARGET) & 0x00FFFFFF) << 2) | ((LINK) & 1))
+
+// Pseudo-ops
+#define BUILD_LIS(RD,IMM16) BUILD_ADDIS(RD,0,IMM16)
+#define BUILD_SLDI(RD,RS,IMM6) BUILD_RLDICR(RD,RS,IMM6,63-IMM6)
+#define BUILD_MTCTR(RS) BUILD_MTSPR(RS,9)
+#define BUILD_BCTR(LINK) BUILD_BCCTRx(20,0,LINK)
+
+static void EmitBranchToAt(uint64_t At, uint64_t To, bool isCall, bool is64Bit){
+ intptr_t Offset = ((intptr_t)To - (intptr_t)At) >> 2;
+ unsigned *AtI = (unsigned*)(intptr_t)At;
+
+ if (Offset >= -(1 << 23) && Offset < (1 << 23)) { // In range?
+ AtI[0] = BUILD_B(Offset, isCall); // b/bl target
+ } else if (!is64Bit) {
+ AtI[0] = BUILD_LIS(12, To >> 16); // lis r12, hi16(address)
+ AtI[1] = BUILD_ORI(12, 12, To); // ori r12, r12, lo16(address)
+ AtI[2] = BUILD_MTCTR(12); // mtctr r12
+ AtI[3] = BUILD_BCTR(isCall); // bctr/bctrl
+ } else {
+ AtI[0] = BUILD_LIS(12, To >> 48); // lis r12, hi16(address)
+ AtI[1] = BUILD_ORI(12, 12, To >> 32); // ori r12, r12, lo16(address)
+ AtI[2] = BUILD_SLDI(12, 12, 32); // sldi r12, r12, 32
+ AtI[3] = BUILD_ORIS(12, 12, To >> 16); // oris r12, r12, hi16(address)
+ AtI[4] = BUILD_ORI(12, 12, To); // ori r12, r12, lo16(address)
+ AtI[5] = BUILD_MTCTR(12); // mtctr r12
+ AtI[6] = BUILD_BCTR(isCall); // bctr/bctrl
+ }
+}
+
+extern "C" void PPC32CompilationCallback();
+extern "C" void PPC64CompilationCallback();
+
+// The first clause of the preprocessor directive looks wrong, but it is
+// necessary when compiling this code on non-PowerPC hosts.
+#if (!defined(__ppc__) && !defined(__powerpc__)) || defined(__powerpc64__) || defined(__ppc64__)
+void PPC32CompilationCallback() {
+ llvm_unreachable("This is not a 32bit PowerPC, you can't execute this!");
+}
+#elif !defined(__ELF__)
+// CompilationCallback stub - We can't use a C function with inline assembly in
+// it, because we the prolog/epilog inserted by GCC won't work for us. Instead,
+// write our own wrapper, which does things our way, so we have complete control
+// over register saving and restoring.
+asm(
+ ".text\n"
+ ".align 2\n"
+ ".globl _PPC32CompilationCallback\n"
+"_PPC32CompilationCallback:\n"
+ // Make space for 8 ints r[3-10] and 13 doubles f[1-13] and the
+ // FIXME: need to save v[0-19] for altivec?
+ // FIXME: could shrink frame
+ // Set up a proper stack frame
+ // FIXME Layout
+ // PowerPC32 ABI linkage - 24 bytes
+ // parameters - 32 bytes
+ // 13 double registers - 104 bytes
+ // 8 int registers - 32 bytes
+ "mflr r0\n"
+ "stw r0, 8(r1)\n"
+ "stwu r1, -208(r1)\n"
+ // Save all int arg registers
+ "stw r10, 204(r1)\n" "stw r9, 200(r1)\n"
+ "stw r8, 196(r1)\n" "stw r7, 192(r1)\n"
+ "stw r6, 188(r1)\n" "stw r5, 184(r1)\n"
+ "stw r4, 180(r1)\n" "stw r3, 176(r1)\n"
+ // Save all call-clobbered FP regs.
+ "stfd f13, 168(r1)\n" "stfd f12, 160(r1)\n"
+ "stfd f11, 152(r1)\n" "stfd f10, 144(r1)\n"
+ "stfd f9, 136(r1)\n" "stfd f8, 128(r1)\n"
+ "stfd f7, 120(r1)\n" "stfd f6, 112(r1)\n"
+ "stfd f5, 104(r1)\n" "stfd f4, 96(r1)\n"
+ "stfd f3, 88(r1)\n" "stfd f2, 80(r1)\n"
+ "stfd f1, 72(r1)\n"
+ // Arguments to Compilation Callback:
+ // r3 - our lr (address of the call instruction in stub plus 4)
+ // r4 - stub's lr (address of instruction that called the stub plus 4)
+ // r5 - is64Bit - always 0.
+ "mr r3, r0\n"
+ "lwz r2, 208(r1)\n" // stub's frame
+ "lwz r4, 8(r2)\n" // stub's lr
+ "li r5, 0\n" // 0 == 32 bit
+ "bl _LLVMPPCCompilationCallback\n"
+ "mtctr r3\n"
+ // Restore all int arg registers
+ "lwz r10, 204(r1)\n" "lwz r9, 200(r1)\n"
+ "lwz r8, 196(r1)\n" "lwz r7, 192(r1)\n"
+ "lwz r6, 188(r1)\n" "lwz r5, 184(r1)\n"
+ "lwz r4, 180(r1)\n" "lwz r3, 176(r1)\n"
+ // Restore all FP arg registers
+ "lfd f13, 168(r1)\n" "lfd f12, 160(r1)\n"
+ "lfd f11, 152(r1)\n" "lfd f10, 144(r1)\n"
+ "lfd f9, 136(r1)\n" "lfd f8, 128(r1)\n"
+ "lfd f7, 120(r1)\n" "lfd f6, 112(r1)\n"
+ "lfd f5, 104(r1)\n" "lfd f4, 96(r1)\n"
+ "lfd f3, 88(r1)\n" "lfd f2, 80(r1)\n"
+ "lfd f1, 72(r1)\n"
+ // Pop 3 frames off the stack and branch to target
+ "lwz r1, 208(r1)\n"
+ "lwz r2, 8(r1)\n"
+ "mtlr r2\n"
+ "bctr\n"
+ );
+
+#else
+// ELF PPC 32 support
+
+// CompilationCallback stub - We can't use a C function with inline assembly in
+// it, because we the prolog/epilog inserted by GCC won't work for us. Instead,
+// write our own wrapper, which does things our way, so we have complete control
+// over register saving and restoring.
+asm(
+ ".text\n"
+ ".align 2\n"
+ ".globl PPC32CompilationCallback\n"
+"PPC32CompilationCallback:\n"
+ // Make space for 8 ints r[3-10] and 8 doubles f[1-8] and the
+ // FIXME: need to save v[0-19] for altivec?
+ // FIXME: could shrink frame
+ // Set up a proper stack frame
+ // FIXME Layout
+ // 8 double registers - 64 bytes
+ // 8 int registers - 32 bytes
+ "mflr 0\n"
+ "stw 0, 4(1)\n"
+ "stwu 1, -104(1)\n"
+ // Save all int arg registers
+ "stw 10, 100(1)\n" "stw 9, 96(1)\n"
+ "stw 8, 92(1)\n" "stw 7, 88(1)\n"
+ "stw 6, 84(1)\n" "stw 5, 80(1)\n"
+ "stw 4, 76(1)\n" "stw 3, 72(1)\n"
+ // Save all call-clobbered FP regs.
+ "stfd 8, 64(1)\n"
+ "stfd 7, 56(1)\n" "stfd 6, 48(1)\n"
+ "stfd 5, 40(1)\n" "stfd 4, 32(1)\n"
+ "stfd 3, 24(1)\n" "stfd 2, 16(1)\n"
+ "stfd 1, 8(1)\n"
+ // Arguments to Compilation Callback:
+ // r3 - our lr (address of the call instruction in stub plus 4)
+ // r4 - stub's lr (address of instruction that called the stub plus 4)
+ // r5 - is64Bit - always 0.
+ "mr 3, 0\n"
+ "lwz 5, 104(1)\n" // stub's frame
+ "lwz 4, 4(5)\n" // stub's lr
+ "li 5, 0\n" // 0 == 32 bit
+ "bl LLVMPPCCompilationCallback\n"
+ "mtctr 3\n"
+ // Restore all int arg registers
+ "lwz 10, 100(1)\n" "lwz 9, 96(1)\n"
+ "lwz 8, 92(1)\n" "lwz 7, 88(1)\n"
+ "lwz 6, 84(1)\n" "lwz 5, 80(1)\n"
+ "lwz 4, 76(1)\n" "lwz 3, 72(1)\n"
+ // Restore all FP arg registers
+ "lfd 8, 64(1)\n"
+ "lfd 7, 56(1)\n" "lfd 6, 48(1)\n"
+ "lfd 5, 40(1)\n" "lfd 4, 32(1)\n"
+ "lfd 3, 24(1)\n" "lfd 2, 16(1)\n"
+ "lfd 1, 8(1)\n"
+ // Pop 3 frames off the stack and branch to target
+ "lwz 1, 104(1)\n"
+ "lwz 0, 4(1)\n"
+ "mtlr 0\n"
+ "bctr\n"
+ );
+#endif
+
+#if !defined(__powerpc64__) && !defined(__ppc64__)
+void PPC64CompilationCallback() {
+ llvm_unreachable("This is not a 64bit PowerPC, you can't execute this!");
+}
+#else
+# ifdef __ELF__
+asm(
+ ".text\n"
+ ".align 2\n"
+ ".globl PPC64CompilationCallback\n"
+#if _CALL_ELF == 2
+ ".type PPC64CompilationCallback,@function\n"
+"PPC64CompilationCallback:\n"
+#else
+ ".section \".opd\",\"aw\",@progbits\n"
+ ".align 3\n"
+"PPC64CompilationCallback:\n"
+ ".quad .L.PPC64CompilationCallback,.TOC.@tocbase,0\n"
+ ".size PPC64CompilationCallback,24\n"
+ ".previous\n"
+ ".align 4\n"
+ ".type PPC64CompilationCallback,@function\n"
+".L.PPC64CompilationCallback:\n"
+#endif
+# else
+asm(
+ ".text\n"
+ ".align 2\n"
+ ".globl _PPC64CompilationCallback\n"
+"_PPC64CompilationCallback:\n"
+# endif
+ // Make space for 8 ints r[3-10] and 13 doubles f[1-13] and the
+ // FIXME: need to save v[0-19] for altivec?
+ // Set up a proper stack frame
+ // Layout
+ // PowerPC64 ABI linkage - 48 bytes
+ // parameters - 64 bytes
+ // 13 double registers - 104 bytes
+ // 8 int registers - 64 bytes
+ "mflr 0\n"
+ "std 0, 16(1)\n"
+ "stdu 1, -280(1)\n"
+ // Save all int arg registers
+ "std 10, 272(1)\n" "std 9, 264(1)\n"
+ "std 8, 256(1)\n" "std 7, 248(1)\n"
+ "std 6, 240(1)\n" "std 5, 232(1)\n"
+ "std 4, 224(1)\n" "std 3, 216(1)\n"
+ // Save all call-clobbered FP regs.
+ "stfd 13, 208(1)\n" "stfd 12, 200(1)\n"
+ "stfd 11, 192(1)\n" "stfd 10, 184(1)\n"
+ "stfd 9, 176(1)\n" "stfd 8, 168(1)\n"
+ "stfd 7, 160(1)\n" "stfd 6, 152(1)\n"
+ "stfd 5, 144(1)\n" "stfd 4, 136(1)\n"
+ "stfd 3, 128(1)\n" "stfd 2, 120(1)\n"
+ "stfd 1, 112(1)\n"
+ // Arguments to Compilation Callback:
+ // r3 - our lr (address of the call instruction in stub plus 4)
+ // r4 - stub's lr (address of instruction that called the stub plus 4)
+ // r5 - is64Bit - always 1.
+ "mr 3, 0\n" // return address (still in r0)
+ "ld 5, 280(1)\n" // stub's frame
+ "ld 4, 16(5)\n" // stub's lr
+ "li 5, 1\n" // 1 == 64 bit
+# ifdef __ELF__
+ "bl LLVMPPCCompilationCallback\n"
+ "nop\n"
+# else
+ "bl _LLVMPPCCompilationCallback\n"
+# endif
+ "mtctr 3\n"
+ // Restore all int arg registers
+ "ld 10, 272(1)\n" "ld 9, 264(1)\n"
+ "ld 8, 256(1)\n" "ld 7, 248(1)\n"
+ "ld 6, 240(1)\n" "ld 5, 232(1)\n"
+ "ld 4, 224(1)\n" "ld 3, 216(1)\n"
+ // Restore all FP arg registers
+ "lfd 13, 208(1)\n" "lfd 12, 200(1)\n"
+ "lfd 11, 192(1)\n" "lfd 10, 184(1)\n"
+ "lfd 9, 176(1)\n" "lfd 8, 168(1)\n"
+ "lfd 7, 160(1)\n" "lfd 6, 152(1)\n"
+ "lfd 5, 144(1)\n" "lfd 4, 136(1)\n"
+ "lfd 3, 128(1)\n" "lfd 2, 120(1)\n"
+ "lfd 1, 112(1)\n"
+ // Pop 3 frames off the stack and branch to target
+ "ld 1, 280(1)\n"
+ "ld 0, 16(1)\n"
+ "mtlr 0\n"
+ // XXX: any special TOC handling in the ELF case for JIT?
+ "bctr\n"
+ );
+#endif
+
+extern "C" {
+LLVM_LIBRARY_VISIBILITY void *
+LLVMPPCCompilationCallback(unsigned *StubCallAddrPlus4,
+ unsigned *OrigCallAddrPlus4,
+ bool is64Bit) {
+ // Adjust the pointer to the address of the call instruction in the stub
+ // emitted by emitFunctionStub, rather than the instruction after it.
+ unsigned *StubCallAddr = StubCallAddrPlus4 - 1;
+ unsigned *OrigCallAddr = OrigCallAddrPlus4 - 1;
+
+ void *Target = JITCompilerFunction(StubCallAddr);
+
+ // Check to see if *OrigCallAddr is a 'bl' instruction, and if we can rewrite
+ // it to branch directly to the destination. If so, rewrite it so it does not
+ // need to go through the stub anymore.
+ unsigned OrigCallInst = *OrigCallAddr;
+ if ((OrigCallInst >> 26) == 18) { // Direct call.
+ intptr_t Offset = ((intptr_t)Target - (intptr_t)OrigCallAddr) >> 2;
+
+ if (Offset >= -(1 << 23) && Offset < (1 << 23)) { // In range?
+ // Clear the original target out.
+ OrigCallInst &= (63 << 26) | 3;
+ // Fill in the new target.
+ OrigCallInst |= (Offset & ((1 << 24)-1)) << 2;
+ // Replace the call.
+ *OrigCallAddr = OrigCallInst;
+ }
+ }
+
+ // Assert that we are coming from a stub that was created with our
+ // emitFunctionStub.
+ if ((*StubCallAddr >> 26) == 18)
+ StubCallAddr -= 3;
+ else {
+ assert((*StubCallAddr >> 26) == 19 && "Call in stub is not indirect!");
+ StubCallAddr -= is64Bit ? 9 : 6;
+ }
+
+ // Rewrite the stub with an unconditional branch to the target, for any users
+ // who took the address of the stub.
+ EmitBranchToAt((intptr_t)StubCallAddr, (intptr_t)Target, false, is64Bit);
+ sys::Memory::InvalidateInstructionCache(StubCallAddr, 7*4);
+
+ // Put the address of the target function to call and the address to return to
+ // after calling the target function in a place that is easy to get on the
+ // stack after we restore all regs.
+ return Target;
+}
+}
+
+
+
+TargetJITInfo::LazyResolverFn
+PPCJITInfo::getLazyResolverFunction(JITCompilerFn Fn) {
+ JITCompilerFunction = Fn;
+ return is64Bit ? PPC64CompilationCallback : PPC32CompilationCallback;
+}
+
+TargetJITInfo::StubLayout PPCJITInfo::getStubLayout() {
+ // The stub contains up to 10 4-byte instructions, aligned at 4 bytes: 3
+ // instructions to save the caller's address if this is a lazy-compilation
+ // stub, plus a 1-, 4-, or 7-instruction sequence to load an arbitrary address
+ // into a register and jump through it.
+ StubLayout Result = {10*4, 4};
+ return Result;
+}
+
+#if (defined(__POWERPC__) || defined (__ppc__) || defined(_POWER)) && \
+defined(__APPLE__)
+extern "C" void sys_icache_invalidate(const void *Addr, size_t len);
+#endif
+
+void *PPCJITInfo::emitFunctionStub(const Function* F, void *Fn,
+ JITCodeEmitter &JCE) {
+ // If this is just a call to an external function, emit a branch instead of a
+ // call. The code is the same except for one bit of the last instruction.
+ if (Fn != (void*)(intptr_t)PPC32CompilationCallback &&
+ Fn != (void*)(intptr_t)PPC64CompilationCallback) {
+ void *Addr = (void*)JCE.getCurrentPCValue();
+ JCE.emitWordBE(0);
+ JCE.emitWordBE(0);
+ JCE.emitWordBE(0);
+ JCE.emitWordBE(0);
+ JCE.emitWordBE(0);
+ JCE.emitWordBE(0);
+ JCE.emitWordBE(0);
+ EmitBranchToAt((intptr_t)Addr, (intptr_t)Fn, false, is64Bit);
+ sys::Memory::InvalidateInstructionCache(Addr, 7*4);
+ return Addr;
+ }
+
+ void *Addr = (void*)JCE.getCurrentPCValue();
+ if (is64Bit) {
+ JCE.emitWordBE(0xf821ffb1); // stdu r1,-80(r1)
+ JCE.emitWordBE(0x7d6802a6); // mflr r11
+ JCE.emitWordBE(0xf9610060); // std r11, 96(r1)
+ } else if (Subtarget.isDarwinABI()){
+ JCE.emitWordBE(0x9421ffe0); // stwu r1,-32(r1)
+ JCE.emitWordBE(0x7d6802a6); // mflr r11
+ JCE.emitWordBE(0x91610028); // stw r11, 40(r1)
+ } else {
+ JCE.emitWordBE(0x9421ffe0); // stwu r1,-32(r1)
+ JCE.emitWordBE(0x7d6802a6); // mflr r11
+ JCE.emitWordBE(0x91610024); // stw r11, 36(r1)
+ }
+ intptr_t BranchAddr = (intptr_t)JCE.getCurrentPCValue();
+ JCE.emitWordBE(0);
+ JCE.emitWordBE(0);
+ JCE.emitWordBE(0);
+ JCE.emitWordBE(0);
+ JCE.emitWordBE(0);
+ JCE.emitWordBE(0);
+ JCE.emitWordBE(0);
+ EmitBranchToAt(BranchAddr, (intptr_t)Fn, true, is64Bit);
+ sys::Memory::InvalidateInstructionCache(Addr, 10*4);
+ return Addr;
+}
+
+
+void PPCJITInfo::relocate(void *Function, MachineRelocation *MR,
+ unsigned NumRelocs, unsigned char* GOTBase) {
+ for (unsigned i = 0; i != NumRelocs; ++i, ++MR) {
+ unsigned *RelocPos = (unsigned*)Function + MR->getMachineCodeOffset()/4;
+ intptr_t ResultPtr = (intptr_t)MR->getResultPointer();
+ switch ((PPC::RelocationType)MR->getRelocationType()) {
+ default: llvm_unreachable("Unknown relocation type!");
+ case PPC::reloc_pcrel_bx:
+ // PC-relative relocation for b and bl instructions.
+ ResultPtr = (ResultPtr-(intptr_t)RelocPos) >> 2;
+ assert(ResultPtr >= -(1 << 23) && ResultPtr < (1 << 23) &&
+ "Relocation out of range!");
+ *RelocPos |= (ResultPtr & ((1 << 24)-1)) << 2;
+ break;
+ case PPC::reloc_pcrel_bcx:
+ // PC-relative relocation for BLT,BLE,BEQ,BGE,BGT,BNE, or other
+ // bcx instructions.
+ ResultPtr = (ResultPtr-(intptr_t)RelocPos) >> 2;
+ assert(ResultPtr >= -(1 << 13) && ResultPtr < (1 << 13) &&
+ "Relocation out of range!");
+ *RelocPos |= (ResultPtr & ((1 << 14)-1)) << 2;
+ break;
+ case PPC::reloc_absolute_high: // high bits of ref -> low 16 of instr
+ case PPC::reloc_absolute_low: { // low bits of ref -> low 16 of instr
+ ResultPtr += MR->getConstantVal();
+
+ // If this is a high-part access, get the high-part.
+ if (MR->getRelocationType() == PPC::reloc_absolute_high) {
+ // If the low part will have a carry (really a borrow) from the low
+ // 16-bits into the high 16, add a bit to borrow from.
+ if (((int)ResultPtr << 16) < 0)
+ ResultPtr += 1 << 16;
+ ResultPtr >>= 16;
+ }
+
+ // Do the addition then mask, so the addition does not overflow the 16-bit
+ // immediate section of the instruction.
+ unsigned LowBits = (*RelocPos + ResultPtr) & 65535;
+ unsigned HighBits = *RelocPos & ~65535;
+ *RelocPos = LowBits | HighBits; // Slam into low 16-bits
+ break;
+ }
+ case PPC::reloc_absolute_low_ix: { // low bits of ref -> low 14 of instr
+ ResultPtr += MR->getConstantVal();
+ // Do the addition then mask, so the addition does not overflow the 16-bit
+ // immediate section of the instruction.
+ unsigned LowBits = (*RelocPos + ResultPtr) & 0xFFFC;
+ unsigned HighBits = *RelocPos & 0xFFFF0003;
+ *RelocPos = LowBits | HighBits; // Slam into low 14-bits.
+ break;
+ }
+ }
+ }
+}
+
+void PPCJITInfo::replaceMachineCodeForFunction(void *Old, void *New) {
+ EmitBranchToAt((intptr_t)Old, (intptr_t)New, false, is64Bit);
+ sys::Memory::InvalidateInstructionCache(Old, 7*4);
+}
--- /dev/null
+//===-- PPCJITInfo.h - PowerPC impl. of the JIT interface -------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the PowerPC implementation of the TargetJITInfo class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef POWERPC_JITINFO_H
+#define POWERPC_JITINFO_H
+
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/Target/TargetJITInfo.h"
+
+namespace llvm {
+class PPCSubtarget;
+class PPCJITInfo : public TargetJITInfo {
+protected:
+ PPCSubtarget &Subtarget;
+ bool is64Bit;
+
+public:
+ PPCJITInfo(PPCSubtarget &STI);
+
+ StubLayout getStubLayout() override;
+ void *emitFunctionStub(const Function *F, void *Fn,
+ JITCodeEmitter &JCE) override;
+ LazyResolverFn getLazyResolverFunction(JITCompilerFn) override;
+ void relocate(void *Function, MachineRelocation *MR, unsigned NumRelocs,
+ unsigned char *GOTBase) override;
+
+ /// replaceMachineCodeForFunction - Make it so that calling the function
+ /// whose machine code is at OLD turns into a call to NEW, perhaps by
+ /// overwriting OLD with a branch to NEW. This is used for self-modifying
+ /// code.
+ ///
+ void replaceMachineCodeForFunction(void *Old, void *New) override;
+};
+}
+
+#endif
: PPCGenSubtargetInfo(TT, CPU, FS), IsPPC64(is64Bit), TargetTriple(TT),
OptLevel(OptLevel), TargetABI(PPC_ABI_UNKNOWN),
FrameLowering(initializeSubtargetDependencies(CPU, FS)),
- DL(getDataLayoutString(*this)), InstrInfo(*this),
+ DL(getDataLayoutString(*this)), InstrInfo(*this), JITInfo(*this),
TLInfo(TM), TSInfo(&DL) {}
+/// SetJITMode - This is called to inform the subtarget info that we are
+/// producing code for the JIT.
+void PPCSubtarget::SetJITMode() {
+ // JIT mode doesn't want lazy resolver stubs, it knows exactly where
+ // everything is. This matters for PPC64, which codegens in PIC mode without
+ // stubs.
+ HasLazyResolverStubs = false;
+
+ // Calls to external functions need to use indirect calls
+ IsJITCodeModel = true;
+}
+
void PPCSubtarget::resetSubtargetFeatures(const MachineFunction *MF) {
AttributeSet FnAttrs = MF->getFunction()->getAttributes();
Attribute CPUAttr = FnAttrs.getAttribute(AttributeSet::FunctionIndex,
DeprecatedMFTB = false;
DeprecatedDST = false;
HasLazyResolverStubs = false;
+ IsJITCodeModel = false;
}
void PPCSubtarget::resetSubtargetFeatures(StringRef CPU, StringRef FS) {
#include "PPCFrameLowering.h"
#include "PPCInstrInfo.h"
#include "PPCISelLowering.h"
+#include "PPCJITInfo.h"
#include "PPCSelectionDAGInfo.h"
#include "llvm/ADT/Triple.h"
#include "llvm/IR/DataLayout.h"
bool DeprecatedMFTB;
bool DeprecatedDST;
bool HasLazyResolverStubs;
+ bool IsJITCodeModel;
bool IsLittleEndian;
/// TargetTriple - What processor and OS we're targeting.
PPCFrameLowering FrameLowering;
const DataLayout DL;
PPCInstrInfo InstrInfo;
+ PPCJITInfo JITInfo;
PPCTargetLowering TLInfo;
PPCSelectionDAGInfo TSInfo;
/// subtarget options. Definition of function is auto generated by tblgen.
void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
+ /// SetJITMode - This is called to inform the subtarget info that we are
+ /// producing code for the JIT.
+ void SetJITMode();
+
/// getStackAlignment - Returns the minimum alignment known to hold of the
/// stack frame on entry to the function and which must be maintained by every
/// function for this subtarget.
}
const DataLayout *getDataLayout() const override { return &DL; }
const PPCInstrInfo *getInstrInfo() const override { return &InstrInfo; }
+ PPCJITInfo *getJITInfo() override { return &JITInfo; }
const PPCTargetLowering *getTargetLowering() const override {
return &TLInfo;
}
bool hasLazyResolverStub(const GlobalValue *GV,
const TargetMachine &TM) const;
+ // isJITCodeModel - True if we're generating code for the JIT
+ bool isJITCodeModel() const { return IsJITCodeModel; }
+
// isLittleEndian - True if generating little-endian code
bool isLittleEndian() const { return IsLittleEndian; }
return false;
}
+bool PPCTargetMachine::addCodeEmitter(PassManagerBase &PM,
+ JITCodeEmitter &JCE) {
+ // Inform the subtarget that we are in JIT mode. FIXME: does this break macho
+ // writing?
+ Subtarget.SetJITMode();
+
+ // Machine code emitter pass for PowerPC.
+ PM.add(createPPCJITCodeEmitterPass(*this, JCE));
+
+ return false;
+}
+
void PPCTargetMachine::addAnalysisPasses(PassManagerBase &PM) {
// Add first the target-independent BasicTTI pass, then our PPC pass. This
// allows the PPC pass to delegate to the target independent layer when
// Pass Pipeline Configuration
TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
+ bool addCodeEmitter(PassManagerBase &PM,
+ JITCodeEmitter &JCE) override;
/// \brief Register PPC analysis passes with a pass manager.
void addAnalysisPasses(PassManagerBase &PM) override;
tablegen(LLVM AMDGPUGenCallingConv.inc -gen-callingconv)
tablegen(LLVM AMDGPUGenSubtargetInfo.inc -gen-subtarget)
tablegen(LLVM AMDGPUGenIntrinsics.inc -gen-tgt-intrinsic)
-tablegen(LLVM AMDGPUGenMCCodeEmitter.inc -gen-emitter)
+tablegen(LLVM AMDGPUGenMCCodeEmitter.inc -gen-emitter -mc-emitter)
tablegen(LLVM AMDGPUGenDFAPacketizer.inc -gen-dfa-packetizer)
tablegen(LLVM AMDGPUGenAsmWriter.inc -gen-asm-writer)
add_public_tablegen_target(AMDGPUCommonTableGen)
tablegen(LLVM SparcGenRegisterInfo.inc -gen-register-info)
tablegen(LLVM SparcGenInstrInfo.inc -gen-instr-info)
+tablegen(LLVM SparcGenCodeEmitter.inc -gen-emitter)
tablegen(LLVM SparcGenDisassemblerTables.inc -gen-disassembler)
-tablegen(LLVM SparcGenMCCodeEmitter.inc -gen-emitter)
+tablegen(LLVM SparcGenMCCodeEmitter.inc -gen-emitter -mc-emitter)
tablegen(LLVM SparcGenAsmWriter.inc -gen-asm-writer)
tablegen(LLVM SparcGenAsmMatcher.inc -gen-asm-matcher)
tablegen(LLVM SparcGenDAGISel.inc -gen-dag-isel)
SparcSubtarget.cpp
SparcTargetMachine.cpp
SparcSelectionDAGInfo.cpp
+ SparcJITInfo.cpp
+ SparcCodeEmitter.cpp
SparcMCInstLower.cpp
SparcTargetObjectFile.cpp
)
SparcGenAsmWriter.inc SparcGenAsmMatcher.inc \
SparcGenDAGISel.inc SparcGenDisassemblerTables.inc \
SparcGenSubtargetInfo.inc SparcGenCallingConv.inc \
- SparcGenMCCodeEmitter.inc
+ SparcGenCodeEmitter.inc SparcGenMCCodeEmitter.inc
DIRS = InstPrinter AsmParser Disassembler TargetInfo MCTargetDesc
FunctionPass *createSparcISelDag(SparcTargetMachine &TM);
FunctionPass *createSparcDelaySlotFillerPass(TargetMachine &TM);
+ FunctionPass *createSparcJITCodeEmitterPass(SparcTargetMachine &TM,
+ JITCodeEmitter &JCE);
void LowerSparcMachineInstrToMCInst(const MachineInstr *MI,
MCInst &OutMI,
--- /dev/null
+//===-- Sparc/SparcCodeEmitter.cpp - Convert Sparc Code to Machine Code ---===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file contains the pass that transforms the Sparc machine instructions
+// into relocatable machine code.
+//
+//===---------------------------------------------------------------------===//
+
+#include "Sparc.h"
+#include "MCTargetDesc/SparcMCExpr.h"
+#include "SparcRelocations.h"
+#include "SparcTargetMachine.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+STATISTIC(NumEmitted, "Number of machine instructions emitted");
+
+namespace {
+
+class SparcCodeEmitter : public MachineFunctionPass {
+ SparcJITInfo *JTI;
+ const SparcInstrInfo *II;
+ const DataLayout *TD;
+ const SparcSubtarget *Subtarget;
+ TargetMachine &TM;
+ JITCodeEmitter &MCE;
+ const std::vector<MachineConstantPoolEntry> *MCPEs;
+ bool IsPIC;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<MachineModuleInfo> ();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ static char ID;
+
+public:
+ SparcCodeEmitter(TargetMachine &tm, JITCodeEmitter &mce)
+ : MachineFunctionPass(ID), JTI(nullptr), II(nullptr), TD(nullptr),
+ TM(tm), MCE(mce), MCPEs(nullptr),
+ IsPIC(TM.getRelocationModel() == Reloc::PIC_) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ const char *getPassName() const override {
+ return "Sparc Machine Code Emitter";
+ }
+
+ /// getBinaryCodeForInstr - This function, generated by the
+ /// CodeEmitterGenerator using TableGen, produces the binary encoding for
+ /// machine instructions.
+ uint64_t getBinaryCodeForInstr(const MachineInstr &MI) const;
+
+ void emitInstruction(MachineBasicBlock::instr_iterator MI,
+ MachineBasicBlock &MBB);
+
+private:
+ /// getMachineOpValue - Return binary encoding of operand. If the machine
+ /// operand requires relocation, record the relocation and return zero.
+ unsigned getMachineOpValue(const MachineInstr &MI,
+ const MachineOperand &MO) const;
+
+ unsigned getCallTargetOpValue(const MachineInstr &MI,
+ unsigned) const;
+ unsigned getBranchTargetOpValue(const MachineInstr &MI,
+ unsigned) const;
+ unsigned getBranchPredTargetOpValue(const MachineInstr &MI,
+ unsigned) const;
+ unsigned getBranchOnRegTargetOpValue(const MachineInstr &MI,
+ unsigned) const;
+
+ void emitWord(unsigned Word);
+
+ unsigned getRelocation(const MachineInstr &MI,
+ const MachineOperand &MO) const;
+
+ void emitGlobalAddress(const GlobalValue *GV, unsigned Reloc) const;
+ void emitExternalSymbolAddress(const char *ES, unsigned Reloc) const;
+ void emitConstPoolAddress(unsigned CPI, unsigned Reloc) const;
+ void emitMachineBasicBlock(MachineBasicBlock *BB, unsigned Reloc) const;
+};
+} // end anonymous namespace.
+
+char SparcCodeEmitter::ID = 0;
+
+bool SparcCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
+ SparcTargetMachine &Target = static_cast<SparcTargetMachine &>(
+ const_cast<TargetMachine &>(MF.getTarget()));
+
+ JTI = Target.getSubtargetImpl()->getJITInfo();
+ II = Target.getSubtargetImpl()->getInstrInfo();
+ TD = Target.getSubtargetImpl()->getDataLayout();
+ Subtarget = &TM.getSubtarget<SparcSubtarget>();
+ MCPEs = &MF.getConstantPool()->getConstants();
+ JTI->Initialize(MF, IsPIC);
+ MCE.setModuleInfo(&getAnalysis<MachineModuleInfo> ());
+
+ do {
+ DEBUG(errs() << "JITTing function '"
+ << MF.getName() << "'\n");
+ MCE.startFunction(MF);
+
+ for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
+ MBB != E; ++MBB){
+ MCE.StartMachineBasicBlock(MBB);
+ for (MachineBasicBlock::instr_iterator I = MBB->instr_begin(),
+ E = MBB->instr_end(); I != E;)
+ emitInstruction(*I++, *MBB);
+ }
+ } while (MCE.finishFunction(MF));
+
+ return false;
+}
+
+void SparcCodeEmitter::emitInstruction(MachineBasicBlock::instr_iterator MI,
+ MachineBasicBlock &MBB) {
+ DEBUG(errs() << "JIT: " << (void*)MCE.getCurrentPCValue() << ":\t" << *MI);
+
+ MCE.processDebugLoc(MI->getDebugLoc(), true);
+
+ ++NumEmitted;
+
+ switch (MI->getOpcode()) {
+ default: {
+ emitWord(getBinaryCodeForInstr(*MI));
+ break;
+ }
+ case TargetOpcode::INLINEASM: {
+ // We allow inline assembler nodes with empty bodies - they can
+ // implicitly define registers, which is ok for JIT.
+ if (MI->getOperand(0).getSymbolName()[0]) {
+ report_fatal_error("JIT does not support inline asm!");
+ }
+ break;
+ }
+ case TargetOpcode::CFI_INSTRUCTION:
+ break;
+ case TargetOpcode::EH_LABEL: {
+ MCE.emitLabel(MI->getOperand(0).getMCSymbol());
+ break;
+ }
+ case TargetOpcode::IMPLICIT_DEF:
+ case TargetOpcode::KILL: {
+ // Do nothing.
+ break;
+ }
+ case SP::GETPCX: {
+ report_fatal_error("JIT does not support pseudo instruction GETPCX yet!");
+ break;
+ }
+ }
+
+ MCE.processDebugLoc(MI->getDebugLoc(), false);
+}
+
+void SparcCodeEmitter::emitWord(unsigned Word) {
+ DEBUG(errs() << " 0x";
+ errs().write_hex(Word) << "\n");
+ MCE.emitWordBE(Word);
+}
+
+/// getMachineOpValue - Return binary encoding of operand. If the machine
+/// operand requires relocation, record the relocation and return zero.
+unsigned SparcCodeEmitter::getMachineOpValue(const MachineInstr &MI,
+ const MachineOperand &MO) const {
+ if (MO.isReg())
+ return TM.getSubtargetImpl()->getRegisterInfo()->getEncodingValue(
+ MO.getReg());
+ else if (MO.isImm())
+ return static_cast<unsigned>(MO.getImm());
+ else if (MO.isGlobal())
+ emitGlobalAddress(MO.getGlobal(), getRelocation(MI, MO));
+ else if (MO.isSymbol())
+ emitExternalSymbolAddress(MO.getSymbolName(), getRelocation(MI, MO));
+ else if (MO.isCPI())
+ emitConstPoolAddress(MO.getIndex(), getRelocation(MI, MO));
+ else if (MO.isMBB())
+ emitMachineBasicBlock(MO.getMBB(), getRelocation(MI, MO));
+ else
+ llvm_unreachable("Unable to encode MachineOperand!");
+ return 0;
+}
+unsigned SparcCodeEmitter::getCallTargetOpValue(const MachineInstr &MI,
+ unsigned opIdx) const {
+ const MachineOperand MO = MI.getOperand(opIdx);
+ return getMachineOpValue(MI, MO);
+}
+
+unsigned SparcCodeEmitter::getBranchTargetOpValue(const MachineInstr &MI,
+ unsigned opIdx) const {
+ const MachineOperand MO = MI.getOperand(opIdx);
+ return getMachineOpValue(MI, MO);
+}
+
+unsigned SparcCodeEmitter::getBranchPredTargetOpValue(const MachineInstr &MI,
+ unsigned opIdx) const {
+ const MachineOperand MO = MI.getOperand(opIdx);
+ return getMachineOpValue(MI, MO);
+}
+
+unsigned SparcCodeEmitter::getBranchOnRegTargetOpValue(const MachineInstr &MI,
+ unsigned opIdx) const {
+ const MachineOperand MO = MI.getOperand(opIdx);
+ return getMachineOpValue(MI, MO);
+}
+
+unsigned SparcCodeEmitter::getRelocation(const MachineInstr &MI,
+ const MachineOperand &MO) const {
+
+ unsigned TF = MO.getTargetFlags();
+ switch (TF) {
+ default:
+ case SparcMCExpr::VK_Sparc_None: break;
+ case SparcMCExpr::VK_Sparc_LO: return SP::reloc_sparc_lo;
+ case SparcMCExpr::VK_Sparc_HI: return SP::reloc_sparc_hi;
+ case SparcMCExpr::VK_Sparc_H44: return SP::reloc_sparc_h44;
+ case SparcMCExpr::VK_Sparc_M44: return SP::reloc_sparc_m44;
+ case SparcMCExpr::VK_Sparc_L44: return SP::reloc_sparc_l44;
+ case SparcMCExpr::VK_Sparc_HH: return SP::reloc_sparc_hh;
+ case SparcMCExpr::VK_Sparc_HM: return SP::reloc_sparc_hm;
+ }
+
+ unsigned Opc = MI.getOpcode();
+ switch (Opc) {
+ default: break;
+ case SP::CALL: return SP::reloc_sparc_pc30;
+ case SP::BA:
+ case SP::BCOND:
+ case SP::FBCOND: return SP::reloc_sparc_pc22;
+ case SP::BPXCC: return SP::reloc_sparc_pc19;
+ }
+ llvm_unreachable("unknown reloc!");
+}
+
+void SparcCodeEmitter::emitGlobalAddress(const GlobalValue *GV,
+ unsigned Reloc) const {
+ MCE.addRelocation(MachineRelocation::getGV(MCE.getCurrentPCOffset(), Reloc,
+ const_cast<GlobalValue *>(GV), 0,
+ true));
+}
+
+void SparcCodeEmitter::
+emitExternalSymbolAddress(const char *ES, unsigned Reloc) const {
+ MCE.addRelocation(MachineRelocation::getExtSym(MCE.getCurrentPCOffset(),
+ Reloc, ES, 0, 0));
+}
+
+void SparcCodeEmitter::
+emitConstPoolAddress(unsigned CPI, unsigned Reloc) const {
+ MCE.addRelocation(MachineRelocation::getConstPool(MCE.getCurrentPCOffset(),
+ Reloc, CPI, 0, false));
+}
+
+void SparcCodeEmitter::emitMachineBasicBlock(MachineBasicBlock *BB,
+ unsigned Reloc) const {
+ MCE.addRelocation(MachineRelocation::getBB(MCE.getCurrentPCOffset(),
+ Reloc, BB));
+}
+
+
+/// createSparcJITCodeEmitterPass - Return a pass that emits the collected Sparc
+/// code to the specified MCE object.
+FunctionPass *llvm::createSparcJITCodeEmitterPass(SparcTargetMachine &TM,
+ JITCodeEmitter &JCE) {
+ return new SparcCodeEmitter(TM, JCE);
+}
+
+#include "SparcGenCodeEmitter.inc"
--- /dev/null
+//===-- SparcJITInfo.cpp - Implement the Sparc JIT Interface --------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the JIT interfaces for the Sparc target.
+//
+//===----------------------------------------------------------------------===//
+#include "SparcJITInfo.h"
+#include "Sparc.h"
+#include "SparcRelocations.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/Support/Memory.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+/// JITCompilerFunction - This contains the address of the JIT function used to
+/// compile a function lazily.
+static TargetJITInfo::JITCompilerFn JITCompilerFunction;
+
+extern "C" void SparcCompilationCallback();
+
+extern "C" {
+#if defined (__sparc__)
+
+#if defined(__arch64__)
+#define FRAME_PTR(X) #X "+2047"
+#else
+#define FRAME_PTR(X) #X
+#endif
+
+ asm(
+ ".text\n"
+ "\t.align 4\n"
+ "\t.global SparcCompilationCallback\n"
+ "\t.type SparcCompilationCallback, #function\n"
+ "SparcCompilationCallback:\n"
+ // Save current register window and create stack.
+ // 128 (save area) + 6*8 (for arguments) + 16*8 (for float regfile) = 304
+ "\tsave %sp, -304, %sp\n"
+ // save float regfile to the stack.
+ "\tstd %f0, [" FRAME_PTR(%fp) "-0]\n"
+ "\tstd %f2, [" FRAME_PTR(%fp) "-8]\n"
+ "\tstd %f4, [" FRAME_PTR(%fp) "-16]\n"
+ "\tstd %f6, [" FRAME_PTR(%fp) "-24]\n"
+ "\tstd %f8, [" FRAME_PTR(%fp) "-32]\n"
+ "\tstd %f10, [" FRAME_PTR(%fp) "-40]\n"
+ "\tstd %f12, [" FRAME_PTR(%fp) "-48]\n"
+ "\tstd %f14, [" FRAME_PTR(%fp) "-56]\n"
+ "\tstd %f16, [" FRAME_PTR(%fp) "-64]\n"
+ "\tstd %f18, [" FRAME_PTR(%fp) "-72]\n"
+ "\tstd %f20, [" FRAME_PTR(%fp) "-80]\n"
+ "\tstd %f22, [" FRAME_PTR(%fp) "-88]\n"
+ "\tstd %f24, [" FRAME_PTR(%fp) "-96]\n"
+ "\tstd %f26, [" FRAME_PTR(%fp) "-104]\n"
+ "\tstd %f28, [" FRAME_PTR(%fp) "-112]\n"
+ "\tstd %f30, [" FRAME_PTR(%fp) "-120]\n"
+ // stubaddr is in %g1.
+ "\tcall SparcCompilationCallbackC\n"
+ "\t mov %g1, %o0\n"
+ // restore float regfile from the stack.
+ "\tldd [" FRAME_PTR(%fp) "-0], %f0\n"
+ "\tldd [" FRAME_PTR(%fp) "-8], %f2\n"
+ "\tldd [" FRAME_PTR(%fp) "-16], %f4\n"
+ "\tldd [" FRAME_PTR(%fp) "-24], %f6\n"
+ "\tldd [" FRAME_PTR(%fp) "-32], %f8\n"
+ "\tldd [" FRAME_PTR(%fp) "-40], %f10\n"
+ "\tldd [" FRAME_PTR(%fp) "-48], %f12\n"
+ "\tldd [" FRAME_PTR(%fp) "-56], %f14\n"
+ "\tldd [" FRAME_PTR(%fp) "-64], %f16\n"
+ "\tldd [" FRAME_PTR(%fp) "-72], %f18\n"
+ "\tldd [" FRAME_PTR(%fp) "-80], %f20\n"
+ "\tldd [" FRAME_PTR(%fp) "-88], %f22\n"
+ "\tldd [" FRAME_PTR(%fp) "-96], %f24\n"
+ "\tldd [" FRAME_PTR(%fp) "-104], %f26\n"
+ "\tldd [" FRAME_PTR(%fp) "-112], %f28\n"
+ "\tldd [" FRAME_PTR(%fp) "-120], %f30\n"
+ // restore original register window and
+ // copy %o0 to %g1
+ "\trestore %o0, 0, %g1\n"
+ // call the new stub
+ "\tjmp %g1\n"
+ "\t nop\n"
+ "\t.size SparcCompilationCallback, .-SparcCompilationCallback"
+ );
+#else
+ void SparcCompilationCallback() {
+ llvm_unreachable(
+ "Cannot call SparcCompilationCallback() on a non-sparc arch!");
+ }
+#endif
+}
+
+
+#define SETHI_INST(imm, rd) (0x01000000 | ((rd) << 25) | ((imm) & 0x3FFFFF))
+#define JMP_INST(rs1, imm, rd) (0x80000000 | ((rd) << 25) | (0x38 << 19) \
+ | ((rs1) << 14) | (1 << 13) | ((imm) & 0x1FFF))
+#define NOP_INST SETHI_INST(0, 0)
+#define OR_INST_I(rs1, imm, rd) (0x80000000 | ((rd) << 25) | (0x02 << 19) \
+ | ((rs1) << 14) | (1 << 13) | ((imm) & 0x1FFF))
+#define OR_INST_R(rs1, rs2, rd) (0x80000000 | ((rd) << 25) | (0x02 << 19) \
+ | ((rs1) << 14) | (0 << 13) | ((rs2) & 0x1F))
+#define RDPC_INST(rd) (0x80000000 | ((rd) << 25) | (0x28 << 19) \
+ | (5 << 14))
+#define LDX_INST(rs1, imm, rd) (0xC0000000 | ((rd) << 25) | (0x0B << 19) \
+ | ((rs1) << 14) | (1 << 13) | ((imm) & 0x1FFF))
+#define SLLX_INST(rs1, imm, rd) (0x80000000 | ((rd) << 25) | (0x25 << 19) \
+ | ((rs1) << 14) | (3 << 12) | ((imm) & 0x3F))
+#define SUB_INST(rs1, imm, rd) (0x80000000 | ((rd) << 25) | (0x04 << 19) \
+ | ((rs1) << 14) | (1 << 13) | ((imm) & 0x1FFF))
+#define XOR_INST(rs1, imm, rd) (0x80000000 | ((rd) << 25) | (0x03 << 19) \
+ | ((rs1) << 14) | (1 << 13) | ((imm) & 0x1FFF))
+#define BA_INST(tgt) (0x10800000 | ((tgt) & 0x3FFFFF))
+
+// Emit instructions to jump to Addr and store the starting address of
+// the instructions emitted in the scratch register.
+static void emitInstrForIndirectJump(intptr_t Addr,
+ unsigned scratch,
+ SmallVectorImpl<uint32_t> &Insts) {
+
+ if (isInt<13>(Addr)) {
+ // Emit: jmpl %g0+Addr, <scratch>
+ // nop
+ Insts.push_back(JMP_INST(0, LO10(Addr), scratch));
+ Insts.push_back(NOP_INST);
+ return;
+ }
+
+ if (isUInt<32>(Addr)) {
+ // Emit: sethi %hi(Addr), scratch
+ // jmpl scratch+%lo(Addr), scratch
+ // sub scratch, 4, scratch
+ Insts.push_back(SETHI_INST(HI22(Addr), scratch));
+ Insts.push_back(JMP_INST(scratch, LO10(Addr), scratch));
+ Insts.push_back(SUB_INST(scratch, 4, scratch));
+ return;
+ }
+
+ if (Addr < 0 && isInt<33>(Addr)) {
+ // Emit: sethi %hix(Addr), scratch)
+ // xor scratch, %lox(Addr), scratch
+ // jmpl scratch+0, scratch
+ // sub scratch, 8, scratch
+ Insts.push_back(SETHI_INST(HIX22(Addr), scratch));
+ Insts.push_back(XOR_INST(scratch, LOX10(Addr), scratch));
+ Insts.push_back(JMP_INST(scratch, 0, scratch));
+ Insts.push_back(SUB_INST(scratch, 8, scratch));
+ return;
+ }
+
+ // Emit: rd %pc, scratch
+ // ldx [scratch+16], scratch
+ // jmpl scratch+0, scratch
+ // sub scratch, 8, scratch
+ // <Addr: 8 byte>
+ Insts.push_back(RDPC_INST(scratch));
+ Insts.push_back(LDX_INST(scratch, 16, scratch));
+ Insts.push_back(JMP_INST(scratch, 0, scratch));
+ Insts.push_back(SUB_INST(scratch, 8, scratch));
+ Insts.push_back((uint32_t)(((int64_t)Addr) >> 32) & 0xffffffff);
+ Insts.push_back((uint32_t)(Addr & 0xffffffff));
+
+ // Instruction sequence without rdpc instruction
+ // 7 instruction and 2 scratch register
+ // Emit: sethi %hh(Addr), scratch
+ // or scratch, %hm(Addr), scratch
+ // sllx scratch, 32, scratch
+ // sethi %hi(Addr), scratch2
+ // or scratch, scratch2, scratch
+ // jmpl scratch+%lo(Addr), scratch
+ // sub scratch, 20, scratch
+ // Insts.push_back(SETHI_INST(HH22(Addr), scratch));
+ // Insts.push_back(OR_INST_I(scratch, HM10(Addr), scratch));
+ // Insts.push_back(SLLX_INST(scratch, 32, scratch));
+ // Insts.push_back(SETHI_INST(HI22(Addr), scratch2));
+ // Insts.push_back(OR_INST_R(scratch, scratch2, scratch));
+ // Insts.push_back(JMP_INST(scratch, LO10(Addr), scratch));
+ // Insts.push_back(SUB_INST(scratch, 20, scratch));
+}
+
+extern "C" void *SparcCompilationCallbackC(intptr_t StubAddr) {
+ // Get the address of the compiled code for this function.
+ intptr_t NewVal = (intptr_t) JITCompilerFunction((void*) StubAddr);
+
+ // Rewrite the function stub so that we don't end up here every time we
+ // execute the call. We're replacing the stub instructions with code
+ // that jumps to the compiled function:
+
+ SmallVector<uint32_t, 8> Insts;
+ intptr_t diff = (NewVal - StubAddr) >> 2;
+ if (isInt<22>(diff)) {
+ // Use branch instruction to jump
+ Insts.push_back(BA_INST(diff));
+ Insts.push_back(NOP_INST);
+ } else {
+ // Otherwise, use indirect jump to the compiled function
+ emitInstrForIndirectJump(NewVal, 1, Insts);
+ }
+
+ for (unsigned i = 0, e = Insts.size(); i != e; ++i)
+ *(uint32_t *)(StubAddr + i*4) = Insts[i];
+
+ sys::Memory::InvalidateInstructionCache((void*) StubAddr, Insts.size() * 4);
+ return (void*)StubAddr;
+}
+
+
+void SparcJITInfo::replaceMachineCodeForFunction(void *Old, void *New) {
+ llvm_unreachable("FIXME: Implement SparcJITInfo::"
+ "replaceMachineCodeForFunction");
+}
+
+
+TargetJITInfo::StubLayout SparcJITInfo::getStubLayout() {
+ // The stub contains maximum of 4 4-byte instructions and 8 bytes for address,
+ // aligned at 32 bytes.
+ // See emitFunctionStub and emitInstrForIndirectJump for details.
+ StubLayout Result = { 4*4 + 8, 32 };
+ return Result;
+}
+
+void *SparcJITInfo::emitFunctionStub(const Function *F, void *Fn,
+ JITCodeEmitter &JCE)
+{
+ JCE.emitAlignment(32);
+ void *Addr = (void*) (JCE.getCurrentPCValue());
+
+ intptr_t CurrentAddr = (intptr_t)Addr;
+ intptr_t EmittedAddr;
+ SmallVector<uint32_t, 8> Insts;
+ if (Fn != (void*)(intptr_t)SparcCompilationCallback) {
+ EmittedAddr = (intptr_t)Fn;
+ intptr_t diff = (EmittedAddr - CurrentAddr) >> 2;
+ if (isInt<22>(diff)) {
+ Insts.push_back(BA_INST(diff));
+ Insts.push_back(NOP_INST);
+ }
+ } else {
+ EmittedAddr = (intptr_t)SparcCompilationCallback;
+ }
+
+ if (Insts.size() == 0)
+ emitInstrForIndirectJump(EmittedAddr, 1, Insts);
+
+
+ if (!sys::Memory::setRangeWritable(Addr, 4 * Insts.size()))
+ llvm_unreachable("ERROR: Unable to mark stub writable.");
+
+ for (unsigned i = 0, e = Insts.size(); i != e; ++i)
+ JCE.emitWordBE(Insts[i]);
+
+ sys::Memory::InvalidateInstructionCache(Addr, 4 * Insts.size());
+ if (!sys::Memory::setRangeExecutable(Addr, 4 * Insts.size()))
+ llvm_unreachable("ERROR: Unable to mark stub executable.");
+
+ return Addr;
+}
+
+
+TargetJITInfo::LazyResolverFn
+SparcJITInfo::getLazyResolverFunction(JITCompilerFn F) {
+ JITCompilerFunction = F;
+ return SparcCompilationCallback;
+}
+
+/// relocate - Before the JIT can run a block of code that has been emitted,
+/// it must rewrite the code to contain the actual addresses of any
+/// referenced global symbols.
+void SparcJITInfo::relocate(void *Function, MachineRelocation *MR,
+ unsigned NumRelocs, unsigned char *GOTBase) {
+ for (unsigned i = 0; i != NumRelocs; ++i, ++MR) {
+ void *RelocPos = (char*) Function + MR->getMachineCodeOffset();
+ intptr_t ResultPtr = (intptr_t) MR->getResultPointer();
+
+ switch ((SP::RelocationType) MR->getRelocationType()) {
+ case SP::reloc_sparc_hi:
+ ResultPtr = (ResultPtr >> 10) & 0x3fffff;
+ break;
+
+ case SP::reloc_sparc_lo:
+ ResultPtr = (ResultPtr & 0x3ff);
+ break;
+
+ case SP::reloc_sparc_pc30:
+ ResultPtr = ((ResultPtr - (intptr_t)RelocPos) >> 2) & 0x3fffffff;
+ break;
+
+ case SP::reloc_sparc_pc22:
+ ResultPtr = ((ResultPtr - (intptr_t)RelocPos) >> 2) & 0x3fffff;
+ break;
+
+ case SP::reloc_sparc_pc19:
+ ResultPtr = ((ResultPtr - (intptr_t)RelocPos) >> 2) & 0x7ffff;
+ break;
+
+ case SP::reloc_sparc_h44:
+ ResultPtr = (ResultPtr >> 22) & 0x3fffff;
+ break;
+
+ case SP::reloc_sparc_m44:
+ ResultPtr = (ResultPtr >> 12) & 0x3ff;
+ break;
+
+ case SP::reloc_sparc_l44:
+ ResultPtr = (ResultPtr & 0xfff);
+ break;
+
+ case SP::reloc_sparc_hh:
+ ResultPtr = (((int64_t)ResultPtr) >> 42) & 0x3fffff;
+ break;
+
+ case SP::reloc_sparc_hm:
+ ResultPtr = (((int64_t)ResultPtr) >> 32) & 0x3ff;
+ break;
+
+ }
+ *((unsigned*) RelocPos) |= (unsigned) ResultPtr;
+ }
+}
--- /dev/null
+//==- SparcJITInfo.h - Sparc Implementation of the JIT Interface -*- C++ -*-==//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the declaration of the SparcJITInfo class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SPARCJITINFO_H
+#define SPARCJITINFO_H
+
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/Target/TargetJITInfo.h"
+
+namespace llvm {
+class SparcTargetMachine;
+
+class SparcJITInfo : public TargetJITInfo {
+
+ bool IsPIC;
+
+ public:
+ explicit SparcJITInfo()
+ : IsPIC(false) {}
+
+ /// replaceMachineCodeForFunction - Make it so that calling the function
+ /// whose machine code is at OLD turns into a call to NEW, perhaps by
+ /// overwriting OLD with a branch to NEW. This is used for self-modifying
+ /// code.
+ ///
+ void replaceMachineCodeForFunction(void *Old, void *New) override;
+
+ // getStubLayout - Returns the size and alignment of the largest call stub
+ // on Sparc.
+ StubLayout getStubLayout() override;
+
+
+ /// emitFunctionStub - Use the specified JITCodeEmitter object to emit a
+ /// small native function that simply calls the function at the specified
+ /// address.
+ void *emitFunctionStub(const Function *F, void *Fn,
+ JITCodeEmitter &JCE) override;
+
+ /// getLazyResolverFunction - Expose the lazy resolver to the JIT.
+ LazyResolverFn getLazyResolverFunction(JITCompilerFn) override;
+
+ /// relocate - Before the JIT can run a block of code that has been emitted,
+ /// it must rewrite the code to contain the actual addresses of any
+ /// referenced global symbols.
+ void relocate(void *Function, MachineRelocation *MR,
+ unsigned NumRelocs, unsigned char *GOTBase) override;
+
+ /// Initialize - Initialize internal stage for the function being JITted.
+ void Initialize(const MachineFunction &MF, bool isPIC) {
+ IsPIC = isPIC;
+ }
+
+};
+}
+
+#endif
#include "SparcFrameLowering.h"
#include "SparcInstrInfo.h"
#include "SparcISelLowering.h"
+#include "SparcJITInfo.h"
#include "SparcSelectionDAGInfo.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/Target/TargetFrameLowering.h"
SparcTargetLowering TLInfo;
SparcSelectionDAGInfo TSInfo;
SparcFrameLowering FrameLowering;
+ SparcJITInfo JITInfo;
public:
SparcSubtarget(const std::string &TT, const std::string &CPU,
const SparcSelectionDAGInfo *getSelectionDAGInfo() const override {
return &TSInfo;
}
+ SparcJITInfo *getJITInfo() override { return &JITInfo; }
const DataLayout *getDataLayout() const override { return &DL; }
bool isV9() const { return IsV9; }
return false;
}
+bool SparcTargetMachine::addCodeEmitter(PassManagerBase &PM,
+ JITCodeEmitter &JCE) {
+ // Machine code emitter pass for Sparc.
+ PM.add(createSparcJITCodeEmitterPass(*this, JCE));
+ return false;
+}
+
/// addPreEmitPass - This pass may be implemented by targets that want to run
/// passes immediately before machine code is emitted. This should return
/// true if -print-machineinstrs should print out the code after the passes.
const SparcSubtarget *getSubtargetImpl() const override { return &Subtarget; }
+ SparcSubtarget *getSubtargetImpl() {
+ return static_cast<SparcSubtarget *>(TargetMachine::getSubtargetImpl());
+ }
+
// Pass Pipeline Configuration
TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
+ bool addCodeEmitter(PassManagerBase &PM, JITCodeEmitter &JCE) override;
};
/// SparcV8TargetMachine - Sparc 32-bit target machine
tablegen(LLVM SystemZGenCallingConv.inc -gen-callingconv)
tablegen(LLVM SystemZGenDAGISel.inc -gen-dag-isel)
tablegen(LLVM SystemZGenDisassemblerTables.inc -gen-disassembler)
-tablegen(LLVM SystemZGenMCCodeEmitter.inc -gen-emitter)
+tablegen(LLVM SystemZGenMCCodeEmitter.inc -gen-emitter -mc-emitter)
tablegen(LLVM SystemZGenInstrInfo.inc -gen-instr-info)
tablegen(LLVM SystemZGenRegisterInfo.inc -gen-register-info)
tablegen(LLVM SystemZGenSubtargetInfo.inc -gen-subtarget)
BUILT_SOURCES = SystemZGenRegisterInfo.inc \
SystemZGenAsmWriter.inc \
SystemZGenAsmMatcher.inc \
+ SystemZGenCodeEmitter.inc \
SystemZGenDisassemblerTables.inc \
SystemZGenInstrInfo.inc \
SystemZGenDAGISel.inc \
--- /dev/null
+//===- Target/TargetJITInfo.h - Target Information for JIT ------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Target/TargetJITInfo.h"
+
+using namespace llvm;
+
+void TargetJITInfo::anchor() { }
set(sources
X86AsmPrinter.cpp
X86AtomicExpandPass.cpp
+ X86CodeEmitter.cpp
X86FastISel.cpp
X86FloatingPoint.cpp
X86FrameLowering.cpp
X86ISelDAGToDAG.cpp
X86ISelLowering.cpp
X86InstrInfo.cpp
+ X86JITInfo.cpp
X86MCInstLower.cpp
X86MachineFunctionInfo.cpp
X86PadShortFunction.cpp
class FunctionPass;
class ImmutablePass;
+class JITCodeEmitter;
class X86TargetMachine;
/// createX86AtomicExpandPass - This pass expands atomic operations that cannot
/// AVX and SSE.
FunctionPass *createX86IssueVZeroUpperPass();
+/// createX86CodeEmitterPass - Return a pass that emits the collected X86 code
+/// to the specified MCE object.
+FunctionPass *createX86JITCodeEmitterPass(X86TargetMachine &TM,
+ JITCodeEmitter &JCE);
+
/// createX86EmitCodeToMemory - Returns a pass that converts a register
/// allocated function into raw machine code in a dynamically
/// allocated chunk of memory.
--- /dev/null
+//===-- X86CodeEmitter.cpp - Convert X86 code to machine code -------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the pass that transforms the X86 machine instructions into
+// relocatable machine code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "X86.h"
+#include "X86InstrInfo.h"
+#include "X86JITInfo.h"
+#include "X86Relocations.h"
+#include "X86Subtarget.h"
+#include "X86TargetMachine.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/PassManager.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetOptions.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "x86-emitter"
+
+STATISTIC(NumEmitted, "Number of machine instructions emitted");
+
+namespace {
+ template<class CodeEmitter>
+ class Emitter : public MachineFunctionPass {
+ const X86InstrInfo *II;
+ const DataLayout *TD;
+ X86TargetMachine &TM;
+ CodeEmitter &MCE;
+ MachineModuleInfo *MMI;
+ intptr_t PICBaseOffset;
+ bool Is64BitMode;
+ bool IsPIC;
+ public:
+ static char ID;
+ explicit Emitter(X86TargetMachine &tm, CodeEmitter &mce)
+ : MachineFunctionPass(ID), II(nullptr), TD(nullptr), TM(tm),
+ MCE(mce), PICBaseOffset(0), Is64BitMode(false),
+ IsPIC(TM.getRelocationModel() == Reloc::PIC_) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ const char *getPassName() const override {
+ return "X86 Machine Code Emitter";
+ }
+
+ void emitOpcodePrefix(uint64_t TSFlags, int MemOperand,
+ const MachineInstr &MI,
+ const MCInstrDesc *Desc) const;
+
+ void emitVEXOpcodePrefix(uint64_t TSFlags, int MemOperand,
+ const MachineInstr &MI,
+ const MCInstrDesc *Desc) const;
+
+ void emitSegmentOverridePrefix(uint64_t TSFlags,
+ int MemOperand,
+ const MachineInstr &MI) const;
+
+ void emitInstruction(MachineInstr &MI, const MCInstrDesc *Desc);
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesAll();
+ AU.addRequired<MachineModuleInfo>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ private:
+ void emitPCRelativeBlockAddress(MachineBasicBlock *MBB);
+ void emitGlobalAddress(const GlobalValue *GV, unsigned Reloc,
+ intptr_t Disp = 0, intptr_t PCAdj = 0,
+ bool Indirect = false);
+ void emitExternalSymbolAddress(const char *ES, unsigned Reloc);
+ void emitConstPoolAddress(unsigned CPI, unsigned Reloc, intptr_t Disp = 0,
+ intptr_t PCAdj = 0);
+ void emitJumpTableAddress(unsigned JTI, unsigned Reloc,
+ intptr_t PCAdj = 0);
+
+ void emitDisplacementField(const MachineOperand *RelocOp, int DispVal,
+ intptr_t Adj = 0, bool IsPCRel = true);
+
+ void emitRegModRMByte(unsigned ModRMReg, unsigned RegOpcodeField);
+ void emitRegModRMByte(unsigned RegOpcodeField);
+ void emitSIBByte(unsigned SS, unsigned Index, unsigned Base);
+ void emitConstant(uint64_t Val, unsigned Size);
+
+ void emitMemModRMByte(const MachineInstr &MI,
+ unsigned Op, unsigned RegOpcodeField,
+ intptr_t PCAdj = 0);
+
+ unsigned getX86RegNum(unsigned RegNo) const {
+ const TargetRegisterInfo *TRI = TM.getSubtargetImpl()->getRegisterInfo();
+ return TRI->getEncodingValue(RegNo) & 0x7;
+ }
+
+ unsigned char getVEXRegisterEncoding(const MachineInstr &MI,
+ unsigned OpNum) const;
+ };
+
+template<class CodeEmitter>
+ char Emitter<CodeEmitter>::ID = 0;
+} // end anonymous namespace.
+
+/// createX86CodeEmitterPass - Return a pass that emits the collected X86 code
+/// to the specified JITCodeEmitter object.
+FunctionPass *llvm::createX86JITCodeEmitterPass(X86TargetMachine &TM,
+ JITCodeEmitter &JCE) {
+ return new Emitter<JITCodeEmitter>(TM, JCE);
+}
+
+template<class CodeEmitter>
+bool Emitter<CodeEmitter>::runOnMachineFunction(MachineFunction &MF) {
+ MMI = &getAnalysis<MachineModuleInfo>();
+ MCE.setModuleInfo(MMI);
+
+ II = TM.getSubtargetImpl()->getInstrInfo();
+ TD = TM.getSubtargetImpl()->getDataLayout();
+ Is64BitMode = TM.getSubtarget<X86Subtarget>().is64Bit();
+ IsPIC = TM.getRelocationModel() == Reloc::PIC_;
+
+ do {
+ DEBUG(dbgs() << "JITTing function '" << MF.getName() << "'\n");
+ MCE.startFunction(MF);
+ for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
+ MBB != E; ++MBB) {
+ MCE.StartMachineBasicBlock(MBB);
+ for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
+ I != E; ++I) {
+ const MCInstrDesc &Desc = I->getDesc();
+ emitInstruction(*I, &Desc);
+ // MOVPC32r is basically a call plus a pop instruction.
+ if (Desc.getOpcode() == X86::MOVPC32r)
+ emitInstruction(*I, &II->get(X86::POP32r));
+ ++NumEmitted; // Keep track of the # of mi's emitted
+ }
+ }
+ } while (MCE.finishFunction(MF));
+
+ return false;
+}
+
+/// determineREX - Determine if the MachineInstr has to be encoded with a X86-64
+/// REX prefix which specifies 1) 64-bit instructions, 2) non-default operand
+/// size, and 3) use of X86-64 extended registers.
+static unsigned determineREX(const MachineInstr &MI) {
+ unsigned REX = 0;
+ const MCInstrDesc &Desc = MI.getDesc();
+
+ // Pseudo instructions do not need REX prefix byte.
+ if ((Desc.TSFlags & X86II::FormMask) == X86II::Pseudo)
+ return 0;
+ if (Desc.TSFlags & X86II::REX_W)
+ REX |= 1 << 3;
+
+ unsigned NumOps = Desc.getNumOperands();
+ if (NumOps) {
+ bool isTwoAddr = NumOps > 1 &&
+ Desc.getOperandConstraint(1, MCOI::TIED_TO) != -1;
+
+ // If it accesses SPL, BPL, SIL, or DIL, then it requires a 0x40 REX prefix.
+ unsigned i = isTwoAddr ? 1 : 0;
+ for (unsigned e = NumOps; i != e; ++i) {
+ const MachineOperand& MO = MI.getOperand(i);
+ if (MO.isReg()) {
+ unsigned Reg = MO.getReg();
+ if (X86II::isX86_64NonExtLowByteReg(Reg))
+ REX |= 0x40;
+ }
+ }
+
+ switch (Desc.TSFlags & X86II::FormMask) {
+ case X86II::MRMSrcReg: {
+ if (X86InstrInfo::isX86_64ExtendedReg(MI.getOperand(0)))
+ REX |= 1 << 2;
+ i = isTwoAddr ? 2 : 1;
+ for (unsigned e = NumOps; i != e; ++i) {
+ const MachineOperand& MO = MI.getOperand(i);
+ if (X86InstrInfo::isX86_64ExtendedReg(MO))
+ REX |= 1 << 0;
+ }
+ break;
+ }
+ case X86II::MRMSrcMem: {
+ if (X86InstrInfo::isX86_64ExtendedReg(MI.getOperand(0)))
+ REX |= 1 << 2;
+ unsigned Bit = 0;
+ i = isTwoAddr ? 2 : 1;
+ for (; i != NumOps; ++i) {
+ const MachineOperand& MO = MI.getOperand(i);
+ if (MO.isReg()) {
+ if (X86InstrInfo::isX86_64ExtendedReg(MO))
+ REX |= 1 << Bit;
+ Bit++;
+ }
+ }
+ break;
+ }
+ case X86II::MRMXm:
+ case X86II::MRM0m: case X86II::MRM1m:
+ case X86II::MRM2m: case X86II::MRM3m:
+ case X86II::MRM4m: case X86II::MRM5m:
+ case X86II::MRM6m: case X86II::MRM7m:
+ case X86II::MRMDestMem: {
+ unsigned e = (isTwoAddr ? X86::AddrNumOperands+1 : X86::AddrNumOperands);
+ i = isTwoAddr ? 1 : 0;
+ if (NumOps > e && X86InstrInfo::isX86_64ExtendedReg(MI.getOperand(e)))
+ REX |= 1 << 2;
+ unsigned Bit = 0;
+ for (; i != e; ++i) {
+ const MachineOperand& MO = MI.getOperand(i);
+ if (MO.isReg()) {
+ if (X86InstrInfo::isX86_64ExtendedReg(MO))
+ REX |= 1 << Bit;
+ Bit++;
+ }
+ }
+ break;
+ }
+ default: {
+ if (X86InstrInfo::isX86_64ExtendedReg(MI.getOperand(0)))
+ REX |= 1 << 0;
+ i = isTwoAddr ? 2 : 1;
+ for (unsigned e = NumOps; i != e; ++i) {
+ const MachineOperand& MO = MI.getOperand(i);
+ if (X86InstrInfo::isX86_64ExtendedReg(MO))
+ REX |= 1 << 2;
+ }
+ break;
+ }
+ }
+ }
+ return REX;
+}
+
+
+/// emitPCRelativeBlockAddress - This method keeps track of the information
+/// necessary to resolve the address of this block later and emits a dummy
+/// value.
+///
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitPCRelativeBlockAddress(MachineBasicBlock *MBB) {
+ // Remember where this reference was and where it is to so we can
+ // deal with it later.
+ MCE.addRelocation(MachineRelocation::getBB(MCE.getCurrentPCOffset(),
+ X86::reloc_pcrel_word, MBB));
+ MCE.emitWordLE(0);
+}
+
+/// emitGlobalAddress - Emit the specified address to the code stream assuming
+/// this is part of a "take the address of a global" instruction.
+///
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitGlobalAddress(const GlobalValue *GV,
+ unsigned Reloc,
+ intptr_t Disp /* = 0 */,
+ intptr_t PCAdj /* = 0 */,
+ bool Indirect /* = false */) {
+ intptr_t RelocCST = Disp;
+ if (Reloc == X86::reloc_picrel_word)
+ RelocCST = PICBaseOffset;
+ else if (Reloc == X86::reloc_pcrel_word)
+ RelocCST = PCAdj;
+ MachineRelocation MR = Indirect
+ ? MachineRelocation::getIndirectSymbol(MCE.getCurrentPCOffset(), Reloc,
+ const_cast<GlobalValue *>(GV),
+ RelocCST, false)
+ : MachineRelocation::getGV(MCE.getCurrentPCOffset(), Reloc,
+ const_cast<GlobalValue *>(GV), RelocCST, false);
+ MCE.addRelocation(MR);
+ // The relocated value will be added to the displacement
+ if (Reloc == X86::reloc_absolute_dword)
+ MCE.emitDWordLE(Disp);
+ else
+ MCE.emitWordLE((int32_t)Disp);
+}
+
+/// emitExternalSymbolAddress - Arrange for the address of an external symbol to
+/// be emitted to the current location in the function, and allow it to be PC
+/// relative.
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitExternalSymbolAddress(const char *ES,
+ unsigned Reloc) {
+ intptr_t RelocCST = (Reloc == X86::reloc_picrel_word) ? PICBaseOffset : 0;
+
+ // X86 never needs stubs because instruction selection will always pick
+ // an instruction sequence that is large enough to hold any address
+ // to a symbol.
+ // (see X86ISelLowering.cpp, near 2039: X86TargetLowering::LowerCall)
+ bool NeedStub = false;
+ MCE.addRelocation(MachineRelocation::getExtSym(MCE.getCurrentPCOffset(),
+ Reloc, ES, RelocCST,
+ 0, NeedStub));
+ if (Reloc == X86::reloc_absolute_dword)
+ MCE.emitDWordLE(0);
+ else
+ MCE.emitWordLE(0);
+}
+
+/// emitConstPoolAddress - Arrange for the address of an constant pool
+/// to be emitted to the current location in the function, and allow it to be PC
+/// relative.
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitConstPoolAddress(unsigned CPI, unsigned Reloc,
+ intptr_t Disp /* = 0 */,
+ intptr_t PCAdj /* = 0 */) {
+ intptr_t RelocCST = 0;
+ if (Reloc == X86::reloc_picrel_word)
+ RelocCST = PICBaseOffset;
+ else if (Reloc == X86::reloc_pcrel_word)
+ RelocCST = PCAdj;
+ MCE.addRelocation(MachineRelocation::getConstPool(MCE.getCurrentPCOffset(),
+ Reloc, CPI, RelocCST));
+ // The relocated value will be added to the displacement
+ if (Reloc == X86::reloc_absolute_dword)
+ MCE.emitDWordLE(Disp);
+ else
+ MCE.emitWordLE((int32_t)Disp);
+}
+
+/// emitJumpTableAddress - Arrange for the address of a jump table to
+/// be emitted to the current location in the function, and allow it to be PC
+/// relative.
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitJumpTableAddress(unsigned JTI, unsigned Reloc,
+ intptr_t PCAdj /* = 0 */) {
+ intptr_t RelocCST = 0;
+ if (Reloc == X86::reloc_picrel_word)
+ RelocCST = PICBaseOffset;
+ else if (Reloc == X86::reloc_pcrel_word)
+ RelocCST = PCAdj;
+ MCE.addRelocation(MachineRelocation::getJumpTable(MCE.getCurrentPCOffset(),
+ Reloc, JTI, RelocCST));
+ // The relocated value will be added to the displacement
+ if (Reloc == X86::reloc_absolute_dword)
+ MCE.emitDWordLE(0);
+ else
+ MCE.emitWordLE(0);
+}
+
+inline static unsigned char ModRMByte(unsigned Mod, unsigned RegOpcode,
+ unsigned RM) {
+ assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
+ return RM | (RegOpcode << 3) | (Mod << 6);
+}
+
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitRegModRMByte(unsigned ModRMReg,
+ unsigned RegOpcodeFld){
+ MCE.emitByte(ModRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)));
+}
+
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitRegModRMByte(unsigned RegOpcodeFld) {
+ MCE.emitByte(ModRMByte(3, RegOpcodeFld, 0));
+}
+
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitSIBByte(unsigned SS,
+ unsigned Index,
+ unsigned Base) {
+ // SIB byte is in the same format as the ModRMByte...
+ MCE.emitByte(ModRMByte(SS, Index, Base));
+}
+
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitConstant(uint64_t Val, unsigned Size) {
+ // Output the constant in little endian byte order...
+ for (unsigned i = 0; i != Size; ++i) {
+ MCE.emitByte(Val & 255);
+ Val >>= 8;
+ }
+}
+
+/// isDisp8 - Return true if this signed displacement fits in a 8-bit
+/// sign-extended field.
+static bool isDisp8(int Value) {
+ return Value == (signed char)Value;
+}
+
+static bool gvNeedsNonLazyPtr(const MachineOperand &GVOp,
+ const TargetMachine &TM) {
+ // For Darwin-64, simulate the linktime GOT by using the same non-lazy-pointer
+ // mechanism as 32-bit mode.
+ if (TM.getSubtarget<X86Subtarget>().is64Bit() &&
+ !TM.getSubtarget<X86Subtarget>().isTargetDarwin())
+ return false;
+
+ // Return true if this is a reference to a stub containing the address of the
+ // global, not the global itself.
+ return isGlobalStubReference(GVOp.getTargetFlags());
+}
+
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitDisplacementField(const MachineOperand *RelocOp,
+ int DispVal,
+ intptr_t Adj /* = 0 */,
+ bool IsPCRel /* = true */) {
+ // If this is a simple integer displacement that doesn't require a relocation,
+ // emit it now.
+ if (!RelocOp) {
+ emitConstant(DispVal, 4);
+ return;
+ }
+
+ // Otherwise, this is something that requires a relocation. Emit it as such
+ // now.
+ unsigned RelocType = Is64BitMode ?
+ (IsPCRel ? X86::reloc_pcrel_word : X86::reloc_absolute_word_sext)
+ : (IsPIC ? X86::reloc_picrel_word : X86::reloc_absolute_word);
+ if (RelocOp->isGlobal()) {
+ // In 64-bit static small code model, we could potentially emit absolute.
+ // But it's probably not beneficial. If the MCE supports using RIP directly
+ // do it, otherwise fallback to absolute (this is determined by IsPCRel).
+ // 89 05 00 00 00 00 mov %eax,0(%rip) # PC-relative
+ // 89 04 25 00 00 00 00 mov %eax,0x0 # Absolute
+ bool Indirect = gvNeedsNonLazyPtr(*RelocOp, TM);
+ emitGlobalAddress(RelocOp->getGlobal(), RelocType, RelocOp->getOffset(),
+ Adj, Indirect);
+ } else if (RelocOp->isSymbol()) {
+ emitExternalSymbolAddress(RelocOp->getSymbolName(), RelocType);
+ } else if (RelocOp->isCPI()) {
+ emitConstPoolAddress(RelocOp->getIndex(), RelocType,
+ RelocOp->getOffset(), Adj);
+ } else {
+ assert(RelocOp->isJTI() && "Unexpected machine operand!");
+ emitJumpTableAddress(RelocOp->getIndex(), RelocType, Adj);
+ }
+}
+
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitMemModRMByte(const MachineInstr &MI,
+ unsigned Op,unsigned RegOpcodeField,
+ intptr_t PCAdj) {
+ const MachineOperand &Op3 = MI.getOperand(Op+3);
+ int DispVal = 0;
+ const MachineOperand *DispForReloc = nullptr;
+
+ // Figure out what sort of displacement we have to handle here.
+ if (Op3.isGlobal()) {
+ DispForReloc = &Op3;
+ } else if (Op3.isSymbol()) {
+ DispForReloc = &Op3;
+ } else if (Op3.isCPI()) {
+ if (!MCE.earlyResolveAddresses() || Is64BitMode || IsPIC) {
+ DispForReloc = &Op3;
+ } else {
+ DispVal += MCE.getConstantPoolEntryAddress(Op3.getIndex());
+ DispVal += Op3.getOffset();
+ }
+ } else if (Op3.isJTI()) {
+ if (!MCE.earlyResolveAddresses() || Is64BitMode || IsPIC) {
+ DispForReloc = &Op3;
+ } else {
+ DispVal += MCE.getJumpTableEntryAddress(Op3.getIndex());
+ }
+ } else {
+ DispVal = Op3.getImm();
+ }
+
+ const MachineOperand &Base = MI.getOperand(Op);
+ const MachineOperand &Scale = MI.getOperand(Op+1);
+ const MachineOperand &IndexReg = MI.getOperand(Op+2);
+
+ unsigned BaseReg = Base.getReg();
+
+ // Handle %rip relative addressing.
+ if (BaseReg == X86::RIP ||
+ (Is64BitMode && DispForReloc)) { // [disp32+RIP] in X86-64 mode
+ assert(IndexReg.getReg() == 0 && Is64BitMode &&
+ "Invalid rip-relative address");
+ MCE.emitByte(ModRMByte(0, RegOpcodeField, 5));
+ emitDisplacementField(DispForReloc, DispVal, PCAdj, true);
+ return;
+ }
+
+ // Indicate that the displacement will use an pcrel or absolute reference
+ // by default. MCEs able to resolve addresses on-the-fly use pcrel by default
+ // while others, unless explicit asked to use RIP, use absolute references.
+ bool IsPCRel = MCE.earlyResolveAddresses() ? true : false;
+
+ // Is a SIB byte needed?
+ // If no BaseReg, issue a RIP relative instruction only if the MCE can
+ // resolve addresses on-the-fly, otherwise use SIB (Intel Manual 2A, table
+ // 2-7) and absolute references.
+ unsigned BaseRegNo = -1U;
+ if (BaseReg != 0 && BaseReg != X86::RIP)
+ BaseRegNo = getX86RegNum(BaseReg);
+
+ if (// The SIB byte must be used if there is an index register.
+ IndexReg.getReg() == 0 &&
+ // The SIB byte must be used if the base is ESP/RSP/R12, all of which
+ // encode to an R/M value of 4, which indicates that a SIB byte is
+ // present.
+ BaseRegNo != N86::ESP &&
+ // If there is no base register and we're in 64-bit mode, we need a SIB
+ // byte to emit an addr that is just 'disp32' (the non-RIP relative form).
+ (!Is64BitMode || BaseReg != 0)) {
+ if (BaseReg == 0 || // [disp32] in X86-32 mode
+ BaseReg == X86::RIP) { // [disp32+RIP] in X86-64 mode
+ MCE.emitByte(ModRMByte(0, RegOpcodeField, 5));
+ emitDisplacementField(DispForReloc, DispVal, PCAdj, true);
+ return;
+ }
+
+ // If the base is not EBP/ESP and there is no displacement, use simple
+ // indirect register encoding, this handles addresses like [EAX]. The
+ // encoding for [EBP] with no displacement means [disp32] so we handle it
+ // by emitting a displacement of 0 below.
+ if (!DispForReloc && DispVal == 0 && BaseRegNo != N86::EBP) {
+ MCE.emitByte(ModRMByte(0, RegOpcodeField, BaseRegNo));
+ return;
+ }
+
+ // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
+ if (!DispForReloc && isDisp8(DispVal)) {
+ MCE.emitByte(ModRMByte(1, RegOpcodeField, BaseRegNo));
+ emitConstant(DispVal, 1);
+ return;
+ }
+
+ // Otherwise, emit the most general non-SIB encoding: [REG+disp32]
+ MCE.emitByte(ModRMByte(2, RegOpcodeField, BaseRegNo));
+ emitDisplacementField(DispForReloc, DispVal, PCAdj, IsPCRel);
+ return;
+ }
+
+ // Otherwise we need a SIB byte, so start by outputting the ModR/M byte first.
+ assert(IndexReg.getReg() != X86::ESP &&
+ IndexReg.getReg() != X86::RSP && "Cannot use ESP as index reg!");
+
+ bool ForceDisp32 = false;
+ bool ForceDisp8 = false;
+ if (BaseReg == 0) {
+ // If there is no base register, we emit the special case SIB byte with
+ // MOD=0, BASE=4, to JUST get the index, scale, and displacement.
+ MCE.emitByte(ModRMByte(0, RegOpcodeField, 4));
+ ForceDisp32 = true;
+ } else if (DispForReloc) {
+ // Emit the normal disp32 encoding.
+ MCE.emitByte(ModRMByte(2, RegOpcodeField, 4));
+ ForceDisp32 = true;
+ } else if (DispVal == 0 && BaseRegNo != N86::EBP) {
+ // Emit no displacement ModR/M byte
+ MCE.emitByte(ModRMByte(0, RegOpcodeField, 4));
+ } else if (isDisp8(DispVal)) {
+ // Emit the disp8 encoding...
+ MCE.emitByte(ModRMByte(1, RegOpcodeField, 4));
+ ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP
+ } else {
+ // Emit the normal disp32 encoding...
+ MCE.emitByte(ModRMByte(2, RegOpcodeField, 4));
+ }
+
+ // Calculate what the SS field value should be...
+ static const unsigned SSTable[] = { ~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3 };
+ unsigned SS = SSTable[Scale.getImm()];
+
+ if (BaseReg == 0) {
+ // Handle the SIB byte for the case where there is no base, see Intel
+ // Manual 2A, table 2-7. The displacement has already been output.
+ unsigned IndexRegNo;
+ if (IndexReg.getReg())
+ IndexRegNo = getX86RegNum(IndexReg.getReg());
+ else // Examples: [ESP+1*<noreg>+4] or [scaled idx]+disp32 (MOD=0,BASE=5)
+ IndexRegNo = 4;
+ emitSIBByte(SS, IndexRegNo, 5);
+ } else {
+ unsigned BaseRegNo = getX86RegNum(BaseReg);
+ unsigned IndexRegNo;
+ if (IndexReg.getReg())
+ IndexRegNo = getX86RegNum(IndexReg.getReg());
+ else
+ IndexRegNo = 4; // For example [ESP+1*<noreg>+4]
+ emitSIBByte(SS, IndexRegNo, BaseRegNo);
+ }
+
+ // Do we need to output a displacement?
+ if (ForceDisp8) {
+ emitConstant(DispVal, 1);
+ } else if (DispVal != 0 || ForceDisp32) {
+ emitDisplacementField(DispForReloc, DispVal, PCAdj, IsPCRel);
+ }
+}
+
+static const MCInstrDesc *UpdateOp(MachineInstr &MI, const X86InstrInfo *II,
+ unsigned Opcode) {
+ const MCInstrDesc *Desc = &II->get(Opcode);
+ MI.setDesc(*Desc);
+ return Desc;
+}
+
+/// Is16BitMemOperand - Return true if the specified instruction has
+/// a 16-bit memory operand. Op specifies the operand # of the memoperand.
+static bool Is16BitMemOperand(const MachineInstr &MI, unsigned Op) {
+ const MachineOperand &BaseReg = MI.getOperand(Op+X86::AddrBaseReg);
+ const MachineOperand &IndexReg = MI.getOperand(Op+X86::AddrIndexReg);
+
+ if ((BaseReg.getReg() != 0 &&
+ X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg.getReg())) ||
+ (IndexReg.getReg() != 0 &&
+ X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg.getReg())))
+ return true;
+ return false;
+}
+
+/// Is32BitMemOperand - Return true if the specified instruction has
+/// a 32-bit memory operand. Op specifies the operand # of the memoperand.
+static bool Is32BitMemOperand(const MachineInstr &MI, unsigned Op) {
+ const MachineOperand &BaseReg = MI.getOperand(Op+X86::AddrBaseReg);
+ const MachineOperand &IndexReg = MI.getOperand(Op+X86::AddrIndexReg);
+
+ if ((BaseReg.getReg() != 0 &&
+ X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg.getReg())) ||
+ (IndexReg.getReg() != 0 &&
+ X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg.getReg())))
+ return true;
+ return false;
+}
+
+/// Is64BitMemOperand - Return true if the specified instruction has
+/// a 64-bit memory operand. Op specifies the operand # of the memoperand.
+#ifndef NDEBUG
+static bool Is64BitMemOperand(const MachineInstr &MI, unsigned Op) {
+ const MachineOperand &BaseReg = MI.getOperand(Op+X86::AddrBaseReg);
+ const MachineOperand &IndexReg = MI.getOperand(Op+X86::AddrIndexReg);
+
+ if ((BaseReg.getReg() != 0 &&
+ X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg.getReg())) ||
+ (IndexReg.getReg() != 0 &&
+ X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg.getReg())))
+ return true;
+ return false;
+}
+#endif
+
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitOpcodePrefix(uint64_t TSFlags,
+ int MemOperand,
+ const MachineInstr &MI,
+ const MCInstrDesc *Desc) const {
+ // Emit the operand size opcode prefix as needed.
+ if (((TSFlags & X86II::OpSizeMask) >> X86II::OpSizeShift) == X86II::OpSize16)
+ MCE.emitByte(0x66);
+
+ switch (Desc->TSFlags & X86II::OpPrefixMask) {
+ case X86II::PD: // 66
+ MCE.emitByte(0x66);
+ break;
+ case X86II::XS: // F3
+ MCE.emitByte(0xF3);
+ break;
+ case X86II::XD: // F2
+ MCE.emitByte(0xF2);
+ break;
+ }
+
+ // Handle REX prefix.
+ if (Is64BitMode) {
+ if (unsigned REX = determineREX(MI))
+ MCE.emitByte(0x40 | REX);
+ }
+
+ // 0x0F escape code must be emitted just before the opcode.
+ switch (Desc->TSFlags & X86II::OpMapMask) {
+ case X86II::TB: // Two-byte opcode map
+ case X86II::T8: // 0F 38
+ case X86II::TA: // 0F 3A
+ MCE.emitByte(0x0F);
+ break;
+ }
+
+ switch (Desc->TSFlags & X86II::OpMapMask) {
+ case X86II::T8: // 0F 38
+ MCE.emitByte(0x38);
+ break;
+ case X86II::TA: // 0F 3A
+ MCE.emitByte(0x3A);
+ break;
+ }
+}
+
+// On regular x86, both XMM0-XMM7 and XMM8-XMM15 are encoded in the range
+// 0-7 and the difference between the 2 groups is given by the REX prefix.
+// In the VEX prefix, registers are seen sequencially from 0-15 and encoded
+// in 1's complement form, example:
+//
+// ModRM field => XMM9 => 1
+// VEX.VVVV => XMM9 => ~9
+//
+// See table 4-35 of Intel AVX Programming Reference for details.
+template<class CodeEmitter>
+unsigned char
+Emitter<CodeEmitter>::getVEXRegisterEncoding(const MachineInstr &MI,
+ unsigned OpNum) const {
+ unsigned SrcReg = MI.getOperand(OpNum).getReg();
+ unsigned SrcRegNum = getX86RegNum(MI.getOperand(OpNum).getReg());
+ if (X86II::isX86_64ExtendedReg(SrcReg))
+ SrcRegNum |= 8;
+
+ // The registers represented through VEX_VVVV should
+ // be encoded in 1's complement form.
+ return (~SrcRegNum) & 0xf;
+}
+
+/// EmitSegmentOverridePrefix - Emit segment override opcode prefix as needed
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitSegmentOverridePrefix(uint64_t TSFlags,
+ int MemOperand,
+ const MachineInstr &MI) const {
+ if (MemOperand < 0)
+ return; // No memory operand
+
+ // Check for explicit segment override on memory operand.
+ switch (MI.getOperand(MemOperand+X86::AddrSegmentReg).getReg()) {
+ default: llvm_unreachable("Unknown segment register!");
+ case 0: break;
+ case X86::CS: MCE.emitByte(0x2E); break;
+ case X86::SS: MCE.emitByte(0x36); break;
+ case X86::DS: MCE.emitByte(0x3E); break;
+ case X86::ES: MCE.emitByte(0x26); break;
+ case X86::FS: MCE.emitByte(0x64); break;
+ case X86::GS: MCE.emitByte(0x65); break;
+ }
+}
+
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitVEXOpcodePrefix(uint64_t TSFlags,
+ int MemOperand,
+ const MachineInstr &MI,
+ const MCInstrDesc *Desc) const {
+ unsigned char Encoding = (TSFlags & X86II::EncodingMask) >>
+ X86II::EncodingShift;
+ bool HasVEX_4V = (TSFlags >> X86II::VEXShift) & X86II::VEX_4V;
+ bool HasVEX_4VOp3 = (TSFlags >> X86II::VEXShift) & X86II::VEX_4VOp3;
+ bool HasMemOp4 = (TSFlags >> X86II::VEXShift) & X86II::MemOp4;
+
+ // VEX_R: opcode externsion equivalent to REX.R in
+ // 1's complement (inverted) form
+ //
+ // 1: Same as REX_R=0 (must be 1 in 32-bit mode)
+ // 0: Same as REX_R=1 (64 bit mode only)
+ //
+ unsigned char VEX_R = 0x1;
+
+ // VEX_X: equivalent to REX.X, only used when a
+ // register is used for index in SIB Byte.
+ //
+ // 1: Same as REX.X=0 (must be 1 in 32-bit mode)
+ // 0: Same as REX.X=1 (64-bit mode only)
+ unsigned char VEX_X = 0x1;
+
+ // VEX_B:
+ //
+ // 1: Same as REX_B=0 (ignored in 32-bit mode)
+ // 0: Same as REX_B=1 (64 bit mode only)
+ //
+ unsigned char VEX_B = 0x1;
+
+ // VEX_W: opcode specific (use like REX.W, or used for
+ // opcode extension, or ignored, depending on the opcode byte)
+ unsigned char VEX_W = 0;
+
+ // VEX_5M (VEX m-mmmmm field):
+ //
+ // 0b00000: Reserved for future use
+ // 0b00001: implied 0F leading opcode
+ // 0b00010: implied 0F 38 leading opcode bytes
+ // 0b00011: implied 0F 3A leading opcode bytes
+ // 0b00100-0b11111: Reserved for future use
+ // 0b01000: XOP map select - 08h instructions with imm byte
+ // 0b01001: XOP map select - 09h instructions with no imm byte
+ // 0b01010: XOP map select - 0Ah instructions with imm dword
+ unsigned char VEX_5M = 0;
+
+ // VEX_4V (VEX vvvv field): a register specifier
+ // (in 1's complement form) or 1111 if unused.
+ unsigned char VEX_4V = 0xf;
+
+ // VEX_L (Vector Length):
+ //
+ // 0: scalar or 128-bit vector
+ // 1: 256-bit vector
+ //
+ unsigned char VEX_L = 0;
+
+ // VEX_PP: opcode extension providing equivalent
+ // functionality of a SIMD prefix
+ //
+ // 0b00: None
+ // 0b01: 66
+ // 0b10: F3
+ // 0b11: F2
+ //
+ unsigned char VEX_PP = 0;
+
+ if ((TSFlags >> X86II::VEXShift) & X86II::VEX_W)
+ VEX_W = 1;
+
+ if ((TSFlags >> X86II::VEXShift) & X86II::VEX_L)
+ VEX_L = 1;
+
+ switch (TSFlags & X86II::OpPrefixMask) {
+ default: break; // VEX_PP already correct
+ case X86II::PD: VEX_PP = 0x1; break; // 66
+ case X86II::XS: VEX_PP = 0x2; break; // F3
+ case X86II::XD: VEX_PP = 0x3; break; // F2
+ }
+
+ switch (TSFlags & X86II::OpMapMask) {
+ default: llvm_unreachable("Invalid prefix!");
+ case X86II::TB: VEX_5M = 0x1; break; // 0F
+ case X86II::T8: VEX_5M = 0x2; break; // 0F 38
+ case X86II::TA: VEX_5M = 0x3; break; // 0F 3A
+ case X86II::XOP8: VEX_5M = 0x8; break;
+ case X86II::XOP9: VEX_5M = 0x9; break;
+ case X86II::XOPA: VEX_5M = 0xA; break;
+ }
+
+ // Classify VEX_B, VEX_4V, VEX_R, VEX_X
+ unsigned NumOps = Desc->getNumOperands();
+ unsigned CurOp = 0;
+ if (NumOps > 1 && Desc->getOperandConstraint(1, MCOI::TIED_TO) == 0)
+ ++CurOp;
+ else if (NumOps > 3 && Desc->getOperandConstraint(2, MCOI::TIED_TO) == 0) {
+ assert(Desc->getOperandConstraint(NumOps - 1, MCOI::TIED_TO) == 1);
+ // Special case for GATHER with 2 TIED_TO operands
+ // Skip the first 2 operands: dst, mask_wb
+ CurOp += 2;
+ }
+
+ switch (TSFlags & X86II::FormMask) {
+ default: llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");
+ case X86II::RawFrm:
+ break;
+ case X86II::MRMDestMem: {
+ // MRMDestMem instructions forms:
+ // MemAddr, src1(ModR/M)
+ // MemAddr, src1(VEX_4V), src2(ModR/M)
+ // MemAddr, src1(ModR/M), imm8
+ //
+ if (X86II::isX86_64ExtendedReg(MI.getOperand(X86::AddrBaseReg).getReg()))
+ VEX_B = 0x0;
+ if (X86II::isX86_64ExtendedReg(MI.getOperand(X86::AddrIndexReg).getReg()))
+ VEX_X = 0x0;
+
+ CurOp = X86::AddrNumOperands;
+ if (HasVEX_4V)
+ VEX_4V = getVEXRegisterEncoding(MI, CurOp++);
+
+ const MachineOperand &MO = MI.getOperand(CurOp);
+ if (MO.isReg() && X86II::isX86_64ExtendedReg(MO.getReg()))
+ VEX_R = 0x0;
+ break;
+ }
+ case X86II::MRMSrcMem:
+ // MRMSrcMem instructions forms:
+ // src1(ModR/M), MemAddr
+ // src1(ModR/M), src2(VEX_4V), MemAddr
+ // src1(ModR/M), MemAddr, imm8
+ // src1(ModR/M), MemAddr, src2(VEX_I8IMM)
+ //
+ // FMA4:
+ // dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(VEX_I8IMM)
+ // dst(ModR/M.reg), src1(VEX_4V), src2(VEX_I8IMM), src3(ModR/M),
+ if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg()))
+ VEX_R = 0x0;
+ CurOp++;
+
+ if (HasVEX_4V) {
+ VEX_4V = getVEXRegisterEncoding(MI, CurOp);
+ CurOp++;
+ }
+
+ if (X86II::isX86_64ExtendedReg(
+ MI.getOperand(MemOperand+X86::AddrBaseReg).getReg()))
+ VEX_B = 0x0;
+ if (X86II::isX86_64ExtendedReg(
+ MI.getOperand(MemOperand+X86::AddrIndexReg).getReg()))
+ VEX_X = 0x0;
+
+ if (HasVEX_4VOp3)
+ VEX_4V = getVEXRegisterEncoding(MI, CurOp+X86::AddrNumOperands);
+ break;
+ case X86II::MRM0m: case X86II::MRM1m:
+ case X86II::MRM2m: case X86II::MRM3m:
+ case X86II::MRM4m: case X86II::MRM5m:
+ case X86II::MRM6m: case X86II::MRM7m: {
+ // MRM[0-9]m instructions forms:
+ // MemAddr
+ // src1(VEX_4V), MemAddr
+ if (HasVEX_4V)
+ VEX_4V = getVEXRegisterEncoding(MI, CurOp++);
+
+ if (X86II::isX86_64ExtendedReg(
+ MI.getOperand(MemOperand+X86::AddrBaseReg).getReg()))
+ VEX_B = 0x0;
+ if (X86II::isX86_64ExtendedReg(
+ MI.getOperand(MemOperand+X86::AddrIndexReg).getReg()))
+ VEX_X = 0x0;
+ break;
+ }
+ case X86II::MRMSrcReg:
+ // MRMSrcReg instructions forms:
+ // dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(VEX_I8IMM)
+ // dst(ModR/M), src1(ModR/M)
+ // dst(ModR/M), src1(ModR/M), imm8
+ //
+ if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg()))
+ VEX_R = 0x0;
+ CurOp++;
+
+ if (HasVEX_4V)
+ VEX_4V = getVEXRegisterEncoding(MI, CurOp++);
+
+ if (HasMemOp4) // Skip second register source (encoded in I8IMM)
+ CurOp++;
+
+ if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg()))
+ VEX_B = 0x0;
+ CurOp++;
+ if (HasVEX_4VOp3)
+ VEX_4V = getVEXRegisterEncoding(MI, CurOp);
+ break;
+ case X86II::MRMDestReg:
+ // MRMDestReg instructions forms:
+ // dst(ModR/M), src(ModR/M)
+ // dst(ModR/M), src(ModR/M), imm8
+ // dst(ModR/M), src1(VEX_4V), src2(ModR/M)
+ if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg()))
+ VEX_B = 0x0;
+ CurOp++;
+
+ if (HasVEX_4V)
+ VEX_4V = getVEXRegisterEncoding(MI, CurOp++);
+
+ if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg()))
+ VEX_R = 0x0;
+ break;
+ case X86II::MRM0r: case X86II::MRM1r:
+ case X86II::MRM2r: case X86II::MRM3r:
+ case X86II::MRM4r: case X86II::MRM5r:
+ case X86II::MRM6r: case X86II::MRM7r:
+ // MRM0r-MRM7r instructions forms:
+ // dst(VEX_4V), src(ModR/M), imm8
+ VEX_4V = getVEXRegisterEncoding(MI, CurOp);
+ CurOp++;
+
+ if (X86II::isX86_64ExtendedReg(MI.getOperand(CurOp).getReg()))
+ VEX_B = 0x0;
+ break;
+ }
+
+ // Emit segment override opcode prefix as needed.
+ emitSegmentOverridePrefix(TSFlags, MemOperand, MI);
+
+ // VEX opcode prefix can have 2 or 3 bytes
+ //
+ // 3 bytes:
+ // +-----+ +--------------+ +-------------------+
+ // | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |
+ // +-----+ +--------------+ +-------------------+
+ // 2 bytes:
+ // +-----+ +-------------------+
+ // | C5h | | R | vvvv | L | pp |
+ // +-----+ +-------------------+
+ //
+ // XOP uses a similar prefix:
+ // +-----+ +--------------+ +-------------------+
+ // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |
+ // +-----+ +--------------+ +-------------------+
+ unsigned char LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3);
+
+ // Can this use the 2 byte VEX prefix?
+ if (Encoding == X86II::VEX && VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) {
+ MCE.emitByte(0xC5);
+ MCE.emitByte(LastByte | (VEX_R << 7));
+ return;
+ }
+
+ // 3 byte VEX prefix
+ MCE.emitByte(Encoding == X86II::XOP ? 0x8F : 0xC4);
+ MCE.emitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M);
+ MCE.emitByte(LastByte | (VEX_W << 7));
+}
+
+template<class CodeEmitter>
+void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
+ const MCInstrDesc *Desc) {
+ DEBUG(dbgs() << MI);
+
+ // If this is a pseudo instruction, lower it.
+ switch (Desc->getOpcode()) {
+ case X86::ADD16rr_DB: Desc = UpdateOp(MI, II, X86::OR16rr); break;
+ case X86::ADD32rr_DB: Desc = UpdateOp(MI, II, X86::OR32rr); break;
+ case X86::ADD64rr_DB: Desc = UpdateOp(MI, II, X86::OR64rr); break;
+ case X86::ADD16ri_DB: Desc = UpdateOp(MI, II, X86::OR16ri); break;
+ case X86::ADD32ri_DB: Desc = UpdateOp(MI, II, X86::OR32ri); break;
+ case X86::ADD64ri32_DB: Desc = UpdateOp(MI, II, X86::OR64ri32); break;
+ case X86::ADD16ri8_DB: Desc = UpdateOp(MI, II, X86::OR16ri8); break;
+ case X86::ADD32ri8_DB: Desc = UpdateOp(MI, II, X86::OR32ri8); break;
+ case X86::ADD64ri8_DB: Desc = UpdateOp(MI, II, X86::OR64ri8); break;
+ case X86::ACQUIRE_MOV8rm: Desc = UpdateOp(MI, II, X86::MOV8rm); break;
+ case X86::ACQUIRE_MOV16rm: Desc = UpdateOp(MI, II, X86::MOV16rm); break;
+ case X86::ACQUIRE_MOV32rm: Desc = UpdateOp(MI, II, X86::MOV32rm); break;
+ case X86::ACQUIRE_MOV64rm: Desc = UpdateOp(MI, II, X86::MOV64rm); break;
+ case X86::RELEASE_MOV8mr: Desc = UpdateOp(MI, II, X86::MOV8mr); break;
+ case X86::RELEASE_MOV16mr: Desc = UpdateOp(MI, II, X86::MOV16mr); break;
+ case X86::RELEASE_MOV32mr: Desc = UpdateOp(MI, II, X86::MOV32mr); break;
+ case X86::RELEASE_MOV64mr: Desc = UpdateOp(MI, II, X86::MOV64mr); break;
+ }
+
+
+ MCE.processDebugLoc(MI.getDebugLoc(), true);
+
+ unsigned Opcode = Desc->Opcode;
+
+ // If this is a two-address instruction, skip one of the register operands.
+ unsigned NumOps = Desc->getNumOperands();
+ unsigned CurOp = 0;
+ if (NumOps > 1 && Desc->getOperandConstraint(1, MCOI::TIED_TO) == 0)
+ ++CurOp;
+ else if (NumOps > 3 && Desc->getOperandConstraint(2, MCOI::TIED_TO) == 0) {
+ assert(Desc->getOperandConstraint(NumOps - 1, MCOI::TIED_TO) == 1);
+ // Special case for GATHER with 2 TIED_TO operands
+ // Skip the first 2 operands: dst, mask_wb
+ CurOp += 2;
+ }
+
+ uint64_t TSFlags = Desc->TSFlags;
+
+ // Encoding type for this instruction.
+ unsigned char Encoding = (TSFlags & X86II::EncodingMask) >>
+ X86II::EncodingShift;
+
+ // It uses the VEX.VVVV field?
+ bool HasVEX_4V = (TSFlags >> X86II::VEXShift) & X86II::VEX_4V;
+ bool HasVEX_4VOp3 = (TSFlags >> X86II::VEXShift) & X86II::VEX_4VOp3;
+ bool HasMemOp4 = (TSFlags >> X86II::VEXShift) & X86II::MemOp4;
+ const unsigned MemOp4_I8IMMOperand = 2;
+
+ // Determine where the memory operand starts, if present.
+ int MemoryOperand = X86II::getMemoryOperandNo(TSFlags, Opcode);
+ if (MemoryOperand != -1) MemoryOperand += CurOp;
+
+ // Emit the lock opcode prefix as needed.
+ if (Desc->TSFlags & X86II::LOCK)
+ MCE.emitByte(0xF0);
+
+ // Emit segment override opcode prefix as needed.
+ emitSegmentOverridePrefix(TSFlags, MemoryOperand, MI);
+
+ // Emit the repeat opcode prefix as needed.
+ if (Desc->TSFlags & X86II::REP)
+ MCE.emitByte(0xF3);
+
+ // Emit the address size opcode prefix as needed.
+ bool need_address_override;
+ if (TSFlags & X86II::AdSize) {
+ need_address_override = true;
+ } else if (MemoryOperand < 0) {
+ need_address_override = false;
+ } else if (Is64BitMode) {
+ assert(!Is16BitMemOperand(MI, MemoryOperand));
+ need_address_override = Is32BitMemOperand(MI, MemoryOperand);
+ } else {
+ assert(!Is64BitMemOperand(MI, MemoryOperand));
+ need_address_override = Is16BitMemOperand(MI, MemoryOperand);
+ }
+
+ if (need_address_override)
+ MCE.emitByte(0x67);
+
+ if (Encoding == 0)
+ emitOpcodePrefix(TSFlags, MemoryOperand, MI, Desc);
+ else
+ emitVEXOpcodePrefix(TSFlags, MemoryOperand, MI, Desc);
+
+ unsigned char BaseOpcode = X86II::getBaseOpcodeFor(Desc->TSFlags);
+ switch (TSFlags & X86II::FormMask) {
+ default:
+ llvm_unreachable("Unknown FormMask value in X86 MachineCodeEmitter!");
+ case X86II::Pseudo:
+ // Remember the current PC offset, this is the PIC relocation
+ // base address.
+ switch (Opcode) {
+ default:
+ llvm_unreachable("pseudo instructions should be removed before code"
+ " emission");
+ // Do nothing for Int_MemBarrier - it's just a comment. Add a debug
+ // to make it slightly easier to see.
+ case X86::Int_MemBarrier:
+ DEBUG(dbgs() << "#MEMBARRIER\n");
+ break;
+
+ case TargetOpcode::INLINEASM:
+ // We allow inline assembler nodes with empty bodies - they can
+ // implicitly define registers, which is ok for JIT.
+ if (MI.getOperand(0).getSymbolName()[0]) {
+ DebugLoc DL = MI.getDebugLoc();
+ DL.print(MI.getParent()->getParent()->getFunction()->getContext(),
+ llvm::errs());
+ report_fatal_error("JIT does not support inline asm!");
+ }
+ break;
+ case TargetOpcode::DBG_VALUE:
+ case TargetOpcode::CFI_INSTRUCTION:
+ break;
+ case TargetOpcode::GC_LABEL:
+ case TargetOpcode::EH_LABEL:
+ MCE.emitLabel(MI.getOperand(0).getMCSymbol());
+ break;
+
+ case TargetOpcode::IMPLICIT_DEF:
+ case TargetOpcode::KILL:
+ break;
+
+ case X86::SEH_PushReg:
+ case X86::SEH_SaveReg:
+ case X86::SEH_SaveXMM:
+ case X86::SEH_StackAlloc:
+ case X86::SEH_SetFrame:
+ case X86::SEH_PushFrame:
+ case X86::SEH_EndPrologue:
+ case X86::SEH_Epilogue:
+ break;
+
+ case X86::MOVPC32r: {
+ // This emits the "call" portion of this pseudo instruction.
+ MCE.emitByte(BaseOpcode);
+ emitConstant(0, X86II::getSizeOfImm(Desc->TSFlags));
+ // Remember PIC base.
+ PICBaseOffset = (intptr_t) MCE.getCurrentPCOffset();
+ X86JITInfo *JTI = TM.getSubtargetImpl()->getJITInfo();
+ JTI->setPICBase(MCE.getCurrentPCValue());
+ break;
+ }
+ }
+ CurOp = NumOps;
+ break;
+ case X86II::RawFrm: {
+ MCE.emitByte(BaseOpcode);
+
+ if (CurOp == NumOps)
+ break;
+
+ const MachineOperand &MO = MI.getOperand(CurOp++);
+
+ DEBUG(dbgs() << "RawFrm CurOp " << CurOp << "\n");
+ DEBUG(dbgs() << "isMBB " << MO.isMBB() << "\n");
+ DEBUG(dbgs() << "isGlobal " << MO.isGlobal() << "\n");
+ DEBUG(dbgs() << "isSymbol " << MO.isSymbol() << "\n");
+ DEBUG(dbgs() << "isImm " << MO.isImm() << "\n");
+
+ if (MO.isMBB()) {
+ emitPCRelativeBlockAddress(MO.getMBB());
+ break;
+ }
+
+ if (MO.isGlobal()) {
+ emitGlobalAddress(MO.getGlobal(), X86::reloc_pcrel_word,
+ MO.getOffset(), 0);
+ break;
+ }
+
+ if (MO.isSymbol()) {
+ emitExternalSymbolAddress(MO.getSymbolName(), X86::reloc_pcrel_word);
+ break;
+ }
+
+ // FIXME: Only used by hackish MCCodeEmitter, remove when dead.
+ if (MO.isJTI()) {
+ emitJumpTableAddress(MO.getIndex(), X86::reloc_pcrel_word);
+ break;
+ }
+
+ assert(MO.isImm() && "Unknown RawFrm operand!");
+ if (Opcode == X86::CALLpcrel32 || Opcode == X86::CALL64pcrel32) {
+ // Fix up immediate operand for pc relative calls.
+ intptr_t Imm = (intptr_t)MO.getImm();
+ Imm = Imm - MCE.getCurrentPCValue() - 4;
+ emitConstant(Imm, X86II::getSizeOfImm(Desc->TSFlags));
+ } else
+ emitConstant(MO.getImm(), X86II::getSizeOfImm(Desc->TSFlags));
+ break;
+ }
+
+ case X86II::AddRegFrm: {
+ MCE.emitByte(BaseOpcode +
+ getX86RegNum(MI.getOperand(CurOp++).getReg()));
+
+ if (CurOp == NumOps)
+ break;
+
+ const MachineOperand &MO1 = MI.getOperand(CurOp++);
+ unsigned Size = X86II::getSizeOfImm(Desc->TSFlags);
+ if (MO1.isImm()) {
+ emitConstant(MO1.getImm(), Size);
+ break;
+ }
+
+ unsigned rt = Is64BitMode ? X86::reloc_pcrel_word
+ : (IsPIC ? X86::reloc_picrel_word : X86::reloc_absolute_word);
+ if (Opcode == X86::MOV32ri64)
+ rt = X86::reloc_absolute_word; // FIXME: add X86II flag?
+ // This should not occur on Darwin for relocatable objects.
+ if (Opcode == X86::MOV64ri)
+ rt = X86::reloc_absolute_dword; // FIXME: add X86II flag?
+ if (MO1.isGlobal()) {
+ bool Indirect = gvNeedsNonLazyPtr(MO1, TM);
+ emitGlobalAddress(MO1.getGlobal(), rt, MO1.getOffset(), 0,
+ Indirect);
+ } else if (MO1.isSymbol())
+ emitExternalSymbolAddress(MO1.getSymbolName(), rt);
+ else if (MO1.isCPI())
+ emitConstPoolAddress(MO1.getIndex(), rt);
+ else if (MO1.isJTI())
+ emitJumpTableAddress(MO1.getIndex(), rt);
+ break;
+ }
+
+ case X86II::MRMDestReg: {
+ MCE.emitByte(BaseOpcode);
+
+ unsigned SrcRegNum = CurOp+1;
+ if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
+ SrcRegNum++;
+
+ emitRegModRMByte(MI.getOperand(CurOp).getReg(),
+ getX86RegNum(MI.getOperand(SrcRegNum).getReg()));
+ CurOp = SrcRegNum + 1;
+ break;
+ }
+ case X86II::MRMDestMem: {
+ MCE.emitByte(BaseOpcode);
+
+ unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
+ if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
+ SrcRegNum++;
+ emitMemModRMByte(MI, CurOp,
+ getX86RegNum(MI.getOperand(SrcRegNum).getReg()));
+ CurOp = SrcRegNum + 1;
+ break;
+ }
+
+ case X86II::MRMSrcReg: {
+ MCE.emitByte(BaseOpcode);
+
+ unsigned SrcRegNum = CurOp+1;
+ if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
+ ++SrcRegNum;
+
+ if (HasMemOp4) // Skip 2nd src (which is encoded in I8IMM)
+ ++SrcRegNum;
+
+ emitRegModRMByte(MI.getOperand(SrcRegNum).getReg(),
+ getX86RegNum(MI.getOperand(CurOp).getReg()));
+ // 2 operands skipped with HasMemOp4, compensate accordingly
+ CurOp = HasMemOp4 ? SrcRegNum : SrcRegNum + 1;
+ if (HasVEX_4VOp3)
+ ++CurOp;
+ break;
+ }
+ case X86II::MRMSrcMem: {
+ int AddrOperands = X86::AddrNumOperands;
+ unsigned FirstMemOp = CurOp+1;
+ if (HasVEX_4V) {
+ ++AddrOperands;
+ ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
+ }
+ if (HasMemOp4) // Skip second register source (encoded in I8IMM)
+ ++FirstMemOp;
+
+ MCE.emitByte(BaseOpcode);
+
+ intptr_t PCAdj = (CurOp + AddrOperands + 1 != NumOps) ?
+ X86II::getSizeOfImm(Desc->TSFlags) : 0;
+ emitMemModRMByte(MI, FirstMemOp,
+ getX86RegNum(MI.getOperand(CurOp).getReg()),PCAdj);
+ CurOp += AddrOperands + 1;
+ if (HasVEX_4VOp3)
+ ++CurOp;
+ break;
+ }
+
+ case X86II::MRMXr:
+ case X86II::MRM0r: case X86II::MRM1r:
+ case X86II::MRM2r: case X86II::MRM3r:
+ case X86II::MRM4r: case X86II::MRM5r:
+ case X86II::MRM6r: case X86II::MRM7r: {
+ if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
+ ++CurOp;
+ MCE.emitByte(BaseOpcode);
+ uint64_t Form = (Desc->TSFlags & X86II::FormMask);
+ emitRegModRMByte(MI.getOperand(CurOp++).getReg(),
+ (Form == X86II::MRMXr) ? 0 : Form-X86II::MRM0r);
+
+ if (CurOp == NumOps)
+ break;
+
+ const MachineOperand &MO1 = MI.getOperand(CurOp++);
+ unsigned Size = X86II::getSizeOfImm(Desc->TSFlags);
+ if (MO1.isImm()) {
+ emitConstant(MO1.getImm(), Size);
+ break;
+ }
+
+ unsigned rt = Is64BitMode ? X86::reloc_pcrel_word
+ : (IsPIC ? X86::reloc_picrel_word : X86::reloc_absolute_word);
+ if (Opcode == X86::MOV64ri32)
+ rt = X86::reloc_absolute_word_sext; // FIXME: add X86II flag?
+ if (MO1.isGlobal()) {
+ bool Indirect = gvNeedsNonLazyPtr(MO1, TM);
+ emitGlobalAddress(MO1.getGlobal(), rt, MO1.getOffset(), 0,
+ Indirect);
+ } else if (MO1.isSymbol())
+ emitExternalSymbolAddress(MO1.getSymbolName(), rt);
+ else if (MO1.isCPI())
+ emitConstPoolAddress(MO1.getIndex(), rt);
+ else if (MO1.isJTI())
+ emitJumpTableAddress(MO1.getIndex(), rt);
+ break;
+ }
+
+ case X86II::MRMXm:
+ case X86II::MRM0m: case X86II::MRM1m:
+ case X86II::MRM2m: case X86II::MRM3m:
+ case X86II::MRM4m: case X86II::MRM5m:
+ case X86II::MRM6m: case X86II::MRM7m: {
+ if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
+ ++CurOp;
+ intptr_t PCAdj = (CurOp + X86::AddrNumOperands != NumOps) ?
+ (MI.getOperand(CurOp+X86::AddrNumOperands).isImm() ?
+ X86II::getSizeOfImm(Desc->TSFlags) : 4) : 0;
+
+ MCE.emitByte(BaseOpcode);
+ uint64_t Form = (Desc->TSFlags & X86II::FormMask);
+ emitMemModRMByte(MI, CurOp, (Form==X86II::MRMXm) ? 0 : Form - X86II::MRM0m,
+ PCAdj);
+ CurOp += X86::AddrNumOperands;
+
+ if (CurOp == NumOps)
+ break;
+
+ const MachineOperand &MO = MI.getOperand(CurOp++);
+ unsigned Size = X86II::getSizeOfImm(Desc->TSFlags);
+ if (MO.isImm()) {
+ emitConstant(MO.getImm(), Size);
+ break;
+ }
+
+ unsigned rt = Is64BitMode ? X86::reloc_pcrel_word
+ : (IsPIC ? X86::reloc_picrel_word : X86::reloc_absolute_word);
+ if (Opcode == X86::MOV64mi32)
+ rt = X86::reloc_absolute_word_sext; // FIXME: add X86II flag?
+ if (MO.isGlobal()) {
+ bool Indirect = gvNeedsNonLazyPtr(MO, TM);
+ emitGlobalAddress(MO.getGlobal(), rt, MO.getOffset(), 0,
+ Indirect);
+ } else if (MO.isSymbol())
+ emitExternalSymbolAddress(MO.getSymbolName(), rt);
+ else if (MO.isCPI())
+ emitConstPoolAddress(MO.getIndex(), rt);
+ else if (MO.isJTI())
+ emitJumpTableAddress(MO.getIndex(), rt);
+ break;
+ }
+
+ case X86II::MRM_C0: case X86II::MRM_C1: case X86II::MRM_C2:
+ case X86II::MRM_C3: case X86II::MRM_C4: case X86II::MRM_C8:
+ case X86II::MRM_C9: case X86II::MRM_CA: case X86II::MRM_CB:
+ case X86II::MRM_CF: case X86II::MRM_D0: case X86II::MRM_D1:
+ case X86II::MRM_D4: case X86II::MRM_D5: case X86II::MRM_D6:
+ case X86II::MRM_D7: case X86II::MRM_D8: case X86II::MRM_D9:
+ case X86II::MRM_DA: case X86II::MRM_DB: case X86II::MRM_DC:
+ case X86II::MRM_DD: case X86II::MRM_DE: case X86II::MRM_DF:
+ case X86II::MRM_E0: case X86II::MRM_E1: case X86II::MRM_E2:
+ case X86II::MRM_E3: case X86II::MRM_E4: case X86II::MRM_E5:
+ case X86II::MRM_E8: case X86II::MRM_E9: case X86II::MRM_EA:
+ case X86II::MRM_EB: case X86II::MRM_EC: case X86II::MRM_ED:
+ case X86II::MRM_EE: case X86II::MRM_F0: case X86II::MRM_F1:
+ case X86II::MRM_F2: case X86II::MRM_F3: case X86II::MRM_F4:
+ case X86II::MRM_F5: case X86II::MRM_F6: case X86II::MRM_F7:
+ case X86II::MRM_F8: case X86II::MRM_F9: case X86II::MRM_FA:
+ case X86II::MRM_FB: case X86II::MRM_FC: case X86II::MRM_FD:
+ case X86II::MRM_FE: case X86II::MRM_FF:
+ MCE.emitByte(BaseOpcode);
+
+ unsigned char MRM;
+ switch (TSFlags & X86II::FormMask) {
+ default: llvm_unreachable("Invalid Form");
+ case X86II::MRM_C0: MRM = 0xC0; break;
+ case X86II::MRM_C1: MRM = 0xC1; break;
+ case X86II::MRM_C2: MRM = 0xC2; break;
+ case X86II::MRM_C3: MRM = 0xC3; break;
+ case X86II::MRM_C4: MRM = 0xC4; break;
+ case X86II::MRM_C8: MRM = 0xC8; break;
+ case X86II::MRM_C9: MRM = 0xC9; break;
+ case X86II::MRM_CA: MRM = 0xCA; break;
+ case X86II::MRM_CB: MRM = 0xCB; break;
+ case X86II::MRM_CF: MRM = 0xCF; break;
+ case X86II::MRM_D0: MRM = 0xD0; break;
+ case X86II::MRM_D1: MRM = 0xD1; break;
+ case X86II::MRM_D4: MRM = 0xD4; break;
+ case X86II::MRM_D5: MRM = 0xD5; break;
+ case X86II::MRM_D6: MRM = 0xD6; break;
+ case X86II::MRM_D7: MRM = 0xD7; break;
+ case X86II::MRM_D8: MRM = 0xD8; break;
+ case X86II::MRM_D9: MRM = 0xD9; break;
+ case X86II::MRM_DA: MRM = 0xDA; break;
+ case X86II::MRM_DB: MRM = 0xDB; break;
+ case X86II::MRM_DC: MRM = 0xDC; break;
+ case X86II::MRM_DD: MRM = 0xDD; break;
+ case X86II::MRM_DE: MRM = 0xDE; break;
+ case X86II::MRM_DF: MRM = 0xDF; break;
+ case X86II::MRM_E0: MRM = 0xE0; break;
+ case X86II::MRM_E1: MRM = 0xE1; break;
+ case X86II::MRM_E2: MRM = 0xE2; break;
+ case X86II::MRM_E3: MRM = 0xE3; break;
+ case X86II::MRM_E4: MRM = 0xE4; break;
+ case X86II::MRM_E5: MRM = 0xE5; break;
+ case X86II::MRM_E8: MRM = 0xE8; break;
+ case X86II::MRM_E9: MRM = 0xE9; break;
+ case X86II::MRM_EA: MRM = 0xEA; break;
+ case X86II::MRM_EB: MRM = 0xEB; break;
+ case X86II::MRM_EC: MRM = 0xEC; break;
+ case X86II::MRM_ED: MRM = 0xED; break;
+ case X86II::MRM_EE: MRM = 0xEE; break;
+ case X86II::MRM_F0: MRM = 0xF0; break;
+ case X86II::MRM_F1: MRM = 0xF1; break;
+ case X86II::MRM_F2: MRM = 0xF2; break;
+ case X86II::MRM_F3: MRM = 0xF3; break;
+ case X86II::MRM_F4: MRM = 0xF4; break;
+ case X86II::MRM_F5: MRM = 0xF5; break;
+ case X86II::MRM_F6: MRM = 0xF6; break;
+ case X86II::MRM_F7: MRM = 0xF7; break;
+ case X86II::MRM_F8: MRM = 0xF8; break;
+ case X86II::MRM_F9: MRM = 0xF9; break;
+ case X86II::MRM_FA: MRM = 0xFA; break;
+ case X86II::MRM_FB: MRM = 0xFB; break;
+ case X86II::MRM_FC: MRM = 0xFC; break;
+ case X86II::MRM_FD: MRM = 0xFD; break;
+ case X86II::MRM_FE: MRM = 0xFE; break;
+ case X86II::MRM_FF: MRM = 0xFF; break;
+ }
+ MCE.emitByte(MRM);
+ break;
+ }
+
+ while (CurOp != NumOps && NumOps - CurOp <= 2) {
+ // The last source register of a 4 operand instruction in AVX is encoded
+ // in bits[7:4] of a immediate byte.
+ if ((TSFlags >> X86II::VEXShift) & X86II::VEX_I8IMM) {
+ const MachineOperand &MO = MI.getOperand(HasMemOp4 ? MemOp4_I8IMMOperand
+ : CurOp);
+ ++CurOp;
+ unsigned RegNum = getX86RegNum(MO.getReg()) << 4;
+ if (X86II::isX86_64ExtendedReg(MO.getReg()))
+ RegNum |= 1 << 7;
+ // If there is an additional 5th operand it must be an immediate, which
+ // is encoded in bits[3:0]
+ if (CurOp != NumOps) {
+ const MachineOperand &MIMM = MI.getOperand(CurOp++);
+ if (MIMM.isImm()) {
+ unsigned Val = MIMM.getImm();
+ assert(Val < 16 && "Immediate operand value out of range");
+ RegNum |= Val;
+ }
+ }
+ emitConstant(RegNum, 1);
+ } else {
+ emitConstant(MI.getOperand(CurOp++).getImm(),
+ X86II::getSizeOfImm(Desc->TSFlags));
+ }
+ }
+
+ if (!MI.isVariadic() && CurOp != NumOps) {
+#ifndef NDEBUG
+ dbgs() << "Cannot encode all operands of: " << MI << "\n";
+#endif
+ llvm_unreachable(nullptr);
+ }
+
+ MCE.processDebugLoc(MI.getDebugLoc(), false);
+}
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SelectionDAGISel.h"
-#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Type.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/Function.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCExpr.h"
--- /dev/null
+//===-- X86JITInfo.cpp - Implement the JIT interfaces for the X86 target --===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the JIT interfaces for the X86 target.
+//
+//===----------------------------------------------------------------------===//
+
+#include "X86JITInfo.h"
+#include "X86Relocations.h"
+#include "X86Subtarget.h"
+#include "X86TargetMachine.h"
+#include "llvm/IR/Function.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Valgrind.h"
+#include <cstdlib>
+#include <cstring>
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+// Determine the platform we're running on
+#if defined (__x86_64__) || defined (_M_AMD64) || defined (_M_X64)
+# define X86_64_JIT
+#elif defined(__i386__) || defined(i386) || defined(_M_IX86)
+# define X86_32_JIT
+#endif
+
+void X86JITInfo::replaceMachineCodeForFunction(void *Old, void *New) {
+ unsigned char *OldByte = (unsigned char *)Old;
+ *OldByte++ = 0xE9; // Emit JMP opcode.
+ unsigned *OldWord = (unsigned *)OldByte;
+ unsigned NewAddr = (intptr_t)New;
+ unsigned OldAddr = (intptr_t)OldWord;
+ *OldWord = NewAddr - OldAddr - 4; // Emit PC-relative addr of New code.
+
+ // X86 doesn't need to invalidate the processor cache, so just invalidate
+ // Valgrind's cache directly.
+ sys::ValgrindDiscardTranslations(Old, 5);
+}
+
+
+/// JITCompilerFunction - This contains the address of the JIT function used to
+/// compile a function lazily.
+static TargetJITInfo::JITCompilerFn JITCompilerFunction;
+
+// Get the ASMPREFIX for the current host. This is often '_'.
+#ifndef __USER_LABEL_PREFIX__
+#define __USER_LABEL_PREFIX__
+#endif
+#define GETASMPREFIX2(X) #X
+#define GETASMPREFIX(X) GETASMPREFIX2(X)
+#define ASMPREFIX GETASMPREFIX(__USER_LABEL_PREFIX__)
+
+// For ELF targets, use a .size and .type directive, to let tools
+// know the extent of functions defined in assembler.
+#if defined(__ELF__)
+# define SIZE(sym) ".size " #sym ", . - " #sym "\n"
+# define TYPE_FUNCTION(sym) ".type " #sym ", @function\n"
+#else
+# define SIZE(sym)
+# define TYPE_FUNCTION(sym)
+#endif
+
+// Provide a convenient way for disabling usage of CFI directives.
+// This is needed for old/broken assemblers (for example, gas on
+// Darwin is pretty old and doesn't support these directives)
+#if defined(__APPLE__)
+# define CFI(x)
+#else
+// FIXME: Disable this until we really want to use it. Also, we will
+// need to add some workarounds for compilers, which support
+// only subset of these directives.
+# define CFI(x)
+#endif
+
+// Provide a wrapper for LLVMX86CompilationCallback2 that saves non-traditional
+// callee saved registers, for the fastcc calling convention.
+extern "C" {
+#if defined(X86_64_JIT)
+# ifndef _MSC_VER
+ // No need to save EAX/EDX for X86-64.
+ void X86CompilationCallback(void);
+ asm(
+ ".text\n"
+ ".align 8\n"
+ ".globl " ASMPREFIX "X86CompilationCallback\n"
+ TYPE_FUNCTION(X86CompilationCallback)
+ ASMPREFIX "X86CompilationCallback:\n"
+ CFI(".cfi_startproc\n")
+ // Save RBP
+ "pushq %rbp\n"
+ CFI(".cfi_def_cfa_offset 16\n")
+ CFI(".cfi_offset %rbp, -16\n")
+ // Save RSP
+ "movq %rsp, %rbp\n"
+ CFI(".cfi_def_cfa_register %rbp\n")
+ // Save all int arg registers
+ "pushq %rdi\n"
+ CFI(".cfi_rel_offset %rdi, 0\n")
+ "pushq %rsi\n"
+ CFI(".cfi_rel_offset %rsi, 8\n")
+ "pushq %rdx\n"
+ CFI(".cfi_rel_offset %rdx, 16\n")
+ "pushq %rcx\n"
+ CFI(".cfi_rel_offset %rcx, 24\n")
+ "pushq %r8\n"
+ CFI(".cfi_rel_offset %r8, 32\n")
+ "pushq %r9\n"
+ CFI(".cfi_rel_offset %r9, 40\n")
+ // Align stack on 16-byte boundary. ESP might not be properly aligned
+ // (8 byte) if this is called from an indirect stub.
+ "andq $-16, %rsp\n"
+ // Save all XMM arg registers
+ "subq $128, %rsp\n"
+ "movaps %xmm0, (%rsp)\n"
+ "movaps %xmm1, 16(%rsp)\n"
+ "movaps %xmm2, 32(%rsp)\n"
+ "movaps %xmm3, 48(%rsp)\n"
+ "movaps %xmm4, 64(%rsp)\n"
+ "movaps %xmm5, 80(%rsp)\n"
+ "movaps %xmm6, 96(%rsp)\n"
+ "movaps %xmm7, 112(%rsp)\n"
+ // JIT callee
+#if defined(_WIN64) || defined(__CYGWIN__)
+ "subq $32, %rsp\n"
+ "movq %rbp, %rcx\n" // Pass prev frame and return address
+ "movq 8(%rbp), %rdx\n"
+ "call " ASMPREFIX "LLVMX86CompilationCallback2\n"
+ "addq $32, %rsp\n"
+#else
+ "movq %rbp, %rdi\n" // Pass prev frame and return address
+ "movq 8(%rbp), %rsi\n"
+ "call " ASMPREFIX "LLVMX86CompilationCallback2\n"
+#endif
+ // Restore all XMM arg registers
+ "movaps 112(%rsp), %xmm7\n"
+ "movaps 96(%rsp), %xmm6\n"
+ "movaps 80(%rsp), %xmm5\n"
+ "movaps 64(%rsp), %xmm4\n"
+ "movaps 48(%rsp), %xmm3\n"
+ "movaps 32(%rsp), %xmm2\n"
+ "movaps 16(%rsp), %xmm1\n"
+ "movaps (%rsp), %xmm0\n"
+ // Restore RSP
+ "movq %rbp, %rsp\n"
+ CFI(".cfi_def_cfa_register %rsp\n")
+ // Restore all int arg registers
+ "subq $48, %rsp\n"
+ CFI(".cfi_adjust_cfa_offset 48\n")
+ "popq %r9\n"
+ CFI(".cfi_adjust_cfa_offset -8\n")
+ CFI(".cfi_restore %r9\n")
+ "popq %r8\n"
+ CFI(".cfi_adjust_cfa_offset -8\n")
+ CFI(".cfi_restore %r8\n")
+ "popq %rcx\n"
+ CFI(".cfi_adjust_cfa_offset -8\n")
+ CFI(".cfi_restore %rcx\n")
+ "popq %rdx\n"
+ CFI(".cfi_adjust_cfa_offset -8\n")
+ CFI(".cfi_restore %rdx\n")
+ "popq %rsi\n"
+ CFI(".cfi_adjust_cfa_offset -8\n")
+ CFI(".cfi_restore %rsi\n")
+ "popq %rdi\n"
+ CFI(".cfi_adjust_cfa_offset -8\n")
+ CFI(".cfi_restore %rdi\n")
+ // Restore RBP
+ "popq %rbp\n"
+ CFI(".cfi_adjust_cfa_offset -8\n")
+ CFI(".cfi_restore %rbp\n")
+ "ret\n"
+ CFI(".cfi_endproc\n")
+ SIZE(X86CompilationCallback)
+ );
+# else
+ // No inline assembler support on this platform. The routine is in external
+ // file.
+ void X86CompilationCallback();
+
+# endif
+#elif defined (X86_32_JIT)
+# ifndef _MSC_VER
+ void X86CompilationCallback(void);
+ asm(
+ ".text\n"
+ ".align 8\n"
+ ".globl " ASMPREFIX "X86CompilationCallback\n"
+ TYPE_FUNCTION(X86CompilationCallback)
+ ASMPREFIX "X86CompilationCallback:\n"
+ CFI(".cfi_startproc\n")
+ "pushl %ebp\n"
+ CFI(".cfi_def_cfa_offset 8\n")
+ CFI(".cfi_offset %ebp, -8\n")
+ "movl %esp, %ebp\n" // Standard prologue
+ CFI(".cfi_def_cfa_register %ebp\n")
+ "pushl %eax\n"
+ CFI(".cfi_rel_offset %eax, 0\n")
+ "pushl %edx\n" // Save EAX/EDX/ECX
+ CFI(".cfi_rel_offset %edx, 4\n")
+ "pushl %ecx\n"
+ CFI(".cfi_rel_offset %ecx, 8\n")
+# if defined(__APPLE__)
+ "andl $-16, %esp\n" // Align ESP on 16-byte boundary
+# endif
+ "subl $16, %esp\n"
+ "movl 4(%ebp), %eax\n" // Pass prev frame and return address
+ "movl %eax, 4(%esp)\n"
+ "movl %ebp, (%esp)\n"
+ "call " ASMPREFIX "LLVMX86CompilationCallback2\n"
+ "movl %ebp, %esp\n" // Restore ESP
+ CFI(".cfi_def_cfa_register %esp\n")
+ "subl $12, %esp\n"
+ CFI(".cfi_adjust_cfa_offset 12\n")
+ "popl %ecx\n"
+ CFI(".cfi_adjust_cfa_offset -4\n")
+ CFI(".cfi_restore %ecx\n")
+ "popl %edx\n"
+ CFI(".cfi_adjust_cfa_offset -4\n")
+ CFI(".cfi_restore %edx\n")
+ "popl %eax\n"
+ CFI(".cfi_adjust_cfa_offset -4\n")
+ CFI(".cfi_restore %eax\n")
+ "popl %ebp\n"
+ CFI(".cfi_adjust_cfa_offset -4\n")
+ CFI(".cfi_restore %ebp\n")
+ "ret\n"
+ CFI(".cfi_endproc\n")
+ SIZE(X86CompilationCallback)
+ );
+
+ // Same as X86CompilationCallback but also saves XMM argument registers.
+ void X86CompilationCallback_SSE(void);
+ asm(
+ ".text\n"
+ ".align 8\n"
+ ".globl " ASMPREFIX "X86CompilationCallback_SSE\n"
+ TYPE_FUNCTION(X86CompilationCallback_SSE)
+ ASMPREFIX "X86CompilationCallback_SSE:\n"
+ CFI(".cfi_startproc\n")
+ "pushl %ebp\n"
+ CFI(".cfi_def_cfa_offset 8\n")
+ CFI(".cfi_offset %ebp, -8\n")
+ "movl %esp, %ebp\n" // Standard prologue
+ CFI(".cfi_def_cfa_register %ebp\n")
+ "pushl %eax\n"
+ CFI(".cfi_rel_offset %eax, 0\n")
+ "pushl %edx\n" // Save EAX/EDX/ECX
+ CFI(".cfi_rel_offset %edx, 4\n")
+ "pushl %ecx\n"
+ CFI(".cfi_rel_offset %ecx, 8\n")
+ "andl $-16, %esp\n" // Align ESP on 16-byte boundary
+ // Save all XMM arg registers
+ "subl $64, %esp\n"
+ // FIXME: provide frame move information for xmm registers.
+ // This can be tricky, because CFA register is ebp (unaligned)
+ // and we need to produce offsets relative to it.
+ "movaps %xmm0, (%esp)\n"
+ "movaps %xmm1, 16(%esp)\n"
+ "movaps %xmm2, 32(%esp)\n"
+ "movaps %xmm3, 48(%esp)\n"
+ "subl $16, %esp\n"
+ "movl 4(%ebp), %eax\n" // Pass prev frame and return address
+ "movl %eax, 4(%esp)\n"
+ "movl %ebp, (%esp)\n"
+ "call " ASMPREFIX "LLVMX86CompilationCallback2\n"
+ "addl $16, %esp\n"
+ "movaps 48(%esp), %xmm3\n"
+ CFI(".cfi_restore %xmm3\n")
+ "movaps 32(%esp), %xmm2\n"
+ CFI(".cfi_restore %xmm2\n")
+ "movaps 16(%esp), %xmm1\n"
+ CFI(".cfi_restore %xmm1\n")
+ "movaps (%esp), %xmm0\n"
+ CFI(".cfi_restore %xmm0\n")
+ "movl %ebp, %esp\n" // Restore ESP
+ CFI(".cfi_def_cfa_register esp\n")
+ "subl $12, %esp\n"
+ CFI(".cfi_adjust_cfa_offset 12\n")
+ "popl %ecx\n"
+ CFI(".cfi_adjust_cfa_offset -4\n")
+ CFI(".cfi_restore %ecx\n")
+ "popl %edx\n"
+ CFI(".cfi_adjust_cfa_offset -4\n")
+ CFI(".cfi_restore %edx\n")
+ "popl %eax\n"
+ CFI(".cfi_adjust_cfa_offset -4\n")
+ CFI(".cfi_restore %eax\n")
+ "popl %ebp\n"
+ CFI(".cfi_adjust_cfa_offset -4\n")
+ CFI(".cfi_restore %ebp\n")
+ "ret\n"
+ CFI(".cfi_endproc\n")
+ SIZE(X86CompilationCallback_SSE)
+ );
+# else
+ void LLVMX86CompilationCallback2(intptr_t *StackPtr, intptr_t RetAddr);
+
+ _declspec(naked) void X86CompilationCallback(void) {
+ __asm {
+ push ebp
+ mov ebp, esp
+ push eax
+ push edx
+ push ecx
+ and esp, -16
+ sub esp, 16
+ mov eax, dword ptr [ebp+4]
+ mov dword ptr [esp+4], eax
+ mov dword ptr [esp], ebp
+ call LLVMX86CompilationCallback2
+ mov esp, ebp
+ sub esp, 12
+ pop ecx
+ pop edx
+ pop eax
+ pop ebp
+ ret
+ }
+ }
+
+# endif // _MSC_VER
+
+#else // Not an i386 host
+ void X86CompilationCallback() {
+ llvm_unreachable("Cannot call X86CompilationCallback() on a non-x86 arch!");
+ }
+#endif
+}
+
+/// This is the target-specific function invoked by the
+/// function stub when we did not know the real target of a call. This function
+/// must locate the start of the stub or call site and pass it into the JIT
+/// compiler function.
+extern "C" {
+LLVM_ATTRIBUTE_USED // Referenced from inline asm.
+LLVM_LIBRARY_VISIBILITY void LLVMX86CompilationCallback2(intptr_t *StackPtr,
+ intptr_t RetAddr) {
+ intptr_t *RetAddrLoc = &StackPtr[1];
+ // We are reading raw stack data here. Tell MemorySanitizer that it is
+ // sufficiently initialized.
+ __msan_unpoison(RetAddrLoc, sizeof(*RetAddrLoc));
+ assert(*RetAddrLoc == RetAddr &&
+ "Could not find return address on the stack!");
+
+ // It's a stub if there is an interrupt marker after the call.
+ bool isStub = ((unsigned char*)RetAddr)[0] == 0xCE;
+
+ // The call instruction should have pushed the return value onto the stack...
+#if defined (X86_64_JIT)
+ RetAddr--; // Backtrack to the reference itself...
+#else
+ RetAddr -= 4; // Backtrack to the reference itself...
+#endif
+
+#if 0
+ DEBUG(dbgs() << "In callback! Addr=" << (void*)RetAddr
+ << " ESP=" << (void*)StackPtr
+ << ": Resolving call to function: "
+ << TheVM->getFunctionReferencedName((void*)RetAddr) << "\n");
+#endif
+
+ // Sanity check to make sure this really is a call instruction.
+#if defined (X86_64_JIT)
+ assert(((unsigned char*)RetAddr)[-2] == 0x41 &&"Not a call instr!");
+ assert(((unsigned char*)RetAddr)[-1] == 0xFF &&"Not a call instr!");
+#else
+ assert(((unsigned char*)RetAddr)[-1] == 0xE8 &&"Not a call instr!");
+#endif
+
+ intptr_t NewVal = (intptr_t)JITCompilerFunction((void*)RetAddr);
+
+ // Rewrite the call target... so that we don't end up here every time we
+ // execute the call.
+#if defined (X86_64_JIT)
+ assert(isStub &&
+ "X86-64 doesn't support rewriting non-stub lazy compilation calls:"
+ " the call instruction varies too much.");
+#else
+ *(intptr_t *)RetAddr = (intptr_t)(NewVal-RetAddr-4);
+#endif
+
+ if (isStub) {
+ // If this is a stub, rewrite the call into an unconditional branch
+ // instruction so that two return addresses are not pushed onto the stack
+ // when the requested function finally gets called. This also makes the
+ // 0xCE byte (interrupt) dead, so the marker doesn't effect anything.
+#if defined (X86_64_JIT)
+ // If the target address is within 32-bit range of the stub, use a
+ // PC-relative branch instead of loading the actual address. (This is
+ // considerably shorter than the 64-bit immediate load already there.)
+ // We assume here intptr_t is 64 bits.
+ intptr_t diff = NewVal-RetAddr+7;
+ if (diff >= -2147483648LL && diff <= 2147483647LL) {
+ *(unsigned char*)(RetAddr-0xc) = 0xE9;
+ *(intptr_t *)(RetAddr-0xb) = diff & 0xffffffff;
+ } else {
+ *(intptr_t *)(RetAddr - 0xa) = NewVal;
+ ((unsigned char*)RetAddr)[0] = (2 | (4 << 3) | (3 << 6));
+ }
+ sys::ValgrindDiscardTranslations((void*)(RetAddr-0xc), 0xd);
+#else
+ ((unsigned char*)RetAddr)[-1] = 0xE9;
+ sys::ValgrindDiscardTranslations((void*)(RetAddr-1), 5);
+#endif
+ }
+
+ // Change the return address to reexecute the call instruction...
+#if defined (X86_64_JIT)
+ *RetAddrLoc -= 0xd;
+#else
+ *RetAddrLoc -= 5;
+#endif
+}
+}
+
+TargetJITInfo::LazyResolverFn
+X86JITInfo::getLazyResolverFunction(JITCompilerFn F) {
+ TsanIgnoreWritesBegin();
+ JITCompilerFunction = F;
+ TsanIgnoreWritesEnd();
+
+#if defined (X86_32_JIT) && !defined (_MSC_VER)
+#if defined(__SSE__)
+ // SSE Callback should be called for SSE-enabled LLVM.
+ return X86CompilationCallback_SSE;
+#else
+ if (useSSE)
+ return X86CompilationCallback_SSE;
+#endif
+#endif
+
+ return X86CompilationCallback;
+}
+
+X86JITInfo::X86JITInfo(bool UseSSE) {
+ useSSE = UseSSE;
+ useGOT = 0;
+ TLSOffset = nullptr;
+}
+
+void *X86JITInfo::emitGlobalValueIndirectSym(const GlobalValue* GV, void *ptr,
+ JITCodeEmitter &JCE) {
+#if defined (X86_64_JIT)
+ const unsigned Alignment = 8;
+ uint8_t Buffer[8];
+ uint8_t *Cur = Buffer;
+ MachineCodeEmitter::emitWordLEInto(Cur, (unsigned)(intptr_t)ptr);
+ MachineCodeEmitter::emitWordLEInto(Cur, (unsigned)(((intptr_t)ptr) >> 32));
+#else
+ const unsigned Alignment = 4;
+ uint8_t Buffer[4];
+ uint8_t *Cur = Buffer;
+ MachineCodeEmitter::emitWordLEInto(Cur, (intptr_t)ptr);
+#endif
+ return JCE.allocIndirectGV(GV, Buffer, sizeof(Buffer), Alignment);
+}
+
+TargetJITInfo::StubLayout X86JITInfo::getStubLayout() {
+ // The 64-bit stub contains:
+ // movabs r10 <- 8-byte-target-address # 10 bytes
+ // call|jmp *r10 # 3 bytes
+ // The 32-bit stub contains a 5-byte call|jmp.
+ // If the stub is a call to the compilation callback, an extra byte is added
+ // to mark it as a stub.
+ StubLayout Result = {14, 4};
+ return Result;
+}
+
+void *X86JITInfo::emitFunctionStub(const Function* F, void *Target,
+ JITCodeEmitter &JCE) {
+ // Note, we cast to intptr_t here to silence a -pedantic warning that
+ // complains about casting a function pointer to a normal pointer.
+#if defined (X86_32_JIT) && !defined (_MSC_VER)
+ bool NotCC = (Target != (void*)(intptr_t)X86CompilationCallback &&
+ Target != (void*)(intptr_t)X86CompilationCallback_SSE);
+#else
+ bool NotCC = Target != (void*)(intptr_t)X86CompilationCallback;
+#endif
+ JCE.emitAlignment(4);
+ void *Result = (void*)JCE.getCurrentPCValue();
+ if (NotCC) {
+#if defined (X86_64_JIT)
+ JCE.emitByte(0x49); // REX prefix
+ JCE.emitByte(0xB8+2); // movabsq r10
+ JCE.emitWordLE((unsigned)(intptr_t)Target);
+ JCE.emitWordLE((unsigned)(((intptr_t)Target) >> 32));
+ JCE.emitByte(0x41); // REX prefix
+ JCE.emitByte(0xFF); // jmpq *r10
+ JCE.emitByte(2 | (4 << 3) | (3 << 6));
+#else
+ JCE.emitByte(0xE9);
+ JCE.emitWordLE((intptr_t)Target-JCE.getCurrentPCValue()-4);
+#endif
+ return Result;
+ }
+
+#if defined (X86_64_JIT)
+ JCE.emitByte(0x49); // REX prefix
+ JCE.emitByte(0xB8+2); // movabsq r10
+ JCE.emitWordLE((unsigned)(intptr_t)Target);
+ JCE.emitWordLE((unsigned)(((intptr_t)Target) >> 32));
+ JCE.emitByte(0x41); // REX prefix
+ JCE.emitByte(0xFF); // callq *r10
+ JCE.emitByte(2 | (2 << 3) | (3 << 6));
+#else
+ JCE.emitByte(0xE8); // Call with 32 bit pc-rel destination...
+
+ JCE.emitWordLE((intptr_t)Target-JCE.getCurrentPCValue()-4);
+#endif
+
+ // This used to use 0xCD, but that value is used by JITMemoryManager to
+ // initialize the buffer with garbage, which means it may follow a
+ // noreturn function call, confusing LLVMX86CompilationCallback2. PR 4929.
+ JCE.emitByte(0xCE); // Interrupt - Just a marker identifying the stub!
+ return Result;
+}
+
+/// getPICJumpTableEntry - Returns the value of the jumptable entry for the
+/// specific basic block.
+uintptr_t X86JITInfo::getPICJumpTableEntry(uintptr_t BB, uintptr_t Entry) {
+#if defined(X86_64_JIT)
+ return BB - Entry;
+#else
+ return BB - PICBase;
+#endif
+}
+
+template<typename T> static void addUnaligned(void *Pos, T Delta) {
+ T Value;
+ std::memcpy(reinterpret_cast<char*>(&Value), reinterpret_cast<char*>(Pos),
+ sizeof(T));
+ Value += Delta;
+ std::memcpy(reinterpret_cast<char*>(Pos), reinterpret_cast<char*>(&Value),
+ sizeof(T));
+}
+
+/// relocate - Before the JIT can run a block of code that has been emitted,
+/// it must rewrite the code to contain the actual addresses of any
+/// referenced global symbols.
+void X86JITInfo::relocate(void *Function, MachineRelocation *MR,
+ unsigned NumRelocs, unsigned char* GOTBase) {
+ for (unsigned i = 0; i != NumRelocs; ++i, ++MR) {
+ void *RelocPos = (char*)Function + MR->getMachineCodeOffset();
+ intptr_t ResultPtr = (intptr_t)MR->getResultPointer();
+ switch ((X86::RelocationType)MR->getRelocationType()) {
+ case X86::reloc_pcrel_word: {
+ // PC relative relocation, add the relocated value to the value already in
+ // memory, after we adjust it for where the PC is.
+ ResultPtr = ResultPtr -(intptr_t)RelocPos - 4 - MR->getConstantVal();
+ addUnaligned<unsigned>(RelocPos, ResultPtr);
+ break;
+ }
+ case X86::reloc_picrel_word: {
+ // PIC base relative relocation, add the relocated value to the value
+ // already in memory, after we adjust it for where the PIC base is.
+ ResultPtr = ResultPtr - ((intptr_t)Function + MR->getConstantVal());
+ addUnaligned<unsigned>(RelocPos, ResultPtr);
+ break;
+ }
+ case X86::reloc_absolute_word:
+ case X86::reloc_absolute_word_sext:
+ // Absolute relocation, just add the relocated value to the value already
+ // in memory.
+ addUnaligned<unsigned>(RelocPos, ResultPtr);
+ break;
+ case X86::reloc_absolute_dword:
+ addUnaligned<intptr_t>(RelocPos, ResultPtr);
+ break;
+ }
+ }
+}
+
+char* X86JITInfo::allocateThreadLocalMemory(size_t size) {
+#if defined(X86_32_JIT) && !defined(__APPLE__) && !defined(_MSC_VER)
+ TLSOffset -= size;
+ return TLSOffset;
+#else
+ llvm_unreachable("Cannot allocate thread local storage on this arch!");
+#endif
+}
--- /dev/null
+//===-- X86JITInfo.h - X86 implementation of the JIT interface --*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the X86 implementation of the TargetJITInfo class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef X86JITINFO_H
+#define X86JITINFO_H
+
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/IR/Function.h"
+#include "llvm/Target/TargetJITInfo.h"
+
+namespace llvm {
+ class X86Subtarget;
+
+ class X86JITInfo : public TargetJITInfo {
+ uintptr_t PICBase;
+ char *TLSOffset;
+ bool useSSE;
+ public:
+ explicit X86JITInfo(bool UseSSE);
+
+ /// replaceMachineCodeForFunction - Make it so that calling the function
+ /// whose machine code is at OLD turns into a call to NEW, perhaps by
+ /// overwriting OLD with a branch to NEW. This is used for self-modifying
+ /// code.
+ ///
+ void replaceMachineCodeForFunction(void *Old, void *New) override;
+
+ /// emitGlobalValueIndirectSym - Use the specified JITCodeEmitter object
+ /// to emit an indirect symbol which contains the address of the specified
+ /// ptr.
+ void *emitGlobalValueIndirectSym(const GlobalValue* GV, void *ptr,
+ JITCodeEmitter &JCE) override;
+
+ // getStubLayout - Returns the size and alignment of the largest call stub
+ // on X86.
+ StubLayout getStubLayout() override;
+
+ /// emitFunctionStub - Use the specified JITCodeEmitter object to emit a
+ /// small native function that simply calls the function at the specified
+ /// address.
+ void *emitFunctionStub(const Function* F, void *Target,
+ JITCodeEmitter &JCE) override;
+
+ /// getPICJumpTableEntry - Returns the value of the jumptable entry for the
+ /// specific basic block.
+ uintptr_t getPICJumpTableEntry(uintptr_t BB, uintptr_t JTBase) override;
+
+ /// getLazyResolverFunction - Expose the lazy resolver to the JIT.
+ LazyResolverFn getLazyResolverFunction(JITCompilerFn) override;
+
+ /// relocate - Before the JIT can run a block of code that has been emitted,
+ /// it must rewrite the code to contain the actual addresses of any
+ /// referenced global symbols.
+ void relocate(void *Function, MachineRelocation *MR,
+ unsigned NumRelocs, unsigned char* GOTBase) override;
+
+ /// allocateThreadLocalMemory - Each target has its own way of
+ /// handling thread local variables. This method returns a value only
+ /// meaningful to the target.
+ char* allocateThreadLocalMemory(size_t size) override;
+
+ /// setPICBase / getPICBase - Getter / setter of PICBase, used to compute
+ /// PIC jumptable entry.
+ void setPICBase(uintptr_t Base) { PICBase = Base; }
+ uintptr_t getPICBase() const { return PICBase; }
+ };
+}
+
+#endif
DL(computeDataLayout(*this)), TSInfo(DL),
InstrInfo(initializeSubtargetDependencies(CPU, FS)), TLInfo(TM),
FrameLowering(TargetFrameLowering::StackGrowsDown, getStackAlignment(),
- is64Bit() ? -8 : -4) {}
+ is64Bit() ? -8 : -4),
+ JITInfo(hasSSE1()) {}
bool X86Subtarget::enableEarlyIfConversion() const {
return hasCMov() && X86EarlyIfConv;
#include "X86FrameLowering.h"
#include "X86ISelLowering.h"
#include "X86InstrInfo.h"
+#include "X86JITInfo.h"
#include "X86SelectionDAGInfo.h"
#include "llvm/ADT/Triple.h"
#include "llvm/IR/CallingConv.h"
X86InstrInfo InstrInfo;
X86TargetLowering TLInfo;
X86FrameLowering FrameLowering;
+ X86JITInfo JITInfo;
public:
/// This constructor initializes the data members to match that
const X86RegisterInfo *getRegisterInfo() const override {
return &getInstrInfo()->getRegisterInfo();
}
+ X86JITInfo *getJITInfo() override { return &JITInfo; }
/// getStackAlignment - Returns the minimum alignment known to hold of the
/// stack frame on entry to the function and which must be maintained by every
return ShouldPrint;
}
+
+bool X86TargetMachine::addCodeEmitter(PassManagerBase &PM,
+ JITCodeEmitter &JCE) {
+ PM.add(createX86JITCodeEmitterPass(*this, JCE));
+
+ return false;
+}
CodeGenOpt::Level OL);
const X86Subtarget *getSubtargetImpl() const override { return &Subtarget; }
+ X86Subtarget *getSubtargetImpl() {
+ return static_cast<X86Subtarget *>(TargetMachine::getSubtargetImpl());
+ }
+
/// \brief Register X86 analysis passes with a pass manager.
void addAnalysisPasses(PassManagerBase &PM) override;
// Set up the pass pipeline.
TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
+
+ bool addCodeEmitter(PassManagerBase &PM, JITCodeEmitter &JCE) override;
};
} // End llvm namespace
; RUN: %lli %s > /dev/null
+; XFAIL: arm
@.LC0 = internal global [10 x i8] c"argc: %d\0A\00" ; <[10 x i8]*> [#uses=1]
; RUN: %lli %s > /dev/null
+; XFAIL: arm
define i32 @foo(i32 %X, i32 %Y, double %A) {
%cond212 = fcmp une double %A, 1.000000e+00 ; <i1> [#uses=1]
; RUN: %lli %s > /dev/null
+; XFAIL: arm
define i32 @main() {
call i32 @mylog( i32 4 ) ; <i32>:1 [#uses=0]
; RUN: %lli %s > /dev/null
+; XFAIL: arm
define i32 @bar(i8* %X) {
; pointer should be 4 byte aligned!
; This testcase should return with an exit code of 1.
;
; RUN: not %lli %s
+; XFAIL: arm
@test = global i64 0 ; <i64*> [#uses=1]
; RUN: %lli %s test
+; XFAIL: arm
declare i32 @puts(i8*)
; RUN: %lli %s > /dev/null
+; XFAIL: arm
; This testcase failed to work because two variable sized allocas confused the
; local register allocator.
; RUN: %lli %s > /dev/null
+; XFAIL: arm
;
; Regression Test: EnvironmentTest.ll
; RUN: %lli %s > /dev/null
+; XFAIL: arm
; This testcase exposes a bug in the local register allocator where it runs out
; of registers (due to too many overlapping live ranges), but then attempts to
; RUN: %lli %s > /dev/null
+; XFAIL: arm
@A = global i32 0 ; <i32*> [#uses=1]
; PR672
; RUN: %lli %s
+; XFAIL: arm
define i32 @main() {
%f = bitcast i32 (i32, i32*, i32)* @check_tail to i32* ; <i32*> [#uses=1]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
@.LC0 = internal global [10 x i8] c"argc: %d\0A\00" ; <[10 x i8]*> [#uses=1]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @foo(i32 %X, i32 %Y, double %A) {
%cond212 = fcmp une double %A, 1.000000e+00 ; <i1> [#uses=1]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @main() {
call i32 @mylog( i32 4 ) ; <i32>:1 [#uses=0]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @main() {
; <label>:0
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
; We were accidentally inverting the signedness of right shifts. Whoops.
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @main() {
%X = fadd double 0.000000e+00, 1.000000e+00 ; <double> [#uses=1]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @bar(i8* %X) {
; pointer should be 4 byte aligned!
; This testcase should return with an exit code of 1.
;
-; RUN: not %lli %s
+; RUN: not %lli_mcjit %s
@test = global i64 0 ; <i64*> [#uses=1]
-; RUN: %lli %s test
+; RUN: %lli_mcjit %s test
declare i32 @puts(i8*)
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
target datalayout = "e-p:32:32"
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
; Testcase distilled from 256.bzip2.
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
; Testcase distilled from 256.bzip2.
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
; This testcase failed to work because two variable sized allocas confused the
; local register allocator.
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
;
; Regression Test: EnvironmentTest.ll
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
; This testcase exposes a bug in the local register allocator where it runs out
; of registers (due to too many overlapping live ranges), but then attempts to
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
@A = global i32 0 ; <i32*> [#uses=1]
; PR672
-; RUN: %lli %s
+; RUN: %lli_mcjit %s
; XFAIL: mcjit-ia32
define i32 @main() {
-; RUN: %lli -force-interpreter %s
+; RUN: %lli_mcjit -force-interpreter %s
; PR1836
define i32 @main() {
-; RUN: %lli -force-interpreter=true %s | FileCheck %s
+; RUN: %lli_mcjit -force-interpreter=true %s | FileCheck %s
; CHECK: 1
target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
-; RUN: %lli -force-interpreter=true %s > /dev/null
+; RUN: %lli_mcjit -force-interpreter=true %s > /dev/null
define i32 @main() {
%a = add i32 0, undef
-; RUN: %lli %s
+; RUN: %lli_mcjit %s
;
; Verify relocations to global symbols with addend work correctly.
;
-; RUN: %lli -extra-module=%p/Inputs/cross-module-b.ll %s > /dev/null
+; RUN: %lli_mcjit -extra-module=%p/Inputs/cross-module-b.ll %s > /dev/null
declare i32 @FB()
-; RUN: %lli -extra-module=%p/Inputs/cross-module-b.ll -relocation-model=pic -code-model=small %s > /dev/null
+; RUN: %lli_mcjit -extra-module=%p/Inputs/cross-module-b.ll -relocation-model=pic -code-model=small %s > /dev/null
; XFAIL: mips, i686, i386, arm
declare i32 @FB()
-; RUN: %lli -relocation-model=pic -code-model=large %s
+; RUN: %lli_mcjit -relocation-model=pic -code-model=large %s
; XFAIL: cygwin, win32, mingw, mips, i686, i386, aarch64, arm
declare i8* @__cxa_allocate_exception(i64)
declare void @__cxa_throw(i8*, i8*, i8*)
-; RUN: %lli -relocation-model=pic -code-model=small %s
+; RUN: %lli_mcjit -relocation-model=pic -code-model=small %s
; XFAIL: cygwin, win32, mingw, mips, i686, i386, darwin, aarch64, arm
declare i8* @__cxa_allocate_exception(i64)
declare void @__cxa_throw(i8*, i8*, i8*)
-; RUN: %lli %s
+; RUN: %lli_mcjit %s
; XFAIL: arm, cygwin, win32, mingw
declare i8* @__cxa_allocate_exception(i64)
declare void @__cxa_throw(i8*, i8*, i8*)
-; RUN: %lli -force-interpreter=true %s | FileCheck %s
+; RUN: %lli_mcjit -force-interpreter=true %s | FileCheck %s
; CHECK: 40091eb8
define i32 @test(double %x) {
-; RUN: %lli -relocation-model=pic -code-model=small %s > /dev/null
+; RUN: %lli_mcjit -relocation-model=pic -code-model=small %s > /dev/null
; XFAIL: mips, i686, i386, darwin, aarch64, arm
@.LC0 = internal global [12 x i8] c"Hello World\00" ; <[12 x i8]*> [#uses=1]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
@.LC0 = internal global [12 x i8] c"Hello World\00" ; <[12 x i8]*> [#uses=1]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
@X = global i32 7 ; <i32*> [#uses=0]
@msg = internal global [13 x i8] c"Hello World\0A\00" ; <[13 x i8]*> [#uses=1]
; This first line will generate the .o files for the next run line
; RUN: rm -rf %t.cachedir %t.cachedir2 %t.cachedir3
; RUN: mkdir -p %t.cachedir %t.cachedir2 %t.cachedir3
-; RUN: %lli -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -enable-cache-manager -object-cache-dir=%t.cachedir %s
+; RUN: %lli_mcjit -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -enable-cache-manager -object-cache-dir=%t.cachedir %s
; Collect generated objects.
; RUN: find %t.cachedir -type f -name 'multi-module-?.o' -exec mv -v '{}' %t.cachedir2 ';'
; This line tests MCJIT object loading
-; RUN: %lli -extra-object=%t.cachedir2/multi-module-b.o -extra-object=%t.cachedir2/multi-module-c.o %s
+; RUN: %lli_mcjit -extra-object=%t.cachedir2/multi-module-b.o -extra-object=%t.cachedir2/multi-module-c.o %s
; These lines put the object files into an archive
; RUN: llvm-ar r %t.cachedir3/load-object.a %t.cachedir2/multi-module-b.o
; RUN: llvm-ar r %t.cachedir3/load-object.a %t.cachedir2/multi-module-c.o
; This line test MCJIT archive loading
-; RUN: %lli -extra-archive=%t.cachedir3/load-object.a %s
+; RUN: %lli_mcjit -extra-archive=%t.cachedir3/load-object.a %s
declare i32 @FB()
-; RUN: %lli -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll %s > /dev/null
+; RUN: %lli_mcjit -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll %s > /dev/null
declare i32 @FB()
-; RUN: %lli -extra-module=%p/Inputs/multi-module-eh-b.ll %s
+; RUN: %lli_mcjit -extra-module=%p/Inputs/multi-module-eh-b.ll %s
; XFAIL: arm, cygwin, win32, mingw
declare i8* @__cxa_allocate_exception(i64)
declare void @__cxa_throw(i8*, i8*, i8*)
-; RUN: %lli -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -relocation-model=pic -code-model=small %s > /dev/null
+; RUN: %lli_mcjit -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -relocation-model=pic -code-model=small %s > /dev/null
; XFAIL: mips, i686, i386, arm
declare i32 @FB()
-; RUN: %lli -code-model=small %s > /dev/null
+; RUN: %lli_mcjit -code-model=small %s > /dev/null
; XFAIL: mips
;
; FIXME: Merge this file with non-extern-addend.ll once AArch64 supports PC-rel
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @foo(i32 %x, i32 %y, double %d) {
entry:
-; RUN: %lli -O0 -disable-lazy-compilation=false %s
+; RUN: %lli_mcjit -O0 -disable-lazy-compilation=false %s
; The intention of this test is to verify that symbols mapped to COMMON in ELF
; work as expected.
-; RUN: %lli -extra-module=%p/Inputs/cross-module-b.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
+; RUN: %lli_mcjit -extra-module=%p/Inputs/cross-module-b.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
declare i32 @FB()
-; RUN: %lli -extra-module=%p/Inputs/cross-module-b.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext -relocation-model=pic -code-model=small %s > /dev/null
+; RUN: %lli_mcjit -extra-module=%p/Inputs/cross-module-b.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext -relocation-model=pic -code-model=small %s > /dev/null
; XFAIL: mips, i686, i386, arm
declare i32 @FB()
-; RUN: %lli -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
+; RUN: %lli_mcjit -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
declare i32 @FB()
-; RUN: %lli -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext -relocation-model=pic -code-model=small %s > /dev/null
+; RUN: %lli_mcjit -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -disable-lazy-compilation=true -remote-mcjit -mcjit-remote-process=lli-child-target%exeext -relocation-model=pic -code-model=small %s > /dev/null
; XFAIL: mips, i686, i386, arm
declare i32 @FB()
-; RUN: %lli -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
+; RUN: %lli_mcjit -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
define i32 @bar() nounwind {
ret i32 0
-; RUN: %lli -remote-mcjit -disable-lazy-compilation=false -mcjit-remote-process=lli-child-target%exeext %s
+; RUN: %lli_mcjit -remote-mcjit -disable-lazy-compilation=false -mcjit-remote-process=lli-child-target%exeext %s
; XFAIL: *
; This test should fail until remote symbol resolution is supported.
-; RUN: %lli -remote-mcjit -disable-lazy-compilation=false -relocation-model=pic -code-model=small %s
+; RUN: %lli_mcjit -remote-mcjit -disable-lazy-compilation=false -relocation-model=pic -code-model=small %s
; XFAIL: *
; This function should fail until remote symbol resolution is supported.
-; RUN: %lli -remote-mcjit -O0 -disable-lazy-compilation=false -mcjit-remote-process=lli-child-target%exeext %s
+; RUN: %lli_mcjit -remote-mcjit -O0 -disable-lazy-compilation=false -mcjit-remote-process=lli-child-target%exeext %s
; The intention of this test is to verify that symbols mapped to COMMON in ELF
; work as expected.
-; RUN: %lli -remote-mcjit -O0 -mcjit-remote-process=lli-child-target%exeext %s
+; RUN: %lli_mcjit -remote-mcjit -O0 -mcjit-remote-process=lli-child-target%exeext %s
; Check that a variable is always aligned as specified.
-; RUN: %lli -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
+; RUN: %lli_mcjit -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
define double @test(double* %DP, double %Arg) nounwind {
%D = load double* %DP ; <double> [#uses=1]
-; RUN: %lli -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
+; RUN: %lli_mcjit -remote-mcjit -mcjit-remote-process=lli-child-target%exeext %s > /dev/null
@count = global i32 1, align 4
-; RUN: %lli -remote-mcjit -relocation-model=pic -code-model=small %s > /dev/null
+; RUN: %lli_mcjit -remote-mcjit -relocation-model=pic -code-model=small %s > /dev/null
; XFAIL: mips, aarch64, arm, i686, i386
@count = global i32 1, align 4
-; RUN: %lli -remote-mcjit -O0 -mcjit-remote-process=lli-child-target%exeext %s
+; RUN: %lli_mcjit -remote-mcjit -O0 -mcjit-remote-process=lli-child-target%exeext %s
@.str = private unnamed_addr constant [6 x i8] c"data1\00", align 1
@ptr = global i8* getelementptr inbounds ([6 x i8]* @.str, i32 0, i32 0), align 4
-; RUN: %lli -remote-mcjit -O0 -relocation-model=pic -code-model=small %s
+; RUN: %lli_mcjit -remote-mcjit -O0 -relocation-model=pic -code-model=small %s
; XFAIL: mips, aarch64, arm, i686, i386
@.str = private unnamed_addr constant [6 x i8] c"data1\00", align 1
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @main() {
ret i32 0
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @bar() {
ret i32 0
-; RUN: %lli -disable-lazy-compilation=false -relocation-model=pic -code-model=small %s
+; RUN: %lli_mcjit -disable-lazy-compilation=false -relocation-model=pic -code-model=small %s
; XFAIL: mips, i686, i386, aarch64, arm
define i32 @main() nounwind {
-; RUN: %lli -disable-lazy-compilation=false %s
+; RUN: %lli_mcjit -disable-lazy-compilation=false %s
define i32 @main() nounwind {
entry:
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @main() {
%A = add i8 0, 12 ; <i8> [#uses=1]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
; test unconditional branch
define i32 @main() {
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @_Z14func_exit_codev() nounwind uwtable {
entry:
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
declare void @exit(i32)
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @foo() {
ret i32 0
-; RUN: %lli -O0 %s
+; RUN: %lli_mcjit -O0 %s
; This test checks that common symbols have been allocated addresses honouring
; the alignment requirement.
-; RUN: %lli -O0 -disable-lazy-compilation=false %s
+; RUN: %lli_mcjit -O0 -disable-lazy-compilation=false %s
; The intention of this test is to verify that symbols mapped to COMMON in ELF
; work as expected.
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
; This tests to make sure that we can evaluate weird constant expressions
-; RUN: %lli -O0 %s
+; RUN: %lli_mcjit -O0 %s
; Check that a variable is always aligned as specified.
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define double @test(double* %DP, double %Arg) {
%D = load double* %DP ; <double> [#uses=1]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define double @test(double* %DP, double %Arg) {
%D = load double* %DP ; <double> [#uses=1]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
; XFAIL: darwin
@var = global i32 1, align 4
@llvm.global_ctors = appending global [1 x { i32, void ()* }] [{ i32, void ()* } { i32 65535, void ()* @ctor_func }]
-; RUN: %lli -relocation-model=pic -code-model=small %s > /dev/null
+; RUN: %lli_mcjit -relocation-model=pic -code-model=small %s > /dev/null
; XFAIL: mips, aarch64, arm, i686, i386
@count = global i32 1, align 4
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
@count = global i32 1, align 4
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
@count = global i32 0, align 4
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define void @test(i8* %P, i16* %P.upgrd.1, i32* %P.upgrd.2, i64* %P.upgrd.3) {
%V = load i8* %P ; <i8> [#uses=1]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @main() nounwind uwtable {
entry:
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @main() {
%A = and i8 4, 8 ; <i8> [#uses=2]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @main() {
; <label>:0
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
; test phi node
@Y = global i32 6 ; <i32*> [#uses=1]
-; RUN: %lli -O0 -relocation-model=pic -code-model=small %s
+; RUN: %lli_mcjit -O0 -relocation-model=pic -code-model=small %s
; XFAIL: mips, aarch64, arm, i686, i386
@.str = private unnamed_addr constant [6 x i8] c"data1\00", align 1
-; RUN: %lli -O0 %s
+; RUN: %lli_mcjit -O0 %s
@.str = private unnamed_addr constant [6 x i8] c"data1\00", align 1
@ptr = global i8* getelementptr inbounds ([6 x i8]* @.str, i32 0, i32 0), align 4
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
; test return instructions
define void @test1() {
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @main() nounwind uwtable {
entry:
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @main() {
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @main() {
%int1 = add i32 0, 0 ; <i32> [#uses=6]
-; RUN: %lli %s > /dev/null
+; RUN: %lli_mcjit %s > /dev/null
define i32 @main() {
%shamt = add i8 0, 1 ; <i8> [#uses=8]
; RUN: %lli %s > /dev/null
+; XFAIL: arm
@.LC0 = internal global [12 x i8] c"Hello World\00" ; <[12 x i8]*> [#uses=1]
; RUN: %lli %s > /dev/null
+; XFAIL: arm
@X = global i32 7 ; <i32*> [#uses=0]
@msg = internal global [13 x i8] c"Hello World\0A\00" ; <[13 x i8]*> [#uses=1]
; RUN: %lli %s > /dev/null
+; XFAIL: arm
define i64 @foo() {
ret i64 42
; RUN: %lli %s > /dev/null
+; XFAIL: arm
define i32 @bar() {
ret i32 0
; RUN: %lli -disable-lazy-compilation=false %s
+; XFAIL: arm
define i32 @main() nounwind {
entry:
; RUN: %lli %s > /dev/null
+; XFAIL: arm
define i32 @_Z14func_exit_codev() nounwind uwtable {
entry:
; RUN: %lli %s > /dev/null
+; XFAIL: arm
declare void @exit(i32)
; RUN: %lli -O0 -disable-lazy-compilation=false %s
+; XFAIL: arm
; The intention of this test is to verify that symbols mapped to COMMON in ELF
; work as expected.
; RUN: %lli %s > /dev/null
+; XFAIL: arm
define double @test(double* %DP, double %Arg) {
%D = load double* %DP ; <double> [#uses=1]
; RUN: %lli %s > /dev/null
+; XFAIL: arm
define double @test(double* %DP, double %Arg) {
%D = load double* %DP ; <double> [#uses=1]
; RUN: %lli %s > /dev/null
+; XFAIL: arm
@count = global i32 1, align 4
; RUN: %lli %s > /dev/null
+; XFAIL: arm
@count = global i32 0, align 4
; RUN: %lli %s > /dev/null
+; XFAIL: arm
define void @test(i8* %P, i16* %P.upgrd.1, i32* %P.upgrd.2, i64* %P.upgrd.3) {
%V = load i8* %P ; <i8> [#uses=1]
; RUN: %lli %s > /dev/null
+; XFAIL: arm
define i32 @main() nounwind uwtable {
entry:
###
-lli = 'lli'
+# Provide a command line for mcjit tests
+lli_mcjit = 'lli -use-mcjit'
# The target triple used by default by lli is the process target triple (some
# triple appropriate for generating code for the current process) but because
# we don't support COFF in MCJIT well enough for the tests, force ELF format on
# Windows. FIXME: the process target triple should be used here, but this is
# difficult to obtain on Windows.
if re.search(r'cygwin|mingw32|win32', config.host_triple):
- lli += ' -mtriple='+config.host_triple+'-elf'
-config.substitutions.append( ('%lli', lli ) )
+ lli_mcjit += ' -mtriple='+config.host_triple+'-elf'
+config.substitutions.append( ('%lli_mcjit', lli_mcjit) )
# Similarly, have a macro to use llc with DWARF even when the host is win32.
llc_dwarf = 'llc'
llc_dwarf += ' -mtriple='+config.target_triple.replace('-win32', '-mingw32')
config.substitutions.append( ('%llc_dwarf', llc_dwarf) )
+# Process jit implementation option
+jit_impl_cfg = lit_config.params.get('jit_impl', None)
+if jit_impl_cfg == 'mcjit':
+ # When running with mcjit, mangle -mcjit into target triple
+ # and add -use-mcjit flag to lli invocation
+ if 'i386' in config.target_triple or 'i686' in config.target_triple:
+ config.target_triple += jit_impl_cfg + '-ia32'
+ elif 'x86_64' in config.target_triple:
+ config.target_triple += jit_impl_cfg + '-ia64'
+ else:
+ config.target_triple += jit_impl_cfg
+
+ config.substitutions.append( ('%lli', 'lli -use-mcjit') )
+else:
+ config.substitutions.append( ('%lli', 'lli') )
+
# Add site-specific substitutions.
config.substitutions.append( ('%ocamlopt', config.ocamlopt_executable) )
config.substitutions.append( ('%llvmshlibdir', config.llvm_shlib_dir) )
IRReader
Instrumentation
Interpreter
+ JIT
MC
MCJIT
Object
type = Tool
name = lli
parent = Tools
-required_libraries = AsmParser BitReader IRReader Instrumentation Interpreter MCJIT NativeCodeGen SelectionDAG Native
+required_libraries = AsmParser BitReader IRReader Instrumentation Interpreter JIT MCJIT NativeCodeGen SelectionDAG Native
include $(LEVEL)/Makefile.config
-LINK_COMPONENTS := mcjit instrumentation interpreter nativecodegen bitreader asmparser irreader selectiondag native
+LINK_COMPONENTS := mcjit jit instrumentation interpreter nativecodegen bitreader asmparser irreader selectiondag native
# If Intel JIT Events support is confiured, link against the LLVM Intel JIT
# Events interface library
#include "llvm/CodeGen/LinkAllCodegenComponents.h"
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/Interpreter.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/ExecutionEngine/JITEventListener.h"
#include "llvm/ExecutionEngine/JITMemoryManager.h"
#include "llvm/ExecutionEngine/MCJIT.h"
cl::desc("Force interpretation: disable JIT"),
cl::init(false));
+ cl::opt<bool> UseMCJIT(
+ "use-mcjit", cl::desc("Enable use of the MC-based JIT (if available)"),
+ cl::init(false));
+
cl::opt<bool> DebugIR(
"debug-ir", cl::desc("Generate debug information to allow debugging IR."),
cl::init(false));
}
if (EnableCacheManager) {
- std::string CacheName("file:");
- CacheName.append(InputFile);
- Mod->setModuleIdentifier(CacheName);
+ if (UseMCJIT) {
+ std::string CacheName("file:");
+ CacheName.append(InputFile);
+ Mod->setModuleIdentifier(CacheName);
+ } else
+ errs() << "warning: -enable-cache-manager can only be used with MCJIT.";
}
// If not jitting lazily, load the whole bitcode file eagerly too.
}
if (DebugIR) {
+ if (!UseMCJIT) {
+ errs() << "warning: -debug-ir used without -use-mcjit. Only partial debug"
+ << " information will be emitted by the non-MC JIT engine. To see full"
+ << " source debug information, enable the flag '-use-mcjit'.\n";
+
+ }
ModulePass *DebugIRPass = createDebugIRPass();
DebugIRPass->runOnModule(*Mod);
}
// Enable MCJIT if desired.
RTDyldMemoryManager *RTDyldMM = nullptr;
- if (!ForceInterpreter) {
+ if (UseMCJIT && !ForceInterpreter) {
+ builder.setUseMCJIT(true);
if (RemoteMCJIT)
RTDyldMM = new RemoteMemoryManager();
else
return 1;
}
if (EnableCacheManager) {
- std::string CacheName("file:");
- CacheName.append(ExtraModules[i]);
- XMod->setModuleIdentifier(CacheName);
+ if (UseMCJIT) {
+ std::string CacheName("file:");
+ CacheName.append(ExtraModules[i]);
+ XMod->setModuleIdentifier(CacheName);
+ }
+ // else, we already printed a warning above.
}
EE->addModule(XMod);
}
NULL);
// Run static constructors.
- if (!ForceInterpreter) {
+ if (UseMCJIT && !ForceInterpreter) {
// Give MCJIT a chance to apply relocations and set page permissions.
EE->finalizeObject();
}
EE->runStaticConstructorsDestructors(false);
+ if (!UseMCJIT && NoLazyCompilation) {
+ for (Module::iterator I = Mod->begin(), E = Mod->end(); I != E; ++I) {
+ Function *Fn = &*I;
+ if (Fn != EntryFn && !Fn->isDeclaration())
+ EE->getPointerToFunction(Fn);
+ }
+ }
+
// Trigger compilation separately so code regions that need to be
// invalidated will be known.
(void)EE->getPointerToFunction(EntryFn);
type = Tool
name = llvm-jitlistener
parent = Tools
-required_libraries = AsmParser BitReader IRReader Interpreter MCJIT NativeCodeGen Object SelectionDAG Native
+required_libraries = AsmParser BitReader IRReader Interpreter JIT MCJIT NativeCodeGen Object SelectionDAG Native
.setEngineKind(EngineKind::JIT)
.setErrorStr(&Error)
.setJITMemoryManager(MemMgr)
+ .setUseMCJIT(true)
.create());
if (Error.empty() == false)
errs() << Error;
type = Tool
name = llvm-rtdyld
parent = Tools
-required_libraries = MC Object RuntimeDyld Support all-targets
+required_libraries = JIT MC Object RuntimeDyld Support all-targets
LEVEL := ../..
TOOLNAME := llvm-rtdyld
-LINK_COMPONENTS := all-targets support MC object RuntimeDyld MCJIT debuginfo
+LINK_COMPONENTS := all-targets support MC object RuntimeDyld JIT debuginfo
# This tool has no plugins, optimize startup time.
TOOL_NO_EXPORTS := 1
ExecutionEngineTest.cpp
)
-# Include MCJIT tests only if native arch is a built JIT target.
+# Include JIT/MCJIT tests only if native arch is a built JIT target.
list(FIND LLVM_TARGETS_TO_BUILD "${LLVM_NATIVE_ARCH}" build_idx)
list(FIND LLVM_TARGETS_WITH_JIT "${LLVM_NATIVE_ARCH}" jit_idx)
if (NOT build_idx LESS 0 AND NOT jit_idx LESS 0)
+ add_subdirectory(JIT)
add_subdirectory(MCJIT)
endif()
--- /dev/null
+set(LLVM_LINK_COMPONENTS
+ AsmParser
+ BitReader
+ BitWriter
+ Core
+ ExecutionEngine
+ JIT
+ MC
+ Support
+ nativecodegen
+ )
+
+# HACK: Declare a couple of source files as optionally compiled to satisfy the
+# missing-file-checker in LLVM's weird CMake build.
+set(LLVM_OPTIONAL_SOURCES
+ IntelJITEventListenerTest.cpp
+ OProfileJITEventListenerTest.cpp
+ )
+
+if( LLVM_USE_INTEL_JITEVENTS )
+ set(ProfileTestSources
+ IntelJITEventListenerTest.cpp
+ )
+ set(LLVM_LINK_COMPONENTS
+ ${LLVM_LINK_COMPONENTS}
+ DebugInfo
+ IntelJITEvents
+ Object
+ )
+endif( LLVM_USE_INTEL_JITEVENTS )
+
+if( LLVM_USE_OPROFILE )
+ set(ProfileTestSources
+ ${ProfileTestSources}
+ OProfileJITEventListenerTest.cpp
+ )
+ set(LLVM_LINK_COMPONENTS
+ ${LLVM_LINK_COMPONENTS}
+ OProfileJIT
+ )
+endif( LLVM_USE_OPROFILE )
+
+set(JITTestsSources
+ JITEventListenerTest.cpp
+ JITMemoryManagerTest.cpp
+ JITTest.cpp
+ MultiJITTest.cpp
+ ${ProfileTestSources}
+ )
+
+if(MSVC)
+ list(APPEND JITTestsSources JITTests.def)
+endif()
+
+# The JIT tests need to dlopen things.
+set(LLVM_NO_DEAD_STRIP 1)
+
+add_llvm_unittest(JITTests
+ ${JITTestsSources}
+ )
+
+if(MINGW OR CYGWIN)
+ set_property(TARGET JITTests PROPERTY LINK_FLAGS -Wl,--export-all-symbols)
+endif()
+set_target_properties(JITTests PROPERTIES ENABLE_EXPORTS 1)
--- /dev/null
+//===- JITEventListenerTest.cpp - Tests for Intel JITEventListener --------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "JITEventListenerTestCommon.h"
+
+using namespace llvm;
+
+// Because we want to keep the implementation details of the Intel API used to
+// communicate with Amplifier out of the public header files, the header below
+// is included from the source tree instead.
+#include "../../../lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h"
+
+#include <map>
+#include <list>
+
+namespace {
+
+// map of function ("method") IDs to source locations
+NativeCodeMap ReportedDebugFuncs;
+
+} // namespace
+
+/// Mock implementaion of Intel JIT API jitprofiling library
+namespace test_jitprofiling {
+
+int NotifyEvent(iJIT_JVM_EVENT EventType, void *EventSpecificData) {
+ switch (EventType) {
+ case iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED: {
+ EXPECT_TRUE(0 != EventSpecificData);
+ iJIT_Method_Load* msg = static_cast<iJIT_Method_Load*>(EventSpecificData);
+
+ ReportedDebugFuncs[msg->method_id];
+
+ for(unsigned int i = 0; i < msg->line_number_size; ++i) {
+ EXPECT_TRUE(0 != msg->line_number_table);
+ std::pair<std::string, unsigned int> loc(
+ std::string(msg->source_file_name),
+ msg->line_number_table[i].LineNumber);
+ ReportedDebugFuncs[msg->method_id].push_back(loc);
+ }
+ }
+ break;
+ case iJVM_EVENT_TYPE_METHOD_UNLOAD_START: {
+ EXPECT_TRUE(0 != EventSpecificData);
+ unsigned int UnloadId
+ = *reinterpret_cast<unsigned int*>(EventSpecificData);
+ EXPECT_TRUE(1 == ReportedDebugFuncs.erase(UnloadId));
+ }
+ default:
+ break;
+ }
+ return 0;
+}
+
+iJIT_IsProfilingActiveFlags IsProfilingActive(void) {
+ // for testing, pretend we have an Intel Parallel Amplifier XE 2011
+ // instance attached
+ return iJIT_SAMPLING_ON;
+}
+
+unsigned int GetNewMethodID(void) {
+ static unsigned int id = 0;
+ return ++id;
+}
+
+} //namespace test_jitprofiling
+
+class IntelJITEventListenerTest
+ : public JITEventListenerTestBase<IntelJITEventsWrapper> {
+public:
+ IntelJITEventListenerTest()
+ : JITEventListenerTestBase<IntelJITEventsWrapper>(
+ new IntelJITEventsWrapper(test_jitprofiling::NotifyEvent, 0,
+ test_jitprofiling::IsProfilingActive, 0, 0,
+ test_jitprofiling::GetNewMethodID))
+ {
+ EXPECT_TRUE(0 != MockWrapper);
+
+ Listener.reset(JITEventListener::createIntelJITEventListener(
+ MockWrapper.release()));
+ EXPECT_TRUE(0 != Listener);
+ EE->RegisterJITEventListener(Listener.get());
+ }
+};
+
+TEST_F(IntelJITEventListenerTest, NoDebugInfo) {
+ TestNoDebugInfo(ReportedDebugFuncs);
+}
+
+TEST_F(IntelJITEventListenerTest, SingleLine) {
+ TestSingleLine(ReportedDebugFuncs);
+}
+
+TEST_F(IntelJITEventListenerTest, MultipleLines) {
+ TestMultipleLines(ReportedDebugFuncs);
+}
+
+// This testcase is disabled because the Intel JIT API does not support a single
+// JITted function with source lines associated with multiple files
+/*
+TEST_F(IntelJITEventListenerTest, MultipleFiles) {
+ TestMultipleFiles(ReportedDebugFuncs);
+}
+*/
+
+testing::Environment* const jit_env =
+ testing::AddGlobalTestEnvironment(new JITEnvironment);
--- /dev/null
+//===- JITEventListenerTest.cpp - Unit tests for JITEventListeners --------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/JITEventListener.h"
+#include "llvm/CodeGen/MachineCodeInfo.h"
+#include "llvm/ExecutionEngine/JIT.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/TypeBuilder.h"
+#include "llvm/Support/TargetSelect.h"
+#include "gtest/gtest.h"
+#include <vector>
+
+using namespace llvm;
+
+namespace {
+
+struct FunctionEmittedEvent {
+ // Indices are local to the RecordingJITEventListener, since the
+ // JITEventListener interface makes no guarantees about the order of
+ // calls between Listeners.
+ unsigned Index;
+ const Function *F;
+ void *Code;
+ size_t Size;
+ JITEvent_EmittedFunctionDetails Details;
+};
+struct FunctionFreedEvent {
+ unsigned Index;
+ void *Code;
+};
+
+struct RecordingJITEventListener : public JITEventListener {
+ std::vector<FunctionEmittedEvent> EmittedEvents;
+ std::vector<FunctionFreedEvent> FreedEvents;
+
+ unsigned NextIndex;
+
+ RecordingJITEventListener() : NextIndex(0) {}
+
+ virtual void NotifyFunctionEmitted(const Function &F,
+ void *Code, size_t Size,
+ const EmittedFunctionDetails &Details) {
+ FunctionEmittedEvent Event = {NextIndex++, &F, Code, Size, Details};
+ EmittedEvents.push_back(Event);
+ }
+
+ virtual void NotifyFreeingMachineCode(void *OldPtr) {
+ FunctionFreedEvent Event = {NextIndex++, OldPtr};
+ FreedEvents.push_back(Event);
+ }
+};
+
+class JITEventListenerTest : public testing::Test {
+ protected:
+ JITEventListenerTest()
+ : M(new Module("module", getGlobalContext())),
+ EE(EngineBuilder(M)
+ .setEngineKind(EngineKind::JIT)
+ .create()) {
+ }
+
+ Module *M;
+ const std::unique_ptr<ExecutionEngine> EE;
+};
+
+// Tests on SystemZ disabled as we're running the old JIT
+#if !defined(__s390__) && !defined(__aarch64__)
+Function *buildFunction(Module *M) {
+ Function *Result = Function::Create(
+ TypeBuilder<int32_t(int32_t), false>::get(getGlobalContext()),
+ GlobalValue::ExternalLinkage, "id", M);
+ Value *Arg = Result->arg_begin();
+ BasicBlock *BB = BasicBlock::Create(M->getContext(), "entry", Result);
+ ReturnInst::Create(M->getContext(), Arg, BB);
+ return Result;
+}
+
+// Tests that a single JITEventListener follows JIT events accurately.
+TEST_F(JITEventListenerTest, Simple) {
+ RecordingJITEventListener Listener;
+ EE->RegisterJITEventListener(&Listener);
+ Function *F1 = buildFunction(M);
+ Function *F2 = buildFunction(M);
+
+ void *F1_addr = EE->getPointerToFunction(F1);
+ void *F2_addr = EE->getPointerToFunction(F2);
+ EE->getPointerToFunction(F1); // Should do nothing.
+ EE->freeMachineCodeForFunction(F1);
+ EE->freeMachineCodeForFunction(F2);
+
+ ASSERT_EQ(2U, Listener.EmittedEvents.size());
+ ASSERT_EQ(2U, Listener.FreedEvents.size());
+
+ EXPECT_EQ(0U, Listener.EmittedEvents[0].Index);
+ EXPECT_EQ(F1, Listener.EmittedEvents[0].F);
+ EXPECT_EQ(F1_addr, Listener.EmittedEvents[0].Code);
+ EXPECT_LT(0U, Listener.EmittedEvents[0].Size)
+ << "We don't know how big the function will be, but it had better"
+ << " contain some bytes.";
+
+ EXPECT_EQ(1U, Listener.EmittedEvents[1].Index);
+ EXPECT_EQ(F2, Listener.EmittedEvents[1].F);
+ EXPECT_EQ(F2_addr, Listener.EmittedEvents[1].Code);
+ EXPECT_LT(0U, Listener.EmittedEvents[1].Size)
+ << "We don't know how big the function will be, but it had better"
+ << " contain some bytes.";
+
+ EXPECT_EQ(2U, Listener.FreedEvents[0].Index);
+ EXPECT_EQ(F1_addr, Listener.FreedEvents[0].Code);
+
+ EXPECT_EQ(3U, Listener.FreedEvents[1].Index);
+ EXPECT_EQ(F2_addr, Listener.FreedEvents[1].Code);
+
+ F1->eraseFromParent();
+ F2->eraseFromParent();
+}
+
+// Tests that a single JITEventListener follows JIT events accurately.
+TEST_F(JITEventListenerTest, MultipleListenersDontInterfere) {
+ RecordingJITEventListener Listener1;
+ RecordingJITEventListener Listener2;
+ RecordingJITEventListener Listener3;
+ Function *F1 = buildFunction(M);
+ Function *F2 = buildFunction(M);
+
+ EE->RegisterJITEventListener(&Listener1);
+ EE->RegisterJITEventListener(&Listener2);
+ void *F1_addr = EE->getPointerToFunction(F1);
+ EE->RegisterJITEventListener(&Listener3);
+ EE->UnregisterJITEventListener(&Listener1);
+ void *F2_addr = EE->getPointerToFunction(F2);
+ EE->UnregisterJITEventListener(&Listener2);
+ EE->UnregisterJITEventListener(&Listener3);
+ EE->freeMachineCodeForFunction(F1);
+ EE->RegisterJITEventListener(&Listener2);
+ EE->RegisterJITEventListener(&Listener3);
+ EE->RegisterJITEventListener(&Listener1);
+ EE->freeMachineCodeForFunction(F2);
+ EE->UnregisterJITEventListener(&Listener1);
+ EE->UnregisterJITEventListener(&Listener2);
+ EE->UnregisterJITEventListener(&Listener3);
+
+ // Listener 1.
+ ASSERT_EQ(1U, Listener1.EmittedEvents.size());
+ ASSERT_EQ(1U, Listener1.FreedEvents.size());
+
+ EXPECT_EQ(0U, Listener1.EmittedEvents[0].Index);
+ EXPECT_EQ(F1, Listener1.EmittedEvents[0].F);
+ EXPECT_EQ(F1_addr, Listener1.EmittedEvents[0].Code);
+ EXPECT_LT(0U, Listener1.EmittedEvents[0].Size)
+ << "We don't know how big the function will be, but it had better"
+ << " contain some bytes.";
+
+ EXPECT_EQ(1U, Listener1.FreedEvents[0].Index);
+ EXPECT_EQ(F2_addr, Listener1.FreedEvents[0].Code);
+
+ // Listener 2.
+ ASSERT_EQ(2U, Listener2.EmittedEvents.size());
+ ASSERT_EQ(1U, Listener2.FreedEvents.size());
+
+ EXPECT_EQ(0U, Listener2.EmittedEvents[0].Index);
+ EXPECT_EQ(F1, Listener2.EmittedEvents[0].F);
+ EXPECT_EQ(F1_addr, Listener2.EmittedEvents[0].Code);
+ EXPECT_LT(0U, Listener2.EmittedEvents[0].Size)
+ << "We don't know how big the function will be, but it had better"
+ << " contain some bytes.";
+
+ EXPECT_EQ(1U, Listener2.EmittedEvents[1].Index);
+ EXPECT_EQ(F2, Listener2.EmittedEvents[1].F);
+ EXPECT_EQ(F2_addr, Listener2.EmittedEvents[1].Code);
+ EXPECT_LT(0U, Listener2.EmittedEvents[1].Size)
+ << "We don't know how big the function will be, but it had better"
+ << " contain some bytes.";
+
+ EXPECT_EQ(2U, Listener2.FreedEvents[0].Index);
+ EXPECT_EQ(F2_addr, Listener2.FreedEvents[0].Code);
+
+ // Listener 3.
+ ASSERT_EQ(1U, Listener3.EmittedEvents.size());
+ ASSERT_EQ(1U, Listener3.FreedEvents.size());
+
+ EXPECT_EQ(0U, Listener3.EmittedEvents[0].Index);
+ EXPECT_EQ(F2, Listener3.EmittedEvents[0].F);
+ EXPECT_EQ(F2_addr, Listener3.EmittedEvents[0].Code);
+ EXPECT_LT(0U, Listener3.EmittedEvents[0].Size)
+ << "We don't know how big the function will be, but it had better"
+ << " contain some bytes.";
+
+ EXPECT_EQ(1U, Listener3.FreedEvents[0].Index);
+ EXPECT_EQ(F2_addr, Listener3.FreedEvents[0].Code);
+
+ F1->eraseFromParent();
+ F2->eraseFromParent();
+}
+
+TEST_F(JITEventListenerTest, MatchesMachineCodeInfo) {
+ RecordingJITEventListener Listener;
+ MachineCodeInfo MCI;
+ Function *F = buildFunction(M);
+
+ EE->RegisterJITEventListener(&Listener);
+ EE->runJITOnFunction(F, &MCI);
+ void *F_addr = EE->getPointerToFunction(F);
+ EE->freeMachineCodeForFunction(F);
+
+ ASSERT_EQ(1U, Listener.EmittedEvents.size());
+ ASSERT_EQ(1U, Listener.FreedEvents.size());
+
+ EXPECT_EQ(0U, Listener.EmittedEvents[0].Index);
+ EXPECT_EQ(F, Listener.EmittedEvents[0].F);
+ EXPECT_EQ(F_addr, Listener.EmittedEvents[0].Code);
+ EXPECT_EQ(MCI.address(), Listener.EmittedEvents[0].Code);
+ EXPECT_EQ(MCI.size(), Listener.EmittedEvents[0].Size);
+
+ EXPECT_EQ(1U, Listener.FreedEvents[0].Index);
+ EXPECT_EQ(F_addr, Listener.FreedEvents[0].Code);
+}
+#endif
+
+class JITEnvironment : public testing::Environment {
+ virtual void SetUp() {
+ // Required to create a JIT.
+ InitializeNativeTarget();
+ }
+};
+testing::Environment* const jit_env =
+ testing::AddGlobalTestEnvironment(new JITEnvironment);
+
+} // anonymous namespace
--- /dev/null
+//===- JITEventListenerTestCommon.h - Helper for JITEventListener tests ------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===-------------------------------------------------------------------------------===//
+
+#ifndef JIT_EVENT_LISTENER_TEST_COMMON_H
+#define JIT_EVENT_LISTENER_TEST_COMMON_H
+
+#include "llvm/CodeGen/MachineCodeInfo.h"
+#include "llvm/Config/config.h"
+#include "llvm/ExecutionEngine/JIT.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
+#include "llvm/IR/DIBuilder.h"
+#include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/TypeBuilder.h"
+#include "llvm/Support/Dwarf.h"
+#include "llvm/Support/TargetSelect.h"
+#include "gtest/gtest.h"
+#include <string>
+#include <utility>
+#include <vector>
+
+typedef std::vector<std::pair<std::string, unsigned int> > SourceLocations;
+typedef std::map<uint64_t, SourceLocations> NativeCodeMap;
+
+class JITEnvironment : public testing::Environment {
+ virtual void SetUp() {
+ // Required to create a JIT.
+ llvm::InitializeNativeTarget();
+ }
+};
+
+inline unsigned int getLine() {
+ return 12;
+}
+
+inline unsigned int getCol() {
+ return 0;
+}
+
+inline const char* getFilename() {
+ return "mock_source_file.cpp";
+}
+
+// Test fixture shared by tests for listener implementations
+template<typename WrapperT>
+class JITEventListenerTestBase : public testing::Test {
+protected:
+ std::unique_ptr<WrapperT> MockWrapper;
+ std::unique_ptr<llvm::JITEventListener> Listener;
+
+public:
+ llvm::Module* M;
+ llvm::MDNode* Scope;
+ llvm::ExecutionEngine* EE;
+ llvm::DIBuilder* DebugBuilder;
+ llvm::IRBuilder<> Builder;
+
+ JITEventListenerTestBase(WrapperT* w)
+ : MockWrapper(w)
+ , M(new llvm::Module("module", llvm::getGlobalContext()))
+ , EE(llvm::EngineBuilder(M)
+ .setEngineKind(llvm::EngineKind::JIT)
+ .setOptLevel(llvm::CodeGenOpt::None)
+ .create())
+ , DebugBuilder(new llvm::DIBuilder(*M))
+ , Builder(llvm::getGlobalContext())
+ {
+ DebugBuilder->createCompileUnit(llvm::dwarf::DW_LANG_C_plus_plus,
+ "JIT",
+ "JIT",
+ "JIT",
+ true,
+ "",
+ 1);
+
+ Scope = DebugBuilder->createFile(getFilename(), ".");
+ }
+
+ llvm::Function *buildFunction(const SourceLocations& DebugLocations) {
+ using namespace llvm;
+
+ LLVMContext& GlobalContext = getGlobalContext();
+
+ SourceLocations::const_iterator CurrentDebugLocation
+ = DebugLocations.begin();
+
+ if (CurrentDebugLocation != DebugLocations.end()) {
+ DebugLoc DebugLocation = DebugLoc::get(getLine(), getCol(),
+ DebugBuilder->createFile(CurrentDebugLocation->first, "."));
+ Builder.SetCurrentDebugLocation(DebugLocation);
+ CurrentDebugLocation++;
+ }
+
+ Function *Result = Function::Create(
+ TypeBuilder<int32_t(int32_t), false>::get(GlobalContext),
+ GlobalValue::ExternalLinkage, "id", M);
+ Value *Arg = Result->arg_begin();
+ BasicBlock *BB = BasicBlock::Create(M->getContext(), "entry", Result);
+ Builder.SetInsertPoint(BB);
+ Value* one = ConstantInt::get(GlobalContext, APInt(32, 1));
+ for(; CurrentDebugLocation != DebugLocations.end();
+ ++CurrentDebugLocation) {
+ Arg = Builder.CreateMul(Arg, Builder.CreateAdd(Arg, one));
+ Builder.SetCurrentDebugLocation(
+ DebugLoc::get(CurrentDebugLocation->second, 0,
+ DebugBuilder->createFile(CurrentDebugLocation->first, ".")));
+ }
+ Builder.CreateRet(Arg);
+ return Result;
+ }
+
+ void TestNoDebugInfo(NativeCodeMap& ReportedDebugFuncs) {
+ SourceLocations DebugLocations;
+ llvm::Function* f = buildFunction(DebugLocations);
+ EXPECT_TRUE(0 != f);
+
+ //Cause JITting and callbacks to our listener
+ EXPECT_TRUE(0 != EE->getPointerToFunction(f));
+ EXPECT_TRUE(1 == ReportedDebugFuncs.size());
+
+ EE->freeMachineCodeForFunction(f);
+ EXPECT_TRUE(ReportedDebugFuncs.size() == 0);
+ }
+
+ void TestSingleLine(NativeCodeMap& ReportedDebugFuncs) {
+ SourceLocations DebugLocations;
+ DebugLocations.push_back(std::make_pair(std::string(getFilename()),
+ getLine()));
+ llvm::Function* f = buildFunction(DebugLocations);
+ EXPECT_TRUE(0 != f);
+
+ EXPECT_TRUE(0 != EE->getPointerToFunction(f));
+ EXPECT_TRUE(1 == ReportedDebugFuncs.size());
+ EXPECT_STREQ(ReportedDebugFuncs.begin()->second.begin()->first.c_str(),
+ getFilename());
+ EXPECT_EQ(ReportedDebugFuncs.begin()->second.begin()->second, getLine());
+
+ EE->freeMachineCodeForFunction(f);
+ EXPECT_TRUE(ReportedDebugFuncs.size() == 0);
+ }
+
+ void TestMultipleLines(NativeCodeMap& ReportedDebugFuncs) {
+ using namespace std;
+
+ SourceLocations DebugLocations;
+ unsigned int c = 5;
+ for(unsigned int i = 0; i < c; ++i) {
+ DebugLocations.push_back(make_pair(string(getFilename()), getLine() + i));
+ }
+
+ llvm::Function* f = buildFunction(DebugLocations);
+ EXPECT_TRUE(0 != f);
+
+ EXPECT_TRUE(0 != EE->getPointerToFunction(f));
+ EXPECT_TRUE(1 == ReportedDebugFuncs.size());
+ SourceLocations& FunctionInfo = ReportedDebugFuncs.begin()->second;
+ EXPECT_EQ(c, FunctionInfo.size());
+
+ int VerifyCount = 0;
+ for(SourceLocations::iterator i = FunctionInfo.begin();
+ i != FunctionInfo.end();
+ ++i) {
+ EXPECT_STREQ(i->first.c_str(), getFilename());
+ EXPECT_EQ(i->second, getLine() + VerifyCount);
+ VerifyCount++;
+ }
+
+ EE->freeMachineCodeForFunction(f);
+ EXPECT_TRUE(ReportedDebugFuncs.size() == 0);
+ }
+
+ void TestMultipleFiles(NativeCodeMap& ReportedDebugFuncs) {
+
+ std::string secondFilename("another_file.cpp");
+
+ SourceLocations DebugLocations;
+ DebugLocations.push_back(std::make_pair(std::string(getFilename()),
+ getLine()));
+ DebugLocations.push_back(std::make_pair(secondFilename, getLine()));
+ llvm::Function* f = buildFunction(DebugLocations);
+ EXPECT_TRUE(0 != f);
+
+ EXPECT_TRUE(0 != EE->getPointerToFunction(f));
+ EXPECT_TRUE(1 == ReportedDebugFuncs.size());
+ SourceLocations& FunctionInfo = ReportedDebugFuncs.begin()->second;
+ EXPECT_TRUE(2 == FunctionInfo.size());
+
+ EXPECT_STREQ(FunctionInfo.at(0).first.c_str(), getFilename());
+ EXPECT_STREQ(FunctionInfo.at(1).first.c_str(), secondFilename.c_str());
+
+ EXPECT_EQ(FunctionInfo.at(0).second, getLine());
+ EXPECT_EQ(FunctionInfo.at(1).second, getLine());
+
+ EE->freeMachineCodeForFunction(f);
+ EXPECT_TRUE(ReportedDebugFuncs.size() == 0);
+ }
+};
+
+#endif //JIT_EVENT_LISTENER_TEST_COMMON_H
--- /dev/null
+//===- JITMemoryManagerTest.cpp - Unit tests for the JIT memory manager ---===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/JITMemoryManager.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/LLVMContext.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+Function *makeFakeFunction() {
+ std::vector<Type*> params;
+ FunctionType *FTy =
+ FunctionType::get(Type::getVoidTy(getGlobalContext()), params, false);
+ return Function::Create(FTy, GlobalValue::ExternalLinkage);
+}
+
+// Allocate three simple functions that fit in the initial slab. This exercises
+// the code in the case that we don't have to allocate more memory to store the
+// function bodies.
+TEST(JITMemoryManagerTest, NoAllocations) {
+ std::unique_ptr<JITMemoryManager> MemMgr(
+ JITMemoryManager::CreateDefaultMemManager());
+ uintptr_t size;
+ std::string Error;
+
+ // Allocate the functions.
+ std::unique_ptr<Function> F1(makeFakeFunction());
+ size = 1024;
+ uint8_t *FunctionBody1 = MemMgr->startFunctionBody(F1.get(), size);
+ memset(FunctionBody1, 0xFF, 1024);
+ MemMgr->endFunctionBody(F1.get(), FunctionBody1, FunctionBody1 + 1024);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+
+ std::unique_ptr<Function> F2(makeFakeFunction());
+ size = 1024;
+ uint8_t *FunctionBody2 = MemMgr->startFunctionBody(F2.get(), size);
+ memset(FunctionBody2, 0xFF, 1024);
+ MemMgr->endFunctionBody(F2.get(), FunctionBody2, FunctionBody2 + 1024);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+
+ std::unique_ptr<Function> F3(makeFakeFunction());
+ size = 1024;
+ uint8_t *FunctionBody3 = MemMgr->startFunctionBody(F3.get(), size);
+ memset(FunctionBody3, 0xFF, 1024);
+ MemMgr->endFunctionBody(F3.get(), FunctionBody3, FunctionBody3 + 1024);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+
+ // Deallocate them out of order, in case that matters.
+ MemMgr->deallocateFunctionBody(FunctionBody2);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+ MemMgr->deallocateFunctionBody(FunctionBody1);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+ MemMgr->deallocateFunctionBody(FunctionBody3);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+}
+
+// Make three large functions that take up most of the space in the slab. Then
+// try allocating three smaller functions that don't require additional slabs.
+TEST(JITMemoryManagerTest, TestCodeAllocation) {
+ std::unique_ptr<JITMemoryManager> MemMgr(
+ JITMemoryManager::CreateDefaultMemManager());
+ uintptr_t size;
+ std::string Error;
+
+ // Big functions are a little less than the largest block size.
+ const uintptr_t smallFuncSize = 1024;
+ const uintptr_t bigFuncSize = (MemMgr->GetDefaultCodeSlabSize() -
+ smallFuncSize * 2);
+
+ // Allocate big functions
+ std::unique_ptr<Function> F1(makeFakeFunction());
+ size = bigFuncSize;
+ uint8_t *FunctionBody1 = MemMgr->startFunctionBody(F1.get(), size);
+ ASSERT_LE(bigFuncSize, size);
+ memset(FunctionBody1, 0xFF, bigFuncSize);
+ MemMgr->endFunctionBody(F1.get(), FunctionBody1, FunctionBody1 + bigFuncSize);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+
+ std::unique_ptr<Function> F2(makeFakeFunction());
+ size = bigFuncSize;
+ uint8_t *FunctionBody2 = MemMgr->startFunctionBody(F2.get(), size);
+ ASSERT_LE(bigFuncSize, size);
+ memset(FunctionBody2, 0xFF, bigFuncSize);
+ MemMgr->endFunctionBody(F2.get(), FunctionBody2, FunctionBody2 + bigFuncSize);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+
+ std::unique_ptr<Function> F3(makeFakeFunction());
+ size = bigFuncSize;
+ uint8_t *FunctionBody3 = MemMgr->startFunctionBody(F3.get(), size);
+ ASSERT_LE(bigFuncSize, size);
+ memset(FunctionBody3, 0xFF, bigFuncSize);
+ MemMgr->endFunctionBody(F3.get(), FunctionBody3, FunctionBody3 + bigFuncSize);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+
+ // Check that each large function took it's own slab.
+ EXPECT_EQ(3U, MemMgr->GetNumCodeSlabs());
+
+ // Allocate small functions
+ std::unique_ptr<Function> F4(makeFakeFunction());
+ size = smallFuncSize;
+ uint8_t *FunctionBody4 = MemMgr->startFunctionBody(F4.get(), size);
+ ASSERT_LE(smallFuncSize, size);
+ memset(FunctionBody4, 0xFF, smallFuncSize);
+ MemMgr->endFunctionBody(F4.get(), FunctionBody4,
+ FunctionBody4 + smallFuncSize);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+
+ std::unique_ptr<Function> F5(makeFakeFunction());
+ size = smallFuncSize;
+ uint8_t *FunctionBody5 = MemMgr->startFunctionBody(F5.get(), size);
+ ASSERT_LE(smallFuncSize, size);
+ memset(FunctionBody5, 0xFF, smallFuncSize);
+ MemMgr->endFunctionBody(F5.get(), FunctionBody5,
+ FunctionBody5 + smallFuncSize);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+
+ std::unique_ptr<Function> F6(makeFakeFunction());
+ size = smallFuncSize;
+ uint8_t *FunctionBody6 = MemMgr->startFunctionBody(F6.get(), size);
+ ASSERT_LE(smallFuncSize, size);
+ memset(FunctionBody6, 0xFF, smallFuncSize);
+ MemMgr->endFunctionBody(F6.get(), FunctionBody6,
+ FunctionBody6 + smallFuncSize);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+
+ // Check that the small functions didn't allocate any new slabs.
+ EXPECT_EQ(3U, MemMgr->GetNumCodeSlabs());
+
+ // Deallocate them out of order, in case that matters.
+ MemMgr->deallocateFunctionBody(FunctionBody2);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+ MemMgr->deallocateFunctionBody(FunctionBody1);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+ MemMgr->deallocateFunctionBody(FunctionBody4);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+ MemMgr->deallocateFunctionBody(FunctionBody3);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+ MemMgr->deallocateFunctionBody(FunctionBody5);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+ MemMgr->deallocateFunctionBody(FunctionBody6);
+ EXPECT_TRUE(MemMgr->CheckInvariants(Error)) << Error;
+}
+
+// Allocate five global ints of varying widths and alignment, and check their
+// alignment and overlap.
+TEST(JITMemoryManagerTest, TestSmallGlobalInts) {
+ std::unique_ptr<JITMemoryManager> MemMgr(
+ JITMemoryManager::CreateDefaultMemManager());
+ uint8_t *a = (uint8_t *)MemMgr->allocateGlobal(8, 0);
+ uint16_t *b = (uint16_t*)MemMgr->allocateGlobal(16, 2);
+ uint32_t *c = (uint32_t*)MemMgr->allocateGlobal(32, 4);
+ uint64_t *d = (uint64_t*)MemMgr->allocateGlobal(64, 8);
+
+ // Check the alignment.
+ EXPECT_EQ(0U, ((uintptr_t)b) & 0x1);
+ EXPECT_EQ(0U, ((uintptr_t)c) & 0x3);
+ EXPECT_EQ(0U, ((uintptr_t)d) & 0x7);
+
+ // Initialize them each one at a time and make sure they don't overlap.
+ *a = 0xff;
+ *b = 0U;
+ *c = 0U;
+ *d = 0U;
+ EXPECT_EQ(0xffU, *a);
+ EXPECT_EQ(0U, *b);
+ EXPECT_EQ(0U, *c);
+ EXPECT_EQ(0U, *d);
+ *a = 0U;
+ *b = 0xffffU;
+ EXPECT_EQ(0U, *a);
+ EXPECT_EQ(0xffffU, *b);
+ EXPECT_EQ(0U, *c);
+ EXPECT_EQ(0U, *d);
+ *b = 0U;
+ *c = 0xffffffffU;
+ EXPECT_EQ(0U, *a);
+ EXPECT_EQ(0U, *b);
+ EXPECT_EQ(0xffffffffU, *c);
+ EXPECT_EQ(0U, *d);
+ *c = 0U;
+ *d = 0xffffffffffffffffULL;
+ EXPECT_EQ(0U, *a);
+ EXPECT_EQ(0U, *b);
+ EXPECT_EQ(0U, *c);
+ EXPECT_EQ(0xffffffffffffffffULL, *d);
+
+ // Make sure we didn't allocate any extra slabs for this tiny amount of data.
+ EXPECT_EQ(1U, MemMgr->GetNumDataSlabs());
+}
+
+// Allocate a small global, a big global, and a third global, and make sure we
+// only use two slabs for that.
+TEST(JITMemoryManagerTest, TestLargeGlobalArray) {
+ std::unique_ptr<JITMemoryManager> MemMgr(
+ JITMemoryManager::CreateDefaultMemManager());
+ size_t Size = 4 * MemMgr->GetDefaultDataSlabSize();
+ uint64_t *a = (uint64_t*)MemMgr->allocateGlobal(64, 8);
+ uint8_t *g = MemMgr->allocateGlobal(Size, 8);
+ uint64_t *b = (uint64_t*)MemMgr->allocateGlobal(64, 8);
+
+ // Check the alignment.
+ EXPECT_EQ(0U, ((uintptr_t)a) & 0x7);
+ EXPECT_EQ(0U, ((uintptr_t)g) & 0x7);
+ EXPECT_EQ(0U, ((uintptr_t)b) & 0x7);
+
+ // Initialize them to make sure we don't segfault and make sure they don't
+ // overlap.
+ memset(a, 0x1, 8);
+ memset(g, 0x2, Size);
+ memset(b, 0x3, 8);
+ EXPECT_EQ(0x0101010101010101ULL, *a);
+ // Just check the edges.
+ EXPECT_EQ(0x02U, g[0]);
+ EXPECT_EQ(0x02U, g[Size - 1]);
+ EXPECT_EQ(0x0303030303030303ULL, *b);
+
+ // Check the number of slabs.
+ EXPECT_EQ(2U, MemMgr->GetNumDataSlabs());
+}
+
+// Allocate lots of medium globals so that we can test moving the bump allocator
+// to a new slab.
+TEST(JITMemoryManagerTest, TestManyGlobals) {
+ std::unique_ptr<JITMemoryManager> MemMgr(
+ JITMemoryManager::CreateDefaultMemManager());
+ size_t SlabSize = MemMgr->GetDefaultDataSlabSize();
+ size_t Size = 128;
+ int Iters = (SlabSize / Size) + 1;
+
+ // We should start with no slabs.
+ EXPECT_EQ(0U, MemMgr->GetNumDataSlabs());
+
+ // After allocating a bunch of globals, we should have two.
+ for (int I = 0; I < Iters; ++I)
+ MemMgr->allocateGlobal(Size, 8);
+ EXPECT_EQ(2U, MemMgr->GetNumDataSlabs());
+
+ // And after much more, we should have three.
+ for (int I = 0; I < Iters; ++I)
+ MemMgr->allocateGlobal(Size, 8);
+ EXPECT_EQ(3U, MemMgr->GetNumDataSlabs());
+}
+
+// Allocate lots of function stubs so that we can test moving the stub bump
+// allocator to a new slab.
+TEST(JITMemoryManagerTest, TestManyStubs) {
+ std::unique_ptr<JITMemoryManager> MemMgr(
+ JITMemoryManager::CreateDefaultMemManager());
+ size_t SlabSize = MemMgr->GetDefaultStubSlabSize();
+ size_t Size = 128;
+ int Iters = (SlabSize / Size) + 1;
+
+ // We should start with no slabs.
+ EXPECT_EQ(0U, MemMgr->GetNumDataSlabs());
+
+ // After allocating a bunch of stubs, we should have two.
+ for (int I = 0; I < Iters; ++I)
+ MemMgr->allocateStub(nullptr, Size, 8);
+ EXPECT_EQ(2U, MemMgr->GetNumStubSlabs());
+
+ // And after much more, we should have three.
+ for (int I = 0; I < Iters; ++I)
+ MemMgr->allocateStub(nullptr, Size, 8);
+ EXPECT_EQ(3U, MemMgr->GetNumStubSlabs());
+}
+
+// Check section allocation and alignment
+TEST(JITMemoryManagerTest, AllocateSection) {
+ std::unique_ptr<JITMemoryManager> MemMgr(
+ JITMemoryManager::CreateDefaultMemManager());
+ uint8_t *code1 = MemMgr->allocateCodeSection(256, 0, 1, StringRef());
+ uint8_t *data1 = MemMgr->allocateDataSection(256, 16, 2, StringRef(), true);
+ uint8_t *code2 = MemMgr->allocateCodeSection(257, 32, 3, StringRef());
+ uint8_t *data2 = MemMgr->allocateDataSection(256, 64, 4, StringRef(), false);
+ uint8_t *code3 = MemMgr->allocateCodeSection(258, 64, 5, StringRef());
+
+ EXPECT_NE((uint8_t*)nullptr, code1);
+ EXPECT_NE((uint8_t*)nullptr, code2);
+ EXPECT_NE((uint8_t*)nullptr, data1);
+ EXPECT_NE((uint8_t*)nullptr, data2);
+
+ // Check alignment
+ EXPECT_EQ((uint64_t)code1 & 0xf, 0u);
+ EXPECT_EQ((uint64_t)code2 & 0x1f, 0u);
+ EXPECT_EQ((uint64_t)code3 & 0x3f, 0u);
+ EXPECT_EQ((uint64_t)data1 & 0xf, 0u);
+ EXPECT_EQ((uint64_t)data2 & 0x3f, 0u);
+}
+
+}
--- /dev/null
+//===- JITTest.cpp - Unit tests for the JIT -------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/JIT.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/Bitcode/ReaderWriter.h"
+#include "llvm/ExecutionEngine/JITMemoryManager.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/TypeBuilder.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetSelect.h"
+#include "gtest/gtest.h"
+#include <vector>
+
+using namespace llvm;
+
+// This variable is intentionally defined differently in the statically-compiled
+// program from the IR input to the JIT to assert that the JIT doesn't use its
+// definition. Note that this variable must be defined even on platforms where
+// JIT tests are disabled as it is referenced from the .def file.
+extern "C" int32_t JITTest_AvailableExternallyGlobal;
+int32_t JITTest_AvailableExternallyGlobal LLVM_ATTRIBUTE_USED = 42;
+
+// This function is intentionally defined differently in the statically-compiled
+// program from the IR input to the JIT to assert that the JIT doesn't use its
+// definition. Note that this function must be defined even on platforms where
+// JIT tests are disabled as it is referenced from the .def file.
+extern "C" int32_t JITTest_AvailableExternallyFunction() LLVM_ATTRIBUTE_USED;
+extern "C" int32_t JITTest_AvailableExternallyFunction() {
+ return 42;
+}
+
+namespace {
+
+// Tests on ARM, PowerPC and SystemZ disabled as we're running the old jit
+#if !defined(__arm__) && !defined(__powerpc__) && !defined(__s390__) \
+ && !defined(__aarch64__)
+
+Function *makeReturnGlobal(std::string Name, GlobalVariable *G, Module *M) {
+ std::vector<Type*> params;
+ FunctionType *FTy = FunctionType::get(G->getType()->getElementType(),
+ params, false);
+ Function *F = Function::Create(FTy, GlobalValue::ExternalLinkage, Name, M);
+ BasicBlock *Entry = BasicBlock::Create(M->getContext(), "entry", F);
+ IRBuilder<> builder(Entry);
+ Value *Load = builder.CreateLoad(G);
+ Type *GTy = G->getType()->getElementType();
+ Value *Add = builder.CreateAdd(Load, ConstantInt::get(GTy, 1LL));
+ builder.CreateStore(Add, G);
+ builder.CreateRet(Add);
+ return F;
+}
+
+std::string DumpFunction(const Function *F) {
+ std::string Result;
+ raw_string_ostream(Result) << "" << *F;
+ return Result;
+}
+
+class RecordingJITMemoryManager : public JITMemoryManager {
+ const std::unique_ptr<JITMemoryManager> Base;
+
+public:
+ RecordingJITMemoryManager()
+ : Base(JITMemoryManager::CreateDefaultMemManager()) {
+ stubsAllocated = 0;
+ }
+ virtual void *getPointerToNamedFunction(const std::string &Name,
+ bool AbortOnFailure = true) {
+ return Base->getPointerToNamedFunction(Name, AbortOnFailure);
+ }
+
+ virtual void setMemoryWritable() { Base->setMemoryWritable(); }
+ virtual void setMemoryExecutable() { Base->setMemoryExecutable(); }
+ virtual void setPoisonMemory(bool poison) { Base->setPoisonMemory(poison); }
+ virtual void AllocateGOT() { Base->AllocateGOT(); }
+ virtual uint8_t *getGOTBase() const { return Base->getGOTBase(); }
+ struct StartFunctionBodyCall {
+ StartFunctionBodyCall(uint8_t *Result, const Function *F,
+ uintptr_t ActualSize, uintptr_t ActualSizeResult)
+ : Result(Result), F(F), F_dump(DumpFunction(F)),
+ ActualSize(ActualSize), ActualSizeResult(ActualSizeResult) {}
+ uint8_t *Result;
+ const Function *F;
+ std::string F_dump;
+ uintptr_t ActualSize;
+ uintptr_t ActualSizeResult;
+ };
+ std::vector<StartFunctionBodyCall> startFunctionBodyCalls;
+ virtual uint8_t *startFunctionBody(const Function *F,
+ uintptr_t &ActualSize) {
+ uintptr_t InitialActualSize = ActualSize;
+ uint8_t *Result = Base->startFunctionBody(F, ActualSize);
+ startFunctionBodyCalls.push_back(
+ StartFunctionBodyCall(Result, F, InitialActualSize, ActualSize));
+ return Result;
+ }
+ int stubsAllocated;
+ uint8_t *allocateStub(const GlobalValue *F, unsigned StubSize,
+ unsigned Alignment) override {
+ stubsAllocated++;
+ return Base->allocateStub(F, StubSize, Alignment);
+ }
+ struct EndFunctionBodyCall {
+ EndFunctionBodyCall(const Function *F, uint8_t *FunctionStart,
+ uint8_t *FunctionEnd)
+ : F(F), F_dump(DumpFunction(F)),
+ FunctionStart(FunctionStart), FunctionEnd(FunctionEnd) {}
+ const Function *F;
+ std::string F_dump;
+ uint8_t *FunctionStart;
+ uint8_t *FunctionEnd;
+ };
+ std::vector<EndFunctionBodyCall> endFunctionBodyCalls;
+ virtual void endFunctionBody(const Function *F, uint8_t *FunctionStart,
+ uint8_t *FunctionEnd) {
+ endFunctionBodyCalls.push_back(
+ EndFunctionBodyCall(F, FunctionStart, FunctionEnd));
+ Base->endFunctionBody(F, FunctionStart, FunctionEnd);
+ }
+ virtual uint8_t *allocateDataSection(
+ uintptr_t Size, unsigned Alignment, unsigned SectionID,
+ StringRef SectionName, bool IsReadOnly) {
+ return Base->allocateDataSection(
+ Size, Alignment, SectionID, SectionName, IsReadOnly);
+ }
+ virtual uint8_t *allocateCodeSection(
+ uintptr_t Size, unsigned Alignment, unsigned SectionID,
+ StringRef SectionName) {
+ return Base->allocateCodeSection(
+ Size, Alignment, SectionID, SectionName);
+ }
+ virtual bool finalizeMemory(std::string *ErrMsg) { return false; }
+ virtual uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) {
+ return Base->allocateSpace(Size, Alignment);
+ }
+ virtual uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) {
+ return Base->allocateGlobal(Size, Alignment);
+ }
+ struct DeallocateFunctionBodyCall {
+ DeallocateFunctionBodyCall(const void *Body) : Body(Body) {}
+ const void *Body;
+ };
+ std::vector<DeallocateFunctionBodyCall> deallocateFunctionBodyCalls;
+ virtual void deallocateFunctionBody(void *Body) {
+ deallocateFunctionBodyCalls.push_back(DeallocateFunctionBodyCall(Body));
+ Base->deallocateFunctionBody(Body);
+ }
+};
+
+bool LoadAssemblyInto(Module *M, const char *assembly) {
+ SMDiagnostic Error;
+ bool success =
+ nullptr != ParseAssemblyString(assembly, M, Error, M->getContext());
+ std::string errMsg;
+ raw_string_ostream os(errMsg);
+ Error.print("", os);
+ EXPECT_TRUE(success) << os.str();
+ return success;
+}
+
+class JITTest : public testing::Test {
+ protected:
+ virtual RecordingJITMemoryManager *createMemoryManager() {
+ return new RecordingJITMemoryManager;
+ }
+
+ virtual void SetUp() {
+ M = new Module("<main>", Context);
+ RJMM = createMemoryManager();
+ RJMM->setPoisonMemory(true);
+ std::string Error;
+ TargetOptions Options;
+ TheJIT.reset(EngineBuilder(M).setEngineKind(EngineKind::JIT)
+ .setJITMemoryManager(RJMM)
+ .setErrorStr(&Error)
+ .setTargetOptions(Options).create());
+ ASSERT_TRUE(TheJIT.get() != nullptr) << Error;
+ }
+
+ void LoadAssembly(const char *assembly) {
+ LoadAssemblyInto(M, assembly);
+ }
+
+ LLVMContext Context;
+ Module *M; // Owned by ExecutionEngine.
+ RecordingJITMemoryManager *RJMM;
+ std::unique_ptr<ExecutionEngine> TheJIT;
+};
+
+// Regression test for a bug. The JIT used to allocate globals inside the same
+// memory block used for the function, and when the function code was freed,
+// the global was left in the same place. This test allocates a function
+// that uses and global, deallocates it, and then makes sure that the global
+// stays alive after that.
+TEST(JIT, GlobalInFunction) {
+ LLVMContext context;
+ Module *M = new Module("<main>", context);
+
+ JITMemoryManager *MemMgr = JITMemoryManager::CreateDefaultMemManager();
+ // Tell the memory manager to poison freed memory so that accessing freed
+ // memory is more easily tested.
+ MemMgr->setPoisonMemory(true);
+ std::string Error;
+ std::unique_ptr<ExecutionEngine> JIT(EngineBuilder(M)
+ .setEngineKind(EngineKind::JIT)
+ .setErrorStr(&Error)
+ .setJITMemoryManager(MemMgr)
+ // The next line enables the fix:
+ .setAllocateGVsWithCode(false)
+ .create());
+ ASSERT_EQ(Error, "");
+
+ // Create a global variable.
+ Type *GTy = Type::getInt32Ty(context);
+ GlobalVariable *G = new GlobalVariable(
+ *M,
+ GTy,
+ false, // Not constant.
+ GlobalValue::InternalLinkage,
+ Constant::getNullValue(GTy),
+ "myglobal");
+
+ // Make a function that points to a global.
+ Function *F1 = makeReturnGlobal("F1", G, M);
+
+ // Get the pointer to the native code to force it to JIT the function and
+ // allocate space for the global.
+ void (*F1Ptr)() =
+ reinterpret_cast<void(*)()>((intptr_t)JIT->getPointerToFunction(F1));
+
+ // Since F1 was codegen'd, a pointer to G should be available.
+ int32_t *GPtr = (int32_t*)JIT->getPointerToGlobalIfAvailable(G);
+ ASSERT_NE((int32_t*)nullptr, GPtr);
+ EXPECT_EQ(0, *GPtr);
+
+ // F1() should increment G.
+ F1Ptr();
+ EXPECT_EQ(1, *GPtr);
+
+ // Make a second function identical to the first, referring to the same
+ // global.
+ Function *F2 = makeReturnGlobal("F2", G, M);
+ void (*F2Ptr)() =
+ reinterpret_cast<void(*)()>((intptr_t)JIT->getPointerToFunction(F2));
+
+ // F2() should increment G.
+ F2Ptr();
+ EXPECT_EQ(2, *GPtr);
+
+ // Deallocate F1.
+ JIT->freeMachineCodeForFunction(F1);
+
+ // F2() should *still* increment G.
+ F2Ptr();
+ EXPECT_EQ(3, *GPtr);
+}
+
+int PlusOne(int arg) {
+ return arg + 1;
+}
+
+TEST_F(JITTest, FarCallToKnownFunction) {
+ // x86-64 can only make direct calls to functions within 32 bits of
+ // the current PC. To call anything farther away, we have to load
+ // the address into a register and call through the register. The
+ // current JIT does this by allocating a stub for any far call.
+ // There was a bug in which the JIT tried to emit a direct call when
+ // the target was already in the JIT's global mappings and lazy
+ // compilation was disabled.
+
+ Function *KnownFunction = Function::Create(
+ TypeBuilder<int(int), false>::get(Context),
+ GlobalValue::ExternalLinkage, "known", M);
+ TheJIT->addGlobalMapping(KnownFunction, (void*)(intptr_t)PlusOne);
+
+ // int test() { return known(7); }
+ Function *TestFunction = Function::Create(
+ TypeBuilder<int(), false>::get(Context),
+ GlobalValue::ExternalLinkage, "test", M);
+ BasicBlock *Entry = BasicBlock::Create(Context, "entry", TestFunction);
+ IRBuilder<> Builder(Entry);
+ Value *result = Builder.CreateCall(
+ KnownFunction,
+ ConstantInt::get(TypeBuilder<int, false>::get(Context), 7));
+ Builder.CreateRet(result);
+
+ TheJIT->DisableLazyCompilation(true);
+ int (*TestFunctionPtr)() = reinterpret_cast<int(*)()>(
+ (intptr_t)TheJIT->getPointerToFunction(TestFunction));
+ // This used to crash in trying to call PlusOne().
+ EXPECT_EQ(8, TestFunctionPtr());
+}
+
+// Test a function C which calls A and B which call each other.
+TEST_F(JITTest, NonLazyCompilationStillNeedsStubs) {
+ TheJIT->DisableLazyCompilation(true);
+
+ FunctionType *Func1Ty =
+ cast<FunctionType>(TypeBuilder<void(void), false>::get(Context));
+ std::vector<Type*> arg_types;
+ arg_types.push_back(Type::getInt1Ty(Context));
+ FunctionType *FuncTy = FunctionType::get(
+ Type::getVoidTy(Context), arg_types, false);
+ Function *Func1 = Function::Create(Func1Ty, Function::ExternalLinkage,
+ "func1", M);
+ Function *Func2 = Function::Create(FuncTy, Function::InternalLinkage,
+ "func2", M);
+ Function *Func3 = Function::Create(FuncTy, Function::InternalLinkage,
+ "func3", M);
+ BasicBlock *Block1 = BasicBlock::Create(Context, "block1", Func1);
+ BasicBlock *Block2 = BasicBlock::Create(Context, "block2", Func2);
+ BasicBlock *True2 = BasicBlock::Create(Context, "cond_true", Func2);
+ BasicBlock *False2 = BasicBlock::Create(Context, "cond_false", Func2);
+ BasicBlock *Block3 = BasicBlock::Create(Context, "block3", Func3);
+ BasicBlock *True3 = BasicBlock::Create(Context, "cond_true", Func3);
+ BasicBlock *False3 = BasicBlock::Create(Context, "cond_false", Func3);
+
+ // Make Func1 call Func2(0) and Func3(0).
+ IRBuilder<> Builder(Block1);
+ Builder.CreateCall(Func2, ConstantInt::getTrue(Context));
+ Builder.CreateCall(Func3, ConstantInt::getTrue(Context));
+ Builder.CreateRetVoid();
+
+ // void Func2(bool b) { if (b) { Func3(false); return; } return; }
+ Builder.SetInsertPoint(Block2);
+ Builder.CreateCondBr(Func2->arg_begin(), True2, False2);
+ Builder.SetInsertPoint(True2);
+ Builder.CreateCall(Func3, ConstantInt::getFalse(Context));
+ Builder.CreateRetVoid();
+ Builder.SetInsertPoint(False2);
+ Builder.CreateRetVoid();
+
+ // void Func3(bool b) { if (b) { Func2(false); return; } return; }
+ Builder.SetInsertPoint(Block3);
+ Builder.CreateCondBr(Func3->arg_begin(), True3, False3);
+ Builder.SetInsertPoint(True3);
+ Builder.CreateCall(Func2, ConstantInt::getFalse(Context));
+ Builder.CreateRetVoid();
+ Builder.SetInsertPoint(False3);
+ Builder.CreateRetVoid();
+
+ // Compile the function to native code
+ void (*F1Ptr)() =
+ reinterpret_cast<void(*)()>((intptr_t)TheJIT->getPointerToFunction(Func1));
+
+ F1Ptr();
+}
+
+// Regression test for PR5162. This used to trigger an AssertingVH inside the
+// JIT's Function to stub mapping.
+TEST_F(JITTest, NonLazyLeaksNoStubs) {
+ TheJIT->DisableLazyCompilation(true);
+
+ // Create two functions with a single basic block each.
+ FunctionType *FuncTy =
+ cast<FunctionType>(TypeBuilder<int(), false>::get(Context));
+ Function *Func1 = Function::Create(FuncTy, Function::ExternalLinkage,
+ "func1", M);
+ Function *Func2 = Function::Create(FuncTy, Function::InternalLinkage,
+ "func2", M);
+ BasicBlock *Block1 = BasicBlock::Create(Context, "block1", Func1);
+ BasicBlock *Block2 = BasicBlock::Create(Context, "block2", Func2);
+
+ // The first function calls the second and returns the result
+ IRBuilder<> Builder(Block1);
+ Value *Result = Builder.CreateCall(Func2);
+ Builder.CreateRet(Result);
+
+ // The second function just returns a constant
+ Builder.SetInsertPoint(Block2);
+ Builder.CreateRet(ConstantInt::get(TypeBuilder<int, false>::get(Context),42));
+
+ // Compile the function to native code
+ (void)TheJIT->getPointerToFunction(Func1);
+
+ // Free the JIT state for the functions
+ TheJIT->freeMachineCodeForFunction(Func1);
+ TheJIT->freeMachineCodeForFunction(Func2);
+
+ // Delete the first function (and show that is has no users)
+ EXPECT_EQ(Func1->getNumUses(), 0u);
+ Func1->eraseFromParent();
+
+ // Delete the second function (and show that it has no users - it had one,
+ // func1 but that's gone now)
+ EXPECT_EQ(Func2->getNumUses(), 0u);
+ Func2->eraseFromParent();
+}
+
+TEST_F(JITTest, ModuleDeletion) {
+ TheJIT->DisableLazyCompilation(false);
+ LoadAssembly("define void @main() { "
+ " call i32 @computeVal() "
+ " ret void "
+ "} "
+ " "
+ "define internal i32 @computeVal() { "
+ " ret i32 0 "
+ "} ");
+ Function *func = M->getFunction("main");
+ TheJIT->getPointerToFunction(func);
+ TheJIT->removeModule(M);
+ delete M;
+
+ SmallPtrSet<const void*, 2> FunctionsDeallocated;
+ for (unsigned i = 0, e = RJMM->deallocateFunctionBodyCalls.size();
+ i != e; ++i) {
+ FunctionsDeallocated.insert(RJMM->deallocateFunctionBodyCalls[i].Body);
+ }
+ for (unsigned i = 0, e = RJMM->startFunctionBodyCalls.size(); i != e; ++i) {
+ EXPECT_TRUE(FunctionsDeallocated.count(
+ RJMM->startFunctionBodyCalls[i].Result))
+ << "Function leaked: \n" << RJMM->startFunctionBodyCalls[i].F_dump;
+ }
+ EXPECT_EQ(RJMM->startFunctionBodyCalls.size(),
+ RJMM->deallocateFunctionBodyCalls.size());
+}
+
+// ARM, MIPS and PPC still emit stubs for calls since the target may be
+// too far away to call directly. This #if can probably be removed when
+// http://llvm.org/PR5201 is fixed.
+#if !defined(__arm__) && !defined(__mips__) && \
+ !defined(__powerpc__) && !defined(__ppc__) && !defined(__aarch64__)
+typedef int (*FooPtr) ();
+
+TEST_F(JITTest, NoStubs) {
+ LoadAssembly("define void @bar() {"
+ "entry: "
+ "ret void"
+ "}"
+ " "
+ "define i32 @foo() {"
+ "entry:"
+ "call void @bar()"
+ "ret i32 undef"
+ "}"
+ " "
+ "define i32 @main() {"
+ "entry:"
+ "%0 = call i32 @foo()"
+ "call void @bar()"
+ "ret i32 undef"
+ "}");
+ Function *foo = M->getFunction("foo");
+ uintptr_t tmp = (uintptr_t)(TheJIT->getPointerToFunction(foo));
+ FooPtr ptr = (FooPtr)(tmp);
+
+ (ptr)();
+
+ // We should now allocate no more stubs, we have the code to foo
+ // and the existing stub for bar.
+ int stubsBefore = RJMM->stubsAllocated;
+ Function *func = M->getFunction("main");
+ TheJIT->getPointerToFunction(func);
+
+ Function *bar = M->getFunction("bar");
+ TheJIT->getPointerToFunction(bar);
+
+ ASSERT_EQ(stubsBefore, RJMM->stubsAllocated);
+}
+#endif // !ARM && !PPC
+
+TEST_F(JITTest, FunctionPointersOutliveTheirCreator) {
+ TheJIT->DisableLazyCompilation(true);
+ LoadAssembly("define i8()* @get_foo_addr() { "
+ " ret i8()* @foo "
+ "} "
+ " "
+ "define i8 @foo() { "
+ " ret i8 42 "
+ "} ");
+ Function *F_get_foo_addr = M->getFunction("get_foo_addr");
+
+ typedef char(*fooT)();
+ fooT (*get_foo_addr)() = reinterpret_cast<fooT(*)()>(
+ (intptr_t)TheJIT->getPointerToFunction(F_get_foo_addr));
+ fooT foo_addr = get_foo_addr();
+
+ // Now free get_foo_addr. This should not free the machine code for foo or
+ // any call stub returned as foo's canonical address.
+ TheJIT->freeMachineCodeForFunction(F_get_foo_addr);
+
+ // Check by calling the reported address of foo.
+ EXPECT_EQ(42, foo_addr());
+
+ // The reported address should also be the same as the result of a subsequent
+ // getPointerToFunction(foo).
+#if 0
+ // Fails until PR5126 is fixed:
+ Function *F_foo = M->getFunction("foo");
+ fooT foo = reinterpret_cast<fooT>(
+ (intptr_t)TheJIT->getPointerToFunction(F_foo));
+ EXPECT_EQ((intptr_t)foo, (intptr_t)foo_addr);
+#endif
+}
+
+// ARM does not have an implementation of replaceMachineCodeForFunction(),
+// so recompileAndRelinkFunction doesn't work.
+#if !defined(__arm__) && !defined(__aarch64__)
+TEST_F(JITTest, FunctionIsRecompiledAndRelinked) {
+ Function *F = Function::Create(TypeBuilder<int(void), false>::get(Context),
+ GlobalValue::ExternalLinkage, "test", M);
+ BasicBlock *Entry = BasicBlock::Create(Context, "entry", F);
+ IRBuilder<> Builder(Entry);
+ Value *Val = ConstantInt::get(TypeBuilder<int, false>::get(Context), 1);
+ Builder.CreateRet(Val);
+
+ TheJIT->DisableLazyCompilation(true);
+ // Compile the function once, and make sure it works.
+ int (*OrigFPtr)() = reinterpret_cast<int(*)()>(
+ (intptr_t)TheJIT->recompileAndRelinkFunction(F));
+ EXPECT_EQ(1, OrigFPtr());
+
+ // Now change the function to return a different value.
+ Entry->eraseFromParent();
+ BasicBlock *NewEntry = BasicBlock::Create(Context, "new_entry", F);
+ Builder.SetInsertPoint(NewEntry);
+ Val = ConstantInt::get(TypeBuilder<int, false>::get(Context), 2);
+ Builder.CreateRet(Val);
+ // Recompile it, which should produce a new function pointer _and_ update the
+ // old one.
+ int (*NewFPtr)() = reinterpret_cast<int(*)()>(
+ (intptr_t)TheJIT->recompileAndRelinkFunction(F));
+
+ EXPECT_EQ(2, NewFPtr())
+ << "The new pointer should call the new version of the function";
+ EXPECT_EQ(2, OrigFPtr())
+ << "The old pointer's target should now jump to the new version";
+}
+#endif // !defined(__arm__)
+
+TEST_F(JITTest, AvailableExternallyGlobalIsntEmitted) {
+ TheJIT->DisableLazyCompilation(true);
+ LoadAssembly("@JITTest_AvailableExternallyGlobal = "
+ " available_externally global i32 7 "
+ " "
+ "define i32 @loader() { "
+ " %result = load i32* @JITTest_AvailableExternallyGlobal "
+ " ret i32 %result "
+ "} ");
+ Function *loaderIR = M->getFunction("loader");
+
+ int32_t (*loader)() = reinterpret_cast<int32_t(*)()>(
+ (intptr_t)TheJIT->getPointerToFunction(loaderIR));
+ EXPECT_EQ(42, loader()) << "func should return 42 from the external global,"
+ << " not 7 from the IR version.";
+}
+
+TEST_F(JITTest, AvailableExternallyFunctionIsntCompiled) {
+ TheJIT->DisableLazyCompilation(true);
+ LoadAssembly("define available_externally i32 "
+ " @JITTest_AvailableExternallyFunction() { "
+ " ret i32 7 "
+ "} "
+ " "
+ "define i32 @func() { "
+ " %result = tail call i32 "
+ " @JITTest_AvailableExternallyFunction() "
+ " ret i32 %result "
+ "} ");
+ Function *funcIR = M->getFunction("func");
+
+ int32_t (*func)() = reinterpret_cast<int32_t(*)()>(
+ (intptr_t)TheJIT->getPointerToFunction(funcIR));
+ EXPECT_EQ(42, func()) << "func should return 42 from the static version,"
+ << " not 7 from the IR version.";
+}
+
+TEST_F(JITTest, EscapedLazyStubStillCallable) {
+ TheJIT->DisableLazyCompilation(false);
+ LoadAssembly("define internal i32 @stubbed() { "
+ " ret i32 42 "
+ "} "
+ " "
+ "define i32()* @get_stub() { "
+ " ret i32()* @stubbed "
+ "} ");
+ typedef int32_t(*StubTy)();
+
+ // Call get_stub() to get the address of @stubbed without actually JITting it.
+ Function *get_stubIR = M->getFunction("get_stub");
+ StubTy (*get_stub)() = reinterpret_cast<StubTy(*)()>(
+ (intptr_t)TheJIT->getPointerToFunction(get_stubIR));
+ StubTy stubbed = get_stub();
+ // Now get_stubIR is the only reference to stubbed's stub.
+ get_stubIR->eraseFromParent();
+ // Now there are no references inside the JIT, but we've got a pointer outside
+ // it. The stub should be callable and return the right value.
+ EXPECT_EQ(42, stubbed());
+}
+
+// Converts the LLVM assembly to bitcode and returns it in a std::string. An
+// empty string indicates an error.
+std::string AssembleToBitcode(LLVMContext &Context, const char *Assembly) {
+ Module TempModule("TempModule", Context);
+ if (!LoadAssemblyInto(&TempModule, Assembly)) {
+ return "";
+ }
+
+ std::string Result;
+ raw_string_ostream OS(Result);
+ WriteBitcodeToFile(&TempModule, OS);
+ OS.flush();
+ return Result;
+}
+
+// Returns a newly-created ExecutionEngine that reads the bitcode in 'Bitcode'
+// lazily. The associated Module (owned by the ExecutionEngine) is returned in
+// M. Both will be NULL on an error. Bitcode must live at least as long as the
+// ExecutionEngine.
+ExecutionEngine *getJITFromBitcode(
+ LLVMContext &Context, const std::string &Bitcode, Module *&M) {
+ // c_str() is null-terminated like MemoryBuffer::getMemBuffer requires.
+ MemoryBuffer *BitcodeBuffer =
+ MemoryBuffer::getMemBuffer(Bitcode, "Bitcode for test");
+ ErrorOr<Module*> ModuleOrErr = getLazyBitcodeModule(BitcodeBuffer, Context);
+ if (std::error_code EC = ModuleOrErr.getError()) {
+ ADD_FAILURE() << EC.message();
+ delete BitcodeBuffer;
+ return nullptr;
+ }
+ M = ModuleOrErr.get();
+ std::string errMsg;
+ ExecutionEngine *TheJIT = EngineBuilder(M)
+ .setEngineKind(EngineKind::JIT)
+ .setErrorStr(&errMsg)
+ .create();
+ if (TheJIT == nullptr) {
+ ADD_FAILURE() << errMsg;
+ delete M;
+ M = nullptr;
+ return nullptr;
+ }
+ return TheJIT;
+}
+
+TEST(LazyLoadedJITTest, MaterializableAvailableExternallyFunctionIsntCompiled) {
+ LLVMContext Context;
+ const std::string Bitcode =
+ AssembleToBitcode(Context,
+ "define available_externally i32 "
+ " @JITTest_AvailableExternallyFunction() { "
+ " ret i32 7 "
+ "} "
+ " "
+ "define i32 @func() { "
+ " %result = tail call i32 "
+ " @JITTest_AvailableExternallyFunction() "
+ " ret i32 %result "
+ "} ");
+ ASSERT_FALSE(Bitcode.empty()) << "Assembling failed";
+ Module *M;
+ std::unique_ptr<ExecutionEngine> TheJIT(
+ getJITFromBitcode(Context, Bitcode, M));
+ ASSERT_TRUE(TheJIT.get()) << "Failed to create JIT.";
+ TheJIT->DisableLazyCompilation(true);
+
+ Function *funcIR = M->getFunction("func");
+ Function *availableFunctionIR =
+ M->getFunction("JITTest_AvailableExternallyFunction");
+
+ // Double-check that the available_externally function is still unmaterialized
+ // when getPointerToFunction needs to find out if it's available_externally.
+ EXPECT_TRUE(availableFunctionIR->isMaterializable());
+
+ int32_t (*func)() = reinterpret_cast<int32_t(*)()>(
+ (intptr_t)TheJIT->getPointerToFunction(funcIR));
+ EXPECT_EQ(42, func()) << "func should return 42 from the static version,"
+ << " not 7 from the IR version.";
+}
+
+TEST(LazyLoadedJITTest, EagerCompiledRecursionThroughGhost) {
+ LLVMContext Context;
+ const std::string Bitcode =
+ AssembleToBitcode(Context,
+ "define i32 @recur1(i32 %a) { "
+ " %zero = icmp eq i32 %a, 0 "
+ " br i1 %zero, label %done, label %notdone "
+ "done: "
+ " ret i32 3 "
+ "notdone: "
+ " %am1 = sub i32 %a, 1 "
+ " %result = call i32 @recur2(i32 %am1) "
+ " ret i32 %result "
+ "} "
+ " "
+ "define i32 @recur2(i32 %b) { "
+ " %result = call i32 @recur1(i32 %b) "
+ " ret i32 %result "
+ "} ");
+ ASSERT_FALSE(Bitcode.empty()) << "Assembling failed";
+ Module *M;
+ std::unique_ptr<ExecutionEngine> TheJIT(
+ getJITFromBitcode(Context, Bitcode, M));
+ ASSERT_TRUE(TheJIT.get()) << "Failed to create JIT.";
+ TheJIT->DisableLazyCompilation(true);
+
+ Function *recur1IR = M->getFunction("recur1");
+ Function *recur2IR = M->getFunction("recur2");
+ EXPECT_TRUE(recur1IR->isMaterializable());
+ EXPECT_TRUE(recur2IR->isMaterializable());
+
+ int32_t (*recur1)(int32_t) = reinterpret_cast<int32_t(*)(int32_t)>(
+ (intptr_t)TheJIT->getPointerToFunction(recur1IR));
+ EXPECT_EQ(3, recur1(4));
+}
+#endif // !defined(__arm__) && !defined(__powerpc__) && !defined(__s390__)
+
+}
--- /dev/null
+EXPORTS
+getPointerToNamedFunction
+JITTest_AvailableExternallyFunction
+JITTest_AvailableExternallyGlobal
--- /dev/null
+##===- unittests/ExecutionEngine/JIT/Makefile --------------*- Makefile -*-===##
+#
+# The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../../..
+TESTNAME = JIT
+LINK_COMPONENTS := asmparser bitreader bitwriter jit native
+
+# The JIT tests need to dlopen things.
+NO_DEAD_STRIP := 1
+
+include $(LEVEL)/Makefile.config
+
+SOURCES := JITEventListenerTest.cpp JITMemoryManagerTest.cpp JITTest.cpp MultiJITTest.cpp
+
+
+ifeq ($(USE_INTEL_JITEVENTS), 1)
+ # Build the Intel JIT Events interface tests
+ SOURCES += IntelJITEventListenerTest.cpp
+
+ # Add the Intel JIT Events include directory
+ CPPFLAGS += -I$(INTEL_JITEVENTS_INCDIR)
+
+ # Link against the LLVM Intel JIT Evens interface library
+ LINK_COMPONENTS += debuginfo inteljitevents object
+endif
+
+ifeq ($(USE_OPROFILE), 1)
+ # Build the OProfile JIT interface tests
+ SOURCES += OProfileJITEventListenerTest.cpp
+
+ # Link against the LLVM oprofile interface library
+ LINK_COMPONENTS += oprofilejit
+endif
+
+EXPORTED_SYMBOL_FILE = $(PROJ_OBJ_DIR)/JITTests.exports
+
+include $(LLVM_SRC_ROOT)/unittests/Makefile.unittest
+
+# Permit these tests to use the JIT's symbolic lookup.
+LD.Flags += $(RDYNAMIC)
+
+# Symbol exports are necessary (at least for now) when building with LTO.
+$(LLVMUnitTestExe): $(NativeExportsFile)
+$(PROJ_OBJ_DIR)/JITTests.exports: $(PROJ_SRC_DIR)/JITTests.def $(PROJ_OBJ_DIR)/.dir
+ tail -n +2 $< > $@
+
--- /dev/null
+//===- MultiJITTest.cpp - Unit tests for instantiating multiple JITs ------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/JIT.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/ExecutionEngine/GenericValue.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/SourceMgr.h"
+#include "gtest/gtest.h"
+#include <vector>
+
+using namespace llvm;
+
+namespace {
+
+// ARM, PowerPC and SystemZ tests disabled pending fix for PR10783.
+#if !defined(__arm__) && !defined(__powerpc__) && !defined(__s390__) \
+ && !defined(__aarch64__)
+
+bool LoadAssemblyInto(Module *M, const char *assembly) {
+ SMDiagnostic Error;
+ bool success =
+ nullptr != ParseAssemblyString(assembly, M, Error, M->getContext());
+ std::string errMsg;
+ raw_string_ostream os(errMsg);
+ Error.print("", os);
+ EXPECT_TRUE(success) << os.str();
+ return success;
+}
+
+void createModule1(LLVMContext &Context1, Module *&M1, Function *&FooF1) {
+ M1 = new Module("test1", Context1);
+ LoadAssemblyInto(M1,
+ "define i32 @add1(i32 %ArgX1) { "
+ "entry: "
+ " %addresult = add i32 1, %ArgX1 "
+ " ret i32 %addresult "
+ "} "
+ " "
+ "define i32 @foo1() { "
+ "entry: "
+ " %add1 = call i32 @add1(i32 10) "
+ " ret i32 %add1 "
+ "} ");
+ FooF1 = M1->getFunction("foo1");
+}
+
+void createModule2(LLVMContext &Context2, Module *&M2, Function *&FooF2) {
+ M2 = new Module("test2", Context2);
+ LoadAssemblyInto(M2,
+ "define i32 @add2(i32 %ArgX2) { "
+ "entry: "
+ " %addresult = add i32 2, %ArgX2 "
+ " ret i32 %addresult "
+ "} "
+ " "
+ "define i32 @foo2() { "
+ "entry: "
+ " %add2 = call i32 @add2(i32 10) "
+ " ret i32 %add2 "
+ "} ");
+ FooF2 = M2->getFunction("foo2");
+}
+
+TEST(MultiJitTest, EagerMode) {
+ LLVMContext Context1;
+ Module *M1 = nullptr;
+ Function *FooF1 = nullptr;
+ createModule1(Context1, M1, FooF1);
+
+ LLVMContext Context2;
+ Module *M2 = nullptr;
+ Function *FooF2 = nullptr;
+ createModule2(Context2, M2, FooF2);
+
+ // Now we create the JIT in eager mode
+ std::unique_ptr<ExecutionEngine> EE1(EngineBuilder(M1).create());
+ EE1->DisableLazyCompilation(true);
+ std::unique_ptr<ExecutionEngine> EE2(EngineBuilder(M2).create());
+ EE2->DisableLazyCompilation(true);
+
+ // Call the `foo' function with no arguments:
+ std::vector<GenericValue> noargs;
+ GenericValue gv1 = EE1->runFunction(FooF1, noargs);
+ GenericValue gv2 = EE2->runFunction(FooF2, noargs);
+
+ // Import result of execution:
+ EXPECT_EQ(gv1.IntVal, 11);
+ EXPECT_EQ(gv2.IntVal, 12);
+
+ EE1->freeMachineCodeForFunction(FooF1);
+ EE2->freeMachineCodeForFunction(FooF2);
+}
+
+TEST(MultiJitTest, LazyMode) {
+ LLVMContext Context1;
+ Module *M1 = nullptr;
+ Function *FooF1 = nullptr;
+ createModule1(Context1, M1, FooF1);
+
+ LLVMContext Context2;
+ Module *M2 = nullptr;
+ Function *FooF2 = nullptr;
+ createModule2(Context2, M2, FooF2);
+
+ // Now we create the JIT in lazy mode
+ std::unique_ptr<ExecutionEngine> EE1(EngineBuilder(M1).create());
+ EE1->DisableLazyCompilation(false);
+ std::unique_ptr<ExecutionEngine> EE2(EngineBuilder(M2).create());
+ EE2->DisableLazyCompilation(false);
+
+ // Call the `foo' function with no arguments:
+ std::vector<GenericValue> noargs;
+ GenericValue gv1 = EE1->runFunction(FooF1, noargs);
+ GenericValue gv2 = EE2->runFunction(FooF2, noargs);
+
+ // Import result of execution:
+ EXPECT_EQ(gv1.IntVal, 11);
+ EXPECT_EQ(gv2.IntVal, 12);
+
+ EE1->freeMachineCodeForFunction(FooF1);
+ EE2->freeMachineCodeForFunction(FooF2);
+}
+
+extern "C" {
+ extern void *getPointerToNamedFunction(const char *Name);
+}
+
+TEST(MultiJitTest, JitPool) {
+ LLVMContext Context1;
+ Module *M1 = nullptr;
+ Function *FooF1 = nullptr;
+ createModule1(Context1, M1, FooF1);
+
+ LLVMContext Context2;
+ Module *M2 = nullptr;
+ Function *FooF2 = nullptr;
+ createModule2(Context2, M2, FooF2);
+
+ // Now we create two JITs
+ std::unique_ptr<ExecutionEngine> EE1(EngineBuilder(M1).create());
+ std::unique_ptr<ExecutionEngine> EE2(EngineBuilder(M2).create());
+
+ Function *F1 = EE1->FindFunctionNamed("foo1");
+ void *foo1 = EE1->getPointerToFunction(F1);
+
+ Function *F2 = EE2->FindFunctionNamed("foo2");
+ void *foo2 = EE2->getPointerToFunction(F2);
+
+ // Function in M1
+ EXPECT_EQ(getPointerToNamedFunction("foo1"), foo1);
+
+ // Function in M2
+ EXPECT_EQ(getPointerToNamedFunction("foo2"), foo2);
+
+ // Symbol search
+ intptr_t
+ sa = (intptr_t)getPointerToNamedFunction("getPointerToNamedFunction");
+ EXPECT_TRUE(sa != 0);
+ intptr_t fa = (intptr_t)&getPointerToNamedFunction;
+ EXPECT_TRUE(fa != 0);
+#ifdef __i386__
+ // getPointerToNamedFunction might be indirect jump on Win32 --enable-shared.
+ // FF 25 <disp32>: jmp *(pointer to IAT)
+ if (sa != fa && memcmp((char *)fa, "\xFF\x25", 2) == 0) {
+ fa = *(intptr_t *)(fa + 2); // Address to IAT
+ EXPECT_TRUE(fa != 0);
+ fa = *(intptr_t *)fa; // Bound value of IAT
+ }
+#elif defined(__x86_64__)
+ // getPointerToNamedFunction might be indirect jump
+ // on Win32 x64 --enable-shared.
+ // FF 25 <pcrel32>: jmp *(RIP + pointer to IAT)
+ if (sa != fa && memcmp((char *)fa, "\xFF\x25", 2) == 0) {
+ fa += *(int32_t *)(fa + 2) + 6; // Address to IAT(RIP)
+ fa = *(intptr_t *)fa; // Bound value of IAT
+ }
+#endif
+ EXPECT_TRUE(sa == fa);
+}
+#endif // !defined(__arm__) && !defined(__powerpc__) && !defined(__s390__)
+
+} // anonymous namespace
--- /dev/null
+//===- OProfileJITEventListenerTest.cpp - Unit tests for OProfileJITEventsListener --------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--------------------------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/OProfileWrapper.h"
+#include "JITEventListenerTestCommon.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
+#include <list>
+#include <map>
+
+using namespace llvm;
+
+namespace {
+
+struct OprofileNativeFunction {
+ const char* Name;
+ uint64_t Addr;
+ const void* CodePtr;
+ unsigned int CodeSize;
+
+ OprofileNativeFunction(const char* name,
+ uint64_t addr,
+ const void* code,
+ unsigned int size)
+ : Name(name)
+ , Addr(addr)
+ , CodePtr(code)
+ , CodeSize(size) {
+ }
+};
+
+typedef std::list<OprofileNativeFunction> NativeFunctionList;
+typedef std::list<debug_line_info> NativeDebugList;
+NativeFunctionList NativeFunctions;
+
+NativeCodeMap ReportedDebugFuncs;
+
+} // namespace
+
+/// Mock implementaion of opagent library
+namespace test_opagent {
+
+op_agent_t globalAgent = reinterpret_cast<op_agent_t>(42);
+
+op_agent_t open_agent()
+{
+ // return non-null op_agent_t
+ return globalAgent;
+}
+
+int close_agent(op_agent_t agent)
+{
+ EXPECT_EQ(globalAgent, agent);
+ return 0;
+}
+
+int write_native_code(op_agent_t agent,
+ const char* name,
+ uint64_t addr,
+ void const* code,
+ unsigned int size)
+{
+ EXPECT_EQ(globalAgent, agent);
+ OprofileNativeFunction func(name, addr, code, size);
+ NativeFunctions.push_back(func);
+
+ // Verify no other registration has take place for the same address
+ EXPECT_TRUE(ReportedDebugFuncs.find(addr) == ReportedDebugFuncs.end());
+
+ ReportedDebugFuncs[addr];
+ return 0;
+}
+
+int write_debug_line_info(op_agent_t agent,
+ void const* code,
+ size_t num_entries,
+ struct debug_line_info const* info)
+{
+ EXPECT_EQ(globalAgent, agent);
+
+ //verify code has been loaded first
+ uint64_t addr = reinterpret_cast<uint64_t>(code);
+ NativeCodeMap::iterator i = ReportedDebugFuncs.find(addr);
+ EXPECT_TRUE(i != ReportedDebugFuncs.end());
+
+ NativeDebugList NativeInfo(info, info + num_entries);
+
+ SourceLocations locs;
+ for(NativeDebugList::iterator i = NativeInfo.begin();
+ i != NativeInfo.end();
+ ++i) {
+ locs.push_back(std::make_pair(std::string(i->filename), i->lineno));
+ }
+ ReportedDebugFuncs[addr] = locs;
+
+ return 0;
+}
+
+int unload_native_code(op_agent_t agent, uint64_t addr) {
+ EXPECT_EQ(globalAgent, agent);
+
+ //verify that something for the given JIT addr has been loaded first
+ NativeCodeMap::iterator i = ReportedDebugFuncs.find(addr);
+ EXPECT_TRUE(i != ReportedDebugFuncs.end());
+ ReportedDebugFuncs.erase(i);
+ return 0;
+}
+
+int version() {
+ return 1;
+}
+
+bool is_oprofile_running() {
+ return true;
+}
+
+} //namespace test_opagent
+
+class OProfileJITEventListenerTest
+: public JITEventListenerTestBase<OProfileWrapper>
+{
+public:
+ OProfileJITEventListenerTest()
+ : JITEventListenerTestBase<OProfileWrapper>(
+ new OProfileWrapper(test_opagent::open_agent,
+ test_opagent::close_agent,
+ test_opagent::write_native_code,
+ test_opagent::write_debug_line_info,
+ test_opagent::unload_native_code,
+ test_opagent::version,
+ test_opagent::version,
+ test_opagent::is_oprofile_running))
+ {
+ EXPECT_TRUE(0 != MockWrapper);
+
+ Listener.reset(JITEventListener::createOProfileJITEventListener(
+ MockWrapper.get()));
+ EXPECT_TRUE(0 != Listener);
+ EE->RegisterJITEventListener(Listener.get());
+ }
+};
+
+TEST_F(OProfileJITEventListenerTest, NoDebugInfo) {
+ TestNoDebugInfo(ReportedDebugFuncs);
+}
+
+TEST_F(OProfileJITEventListenerTest, SingleLine) {
+ TestSingleLine(ReportedDebugFuncs);
+}
+
+TEST_F(OProfileJITEventListenerTest, MultipleLines) {
+ TestMultipleLines(ReportedDebugFuncs);
+}
+
+TEST_F(OProfileJITEventListenerTest, MultipleFiles) {
+ TestMultipleFiles(ReportedDebugFuncs);
+}
+
+testing::Environment* const jit_env =
+ testing::AddGlobalTestEnvironment(new JITEnvironment);
Core
ExecutionEngine
IPO
+ JIT
MC
MCJIT
ScalarOpts
//===----------------------------------------------------------------------===//
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "gtest/gtest.h"
using namespace llvm;
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringSet.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/ExecutionEngine/MCJIT.h"
#include "llvm/ExecutionEngine/ObjectCache.h"
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
EngineBuilder EB(M);
std::string Error;
TheJIT.reset(EB.setEngineKind(EngineKind::JIT)
+ .setUseMCJIT(true) /* can this be folded into the EngineKind enum? */
.setMCJITMemoryManager(MM)
.setErrorStr(&Error)
.setOptLevel(CodeGenOpt::None)
+ .setAllocateGVsWithCode(false) /*does this do anything?*/
.setCodeModel(CodeModel::JITDefault)
.setRelocationModel(Reloc::Default)
.setMArch(MArch)
LEVEL = ../../..
TESTNAME = MCJIT
-LINK_COMPONENTS := core ipo mcjit native support
+LINK_COMPONENTS := core ipo jit mcjit native support
include $(LEVEL)/Makefile.config
include $(LLVM_SRC_ROOT)/unittests/Makefile.unittest
include $(LEVEL)/Makefile.config
ifeq ($(TARGET_HAS_JIT),1)
- PARALLEL_DIRS = MCJIT
+ PARALLEL_DIRS = JIT MCJIT
endif
include $(LLVM_SRC_ROOT)/unittests/Makefile.unittest
#include <vector>
using namespace llvm;
+// FIXME: Somewhat hackish to use a command line option for this. There should
+// be a CodeEmitter class in the Target.td that controls this sort of thing
+// instead.
+static cl::opt<bool>
+MCEmitter("mc-emitter",
+ cl::desc("Generate CodeEmitter for use with the MC library."),
+ cl::init(false));
+
namespace {
class CodeEmitterGen {
if (SO.second == 0) {
Case += " // op: " + VarName + "\n" +
" op = " + EncoderMethodName + "(MI, " + utostr(OpIdx);
- Case += ", Fixups, STI";
+ if (MCEmitter)
+ Case += ", Fixups, STI";
Case += ");\n";
}
} else {
Case += " // op: " + VarName + "\n" +
" op = getMachineOpValue(MI, MI.getOperand(" + utostr(OpIdx) + ")";
- Case += ", Fixups, STI";
+ if (MCEmitter)
+ Case += ", Fixups, STI";
Case += ");\n";
}
std::string PostEmitter = R->getValueAsString("PostEncoderMethod");
if (!PostEmitter.empty()) {
Case += " Value = " + PostEmitter + "(MI, Value";
- Case += ", STI";
+ if (MCEmitter)
+ Case += ", STI";
Case += ");\n";
}
// Emit function declaration
o << "uint64_t " << Target.getName();
- o << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n"
- << " SmallVectorImpl<MCFixup> &Fixups,\n"
- << " const MCSubtargetInfo &STI) const {\n";
+ if (MCEmitter)
+ o << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n"
+ << " SmallVectorImpl<MCFixup> &Fixups,\n"
+ << " const MCSubtargetInfo &STI) const {\n";
+ else
+ o << "CodeEmitter::getBinaryCodeForInstr(const MachineInstr &MI) const {\n";
// Emit instruction base values
o << " static const uint64_t InstBits[] = {\n";
# If we have a native target with a JIT, use that for the engine. Otherwise,
# use the interpreter.
if native_target and native_target.enabled and native_target.has_jit:
- engine_group.required_libraries.append('MCJIT')
+ engine_group.required_libraries.append('JIT')
engine_group.required_libraries.append(native_group.name)
else:
engine_group.required_libraries.append('Interpreter')
projects / oota-llvm.git / commitdiff