{

/* put your panic function or similar in here */

fprintf(stderr, "fatal error: stack corruption detected\n");

gc_debug_critical_error();

abort(); // end with abort, since the compiler destroyed the stack upon entry to this function, there's no going back now

{

#if JL_LLVM_VERSION >= 50000

f->addAttribute(AttributeList::ReturnIndex, Kind);

#else

f->addAttribute(AttributeSet::ReturnIndex, Kind);

#endif

Value *V; // may be of type T* or T, or set to NULL if ghost (or if the value has not been initialized yet, for a variable definition)

// For unions, we may need to keep a reference to the boxed part individually.

// If this is non-NULL, then, at runtime, we satisfy the invariant that (for the corresponding

// runtime values) if `(TIndex | 0x80) != 0`, then `Vboxed == V` (by value).

// For convenience, we also set this value of isboxed values, in which case

// it is equal (at compile time) to V.

Value *Vboxed;

Value *TIndex; // if `V` is an unboxed (tagged) Union described by `typ`, this gives the DataType index (1-based, small int) as an i8

jl_value_t *constant; // constant value (rooted in linfo.def.roots)

jl_value_t *typ; // the original type of V, never NULL

bool isboxed; // whether this value is a jl_value_t* allocated on the heap with the right type tag

bool isghost; // whether this value is "ghost"

bool isimmutable; // V points to something that is definitely immutable (e.g. single-assignment, but including memory)

MDNode *tbaa; // The related tbaa node. Non-NULL iff this holds an address.

bool ispointer() const

{

// whether this value is compatible with `data_pointer`

return tbaa != nullptr;

}

jl_cgval_t(Value *V, Value *gcroot, bool isboxed, jl_value_t *typ, Value *tindex) : // general constructor (with pointer type auto-detect)

V(V), // V is allowed to be NULL in a jl_varinfo_t context, but not during codegen contexts

Vboxed(isboxed ? V : nullptr),

TIndex(tindex),

constant(NULL),

typ(typ),

isboxed(isboxed),

isghost(false),

isimmutable(isboxed && jl_is_immutable_datatype(typ)),

tbaa(isboxed ? best_tbaa(typ) : nullptr)

{

assert(gcroot == nullptr);

assert(!(isboxed && TIndex != NULL));

assert(TIndex == NULL || TIndex->getType() == T_int8);

}

explicit jl_cgval_t(jl_value_t *typ) : // ghost value constructor

// mark explicit to avoid being used implicitly for conversion from NULL (use jl_cgval_t() instead)

V(NULL),

Vboxed(NULL),

TIndex(NULL),

constant(((jl_datatype_t*)typ)->instance),

typ(typ),

isboxed(false),

isghost(true),

isimmutable(true),

tbaa(nullptr)

{

assert(jl_is_datatype(typ));

assert(constant);

}

jl_cgval_t(const jl_cgval_t &v, jl_value_t *typ, Value *tindex) : // copy constructor with new type

V(v.V),

Vboxed(v.Vboxed),

TIndex(tindex),

constant(v.constant),

typ(typ),

isboxed(v.isboxed),

isghost(v.isghost),

isimmutable(v.isimmutable),

tbaa(v.tbaa)

{

// this constructor expects we had a badly or equivalently typed version

// make sure we aren't discarding the actual type information

if (v.TIndex) {

assert((TIndex == NULL) == jl_is_concrete_type(typ));

}

else {

assert(isboxed || v.typ == typ || tindex);

}

}

jl_cgval_t() : // undef / unreachable / default constructor

V(UndefValue::get(T_void)),

Vboxed(NULL),

TIndex(NULL),

constant(NULL),

typ(jl_bottom_type),

isboxed(false),

isghost(true),

isimmutable(true),

tbaa(nullptr)

{

}

Instruction *boxroot; // an address, if the var might be in a jl_value_t** stack slot (marked tbaa_const, if appropriate)

jl_cgval_t value; // a stack slot or constant value

Value *pTIndex; // i8* stack slot for the value.TIndex tag describing `value.V`

DILocalVariable *dinfo;

// if the variable might be used undefined and is not boxed

// this i1 flag is true when it is defined

Value *defFlag;

bool isSA;

bool isVolatile;

bool isArgument;

bool usedUndef;

bool used;

jl_varinfo_t() : boxroot(NULL),

value(jl_cgval_t()),

pTIndex(NULL),

dinfo(NULL),

defFlag(NULL),

isSA(false),

isVolatile(false),

isArgument(false),

usedUndef(false),

used(false)

{

}

Function *decl;

enum CallingConv {

Boxed = 0,

Register,

SRet,

Union,

Ghosts

} cc;

size_t union_bytes;

size_t union_align;

size_t union_minalign;

IRBuilder<> builder;

Function *f = NULL;

// local var info. globals are not in here.

std::vector<jl_varinfo_t> slots;

std::map<int, jl_varinfo_t> phic_slots;

std::vector<jl_cgval_t> SAvalues;

std::vector<std::tuple<jl_cgval_t, BasicBlock *, AllocaInst *, PHINode *, jl_value_t *>> PhiNodes;

std::vector<bool> ssavalue_assigned;

std::map<int, jl_arrayvar_t> *arrayvars = NULL;

jl_module_t *module = NULL;

jl_method_instance_t *linfo = NULL;

jl_code_info_t *source = NULL;

jl_array_t *code = NULL;

size_t world = 0;

jl_array_t *roots = NULL;

const char *name = NULL;

StringRef file{};

ssize_t *line = NULL;

Value *spvals_ptr = NULL;

Value *argArray = NULL;

Value *argCount = NULL;

std::string funcName;

int vaSlot = -1; // name of vararg argument

bool has_sret = false;

int nReqArgs = 0;

int nargs = 0;

int nvargs = -1;

CallInst *ptlsStates = NULL;

Value *signalPage = NULL;

Value *world_age_field = NULL;

bool debug_enabled = false;

const jl_cgparams_t *params = NULL;

jl_codectx_t(LLVMContext &llvmctx)

: builder(llvmctx) { }

~jl_codectx_t() {

assert(this->roots == NULL);

}

{

if (typ == jl_bottom_type)

return jl_cgval_t(); // Undef{}

if (typ == (jl_value_t*)jl_typeofbottom_type) {

// normalize TypeofBottom to Type{Union{}}

typ = (jl_value_t*)jl_wrap_Type(jl_bottom_type);

}

if (jl_is_type_type(typ)) {

// replace T::Type{T} with T, by assuming that T must be a leaftype of some sort

jl_cgval_t constant(NULL, NULL, true, typ, NULL);

constant.constant = jl_tparam0(typ);

return constant;

}

return jl_cgval_t(typ);

{

jl_value_t *typ;

if (jl_is_type(jv)) {

typ = (jl_value_t*)jl_wrap_Type(jv); // TODO: gc-root this?

}

else {

typ = jl_typeof(jv);

if (type_is_ghost(julia_type_to_llvm(typ))) {

return ghostValue(typ);

}

}

jl_cgval_t constant(NULL, NULL, true, typ, NULL);

constant.constant = jv;

return constant;

{

// this enables lazy-copying of immutable values and stack or argument slots

assert(tbaa);

jl_cgval_t tagval(v, NULL, false, typ, tindex);

tagval.tbaa = tbaa;

tagval.isimmutable = true;

return tagval;

{

Value *loc;

if (Constant *cv = dyn_cast<Constant>(v)) {

loc = get_pointer_to_constant(cv, "", *jl_Module);

}

else {

loc = emit_static_alloca(ctx, v->getType());

ctx.builder.CreateStore(v, loc);

}

return mark_julia_slot(loc, typ, tindex, tbaa_stack);

{

if (v.ispointer())

return v;

return value_to_pointer(ctx, v.V, v.typ, v.TIndex);

{

if (jl_is_datatype(typ) && jl_is_datatype_singleton((jl_datatype_t*)typ)) {

// no need to explicitly load/store a constant/ghost value

return ghostValue(typ);

}

if (jl_is_type_type(typ)) {

jl_value_t *tp0 = jl_tparam0(typ);

if (jl_is_concrete_type(tp0) || tp0 == jl_bottom_type) {

// replace T::Type{T} with T

return ghostValue(typ);

}

}

Type *T = julia_type_to_llvm(typ);

if (type_is_ghost(T)) {

return ghostValue(typ);

}

if (v && !isboxed && v->getType()->isAggregateType()) {

// eagerly put this back onto the stack

// llvm mem2reg pass will remove this if unneeded

return value_to_pointer(ctx, v, typ, NULL);

}

return jl_cgval_t(v, NULL, isboxed, typ, NULL);

{

if (v.typ == typ || v.typ == jl_bottom_type || v.constant || typ == (jl_value_t*)jl_any_type || jl_egal(v.typ, typ))

return v; // fast-path

if (jl_is_concrete_type(v.typ) && !jl_is_kind(v.typ)) {

if (jl_is_concrete_type(typ) && !jl_is_kind(typ)) {

// type mismatch: changing from one leaftype to another

CreateTrap(ctx.builder);

return jl_cgval_t();

}

return v; // doesn't improve type info

}

if (v.TIndex) {

jl_value_t *utyp = jl_unwrap_unionall(typ);

if (jl_is_datatype(utyp)) {

bool alwaysboxed;

if (jl_is_concrete_type(utyp))

alwaysboxed = !jl_justbits(utyp);

else

alwaysboxed = !((jl_datatype_t*)utyp)->abstract && ((jl_datatype_t*)utyp)->mutabl;

if (alwaysboxed) {

// discovered that this union-split type must actually be isboxed

if (v.Vboxed) {

return jl_cgval_t(v.Vboxed, nullptr, true, typ, NULL);

}

else {

// type mismatch (there weren't any boxed values in the union)

CreateTrap(ctx.builder);

return jl_cgval_t();

}

}

}

if (!jl_is_concrete_type(typ))

return v; // not generally worth trying to change type info (which would require recomputing tindex)

}

Type *T = julia_type_to_llvm(typ);

if (type_is_ghost(T))

return ghostValue(typ);

return jl_cgval_t(v, typ, NULL);

{

assert((!vi.boxroot || vi.pTIndex) && "undef check is null pointer for boxed things");

assert(vi.usedUndef && vi.defFlag && "undef flag codegen corrupted");

ctx.builder.CreateStore(ConstantInt::get(T_int1, val), vi.defFlag, vi.isVolatile);

{

assert((!vi.boxroot || vi.pTIndex) && "undef check is null pointer for boxed things");

if (vi.usedUndef) {

vi.defFlag = emit_static_alloca(ctx, T_int1);

store_def_flag(ctx, vi, false);

}

{

Function *f = irbuilder.GetInsertBlock()->getParent();

Function *trap_func = Intrinsic::getDeclaration(

f->getParent(),

Intrinsic::trap);

irbuilder.CreateCall(trap_func);

irbuilder.CreateUnreachable();

BasicBlock *newBB = BasicBlock::Create(irbuilder.getContext(), "after_noret", f);

irbuilder.SetInsertPoint(newBB);

{

Function *f = irbuilder.GetInsertBlock()->getParent();

BasicBlock *abortBB = BasicBlock::Create(jl_LLVMContext, "debug_abort", f);

BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_abort", f);

irbuilder.CreateCondBr(test, abortBB, postBB);

irbuilder.SetInsertPoint(abortBB);

Function *trap_func = Intrinsic::getDeclaration(

f->getParent(),

Intrinsic::trap);

irbuilder.CreateCall(trap_func);

irbuilder.CreateUnreachable();

irbuilder.SetInsertPoint(postBB);

{

jl_ptls_t ptls = jl_get_ptls_states();

if (jl_typeis(ptls->exception_in_transit, jl_errorexception_type)) {

char *str = jl_string_data(jl_fieldref(ptls->exception_in_transit,0));

char buf[1024];

va_list args;

va_start(args, fmt);

int nc = vsnprintf(buf, sizeof(buf), fmt, args);

va_end(args);

nc += snprintf(buf+nc, sizeof(buf)-nc, ": %s", str);

jl_value_t *msg = jl_pchar_to_string(buf, nc);

JL_GC_PUSH1(&msg);

jl_throw(jl_new_struct(jl_errorexception_type, msg));

}

jl_rethrow();

{

// previous value was a split union, compute new index, or box

Value *new_tindex = ConstantInt::get(T_int8, 0x80);

SmallBitVector skip_box(1, true);

Value *tindex = ctx.builder.CreateAnd(v.TIndex, ConstantInt::get(T_int8, 0x7f));

if (jl_is_uniontype(typ)) {

// compute the TIndex mapping from v.typ -> typ

unsigned counter = 0;

for_each_uniontype_small(

// for each old union-split value

[&](unsigned idx, jl_datatype_t *jt) {

unsigned new_idx = get_box_tindex(jt, typ);

bool t;

if (new_idx) {

// found a matching element,

// match it against either the unboxed index

Value *cmp = ctx.builder.CreateICmpEQ(tindex, ConstantInt::get(T_int8, idx));

new_tindex = ctx.builder.CreateSelect(cmp, ConstantInt::get(T_int8, new_idx), new_tindex);

t = true;

}

else if (!jl_subtype((jl_value_t*)jt, typ)) {

// new value doesn't need to be boxed

// since it isn't part of the new union

t = true;

}

else {

// will actually need to box this element

// since it appeared as a leaftype in the original type

// but not in the remark type

t = false;

}

skip_box.resize(idx + 1, t);

},

v.typ,

counter);

}

// some of the values are still unboxed

if (!isa<Constant>(new_tindex)) {

Value *wasboxed = NULL;

// If the old value was boxed and unknown (type tag 0x80),

// it is possible that the tag was actually one of the types

// that are now explicitly represented. To find out, we need

// to compare typeof(v.Vboxed) (i.e. the type of the unknown

// value) against all the types that are now explicitly

// selected and select the appropriate one as our new tindex.

if (v.Vboxed) {

wasboxed = ctx.builder.CreateAnd(v.TIndex, ConstantInt::get(T_int8, 0x80));

new_tindex = ctx.builder.CreateOr(wasboxed, new_tindex);

wasboxed = ctx.builder.CreateICmpNE(wasboxed, ConstantInt::get(T_int8, 0));

BasicBlock *currBB = ctx.builder.GetInsertBlock();

// We lazily create a BB for this, once we decide that we

// actually need it.

Value *union_box_dt = NULL;

BasicBlock *union_isaBB = NULL;

auto maybe_setup_union_isa = [&]() {

if (!union_isaBB) {

union_isaBB = BasicBlock::Create(jl_LLVMContext, "union_isa", ctx.f);

ctx.builder.SetInsertPoint(union_isaBB);

union_box_dt = emit_typeof(ctx, v.Vboxed);

}

};

// If we don't find a match. The type remains unknown

// (0x80). We could use `v.Tindex`, here, since we know

// it has to be 0x80, but it seems likely the backend

// will like the explicit constant better.

Value *union_box_tindex = ConstantInt::get(T_int8, 0x80);

unsigned counter = 0;

for_each_uniontype_small(

// for each new union-split value

[&](unsigned idx, jl_datatype_t *jt) {

unsigned old_idx = get_box_tindex(jt, v.typ);

if (old_idx == 0) {

// didn't handle this item before, select its new union index

maybe_setup_union_isa();

Value *cmp = ctx.builder.CreateICmpEQ(maybe_decay_untracked(literal_pointer_val(ctx, (jl_value_t*)jt)), union_box_dt);

union_box_tindex = ctx.builder.CreateSelect(cmp, ConstantInt::get(T_int8, 0x80 | idx), union_box_tindex);

}

},

typ,

counter);

if (union_box_dt) {

BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_union_isa", ctx.f);

ctx.builder.CreateBr(postBB);

ctx.builder.SetInsertPoint(currBB);

Value *wasunknown = ctx.builder.CreateICmpEQ(v.TIndex, ConstantInt::get(T_int8, 0x80));

ctx.builder.CreateCondBr(wasunknown, union_isaBB, postBB);

ctx.builder.SetInsertPoint(postBB);

PHINode *tindex_phi = ctx.builder.CreatePHI(T_int8, 2);

tindex_phi->addIncoming(new_tindex, currBB);

tindex_phi->addIncoming(union_box_tindex, union_isaBB);

new_tindex = tindex_phi;

}

}

if (!skip_box.all()) {

// some values weren't unboxed in the new union

// box them now (tindex above already selected 0x80 = box for them)

Value *boxv = box_union(ctx, v, skip_box);

if (v.Vboxed) {

// If the value is boxed both before and after, we don't need

// to touch it at all. Otherwise we're either transitioning

// unboxed->boxed, or leaving an unboxed value in place.

Value *isboxed = ctx.builder.CreateICmpNE(

ctx.builder.CreateAnd(new_tindex, ConstantInt::get(T_int8, 0x80)),

ConstantInt::get(T_int8, 0));

boxv = ctx.builder.CreateSelect(

ctx.builder.CreateAnd(wasboxed, isboxed), v.Vboxed, boxv);

}

if (v.V == NULL) {

// v.V might be NULL if it was all ghost objects before

return jl_cgval_t(boxv, NULL, false, typ, new_tindex);

} else {

Value *isboxv = ctx.builder.CreateIsNotNull(boxv);

Value *slotv;

MDNode *tbaa;

bool isimmutable;

if (v.ispointer()) {

slotv = v.V;

tbaa = v.tbaa;

isimmutable = v.isimmutable;

}

else {

slotv = emit_static_alloca(ctx, v.V->getType());

ctx.builder.CreateStore(v.V, slotv);

tbaa = tbaa_stack;

isimmutable = true;

}

slotv = ctx.builder.CreateSelect(isboxv,

decay_derived(boxv),

decay_derived(emit_bitcast(ctx, slotv, boxv->getType())));

jl_cgval_t newv = jl_cgval_t(slotv, NULL, false, typ, new_tindex);

newv.Vboxed = boxv;

newv.tbaa = tbaa;

newv.isimmutable = isimmutable;

return newv;

}

}

}

else {

return jl_cgval_t(boxed(ctx, v), NULL, true, typ, NULL);

}

return jl_cgval_t(v, typ, new_tindex);

{

if (typ == (jl_value_t*)jl_typeofbottom_type)

return ghostValue(typ); // normalize TypeofBottom to Type{Union{}}

if (v.typ == typ || v.typ == jl_bottom_type || jl_egal(v.typ, typ))

return v; // fast-path

Type *T = julia_type_to_llvm(typ);

if (type_is_ghost(T))

return ghostValue(typ);

Value *new_tindex = NULL;

if (jl_is_concrete_type(typ)) {

if (v.TIndex && !jl_justbits(typ)) {

// discovered that this union-split type must actually be isboxed

if (v.Vboxed) {

return jl_cgval_t(v.Vboxed, nullptr, true, typ, NULL);

}

else {

// type mismatch: there weren't any boxed values in the union