Skip to content

Commit

Permalink
Merge pull request #4 from LuaJIT/v2.1
Browse files Browse the repository at this point in the history
  • Loading branch information
@xrSimpodin
xrSimpodin authored Jun 24, 2018
2 parents 6671935 + 9b41062 commit 1d37c5b
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 39 deletions.
1 change: 1 addition & 0 deletions dynasm/dasm_arm64.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,7 @@ int dasm_encode(Dst_DECL, void *buffer)
break;
case DASM_REL_LG:
CK(n >= 0, UNDEF_LG);
/* fallthrough */
case DASM_REL_PC:
CK(n >= 0, UNDEF_PC);
n = *DASM_POS2PTR(D, n) - (int)((char *)cp - base) + 4;
Expand Down
64 changes: 37 additions & 27 deletions src/lj_asm_arm64.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,11 @@ static void asm_exitstub_setup(ASMState *as, ExitNo nexits)
asm_mclimit(as);
/* 1: str lr,[sp]; bl ->vm_exit_handler; movz w0,traceno; bl <1; bl <1; ... */
for (i = nexits-1; (int32_t)i >= 0; i--)
*--mxp = A64I_LE(A64I_BL|((-3-i)&0x03ffffffu));
*--mxp = A64I_LE(A64I_MOVZw|A64F_U16(as->T->traceno));
*--mxp = A64I_LE(A64I_BL | A64F_S26(-3-i));
*--mxp = A64I_LE(A64I_MOVZw | A64F_U16(as->T->traceno));
mxp--;
*mxp = A64I_LE(A64I_BL|(((MCode *)(void *)lj_vm_exit_handler-mxp)&0x03ffffffu));
*--mxp = A64I_LE(A64I_STRx|A64F_D(RID_LR)|A64F_N(RID_SP));
*mxp = A64I_LE(A64I_BL | A64F_S26(((MCode *)(void *)lj_vm_exit_handler-mxp)));
*--mxp = A64I_LE(A64I_STRx | A64F_D(RID_LR) | A64F_N(RID_SP));
as->mctop = mxp;
}

Expand All @@ -77,7 +77,7 @@ static void asm_guardcc(ASMState *as, A64CC cc)
MCode *p = as->mcp;
if (LJ_UNLIKELY(p == as->invmcp)) {
as->loopinv = 1;
*p = A64I_B | ((target-p) & 0x03ffffffu);
*p = A64I_B | A64F_S26(target-p);
emit_cond_branch(as, cc^1, p-1);
return;
}
Expand All @@ -91,7 +91,7 @@ static void asm_guardtnb(ASMState *as, A64Ins ai, Reg r, uint32_t bit)
MCode *p = as->mcp;
if (LJ_UNLIKELY(p == as->invmcp)) {
as->loopinv = 1;
*p = A64I_B | ((target-p) & 0x03ffffffu);
*p = A64I_B | A64F_S26(target-p);
emit_tnb(as, ai^0x01000000u, r, bit, p-1);
return;
}
Expand All @@ -105,7 +105,7 @@ static void asm_guardcnb(ASMState *as, A64Ins ai, Reg r)
MCode *p = as->mcp;
if (LJ_UNLIKELY(p == as->invmcp)) {
as->loopinv = 1;
*p = A64I_B | ((target-p) & 0x03ffffffu);
*p = A64I_B | A64F_S26(target-p);
emit_cnb(as, ai^0x01000000u, r, p-1);
return;
}
Expand Down Expand Up @@ -1850,7 +1850,7 @@ static void asm_loop_fixup(ASMState *as)
p[-2] |= ((uint32_t)delta & mask) << 5;
} else {
ptrdiff_t delta = target - (p - 1);
p[-1] = A64I_B | ((uint32_t)(delta) & 0x03ffffffu);
p[-1] = A64I_B | A64F_S26(delta);
}
}

Expand Down Expand Up @@ -1919,7 +1919,7 @@ static void asm_tail_fixup(ASMState *as, TraceNo lnk)
}
/* Patch exit branch. */
target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp;
p[-1] = A64I_B | (((target-p)+1)&0x03ffffffu);
p[-1] = A64I_B | A64F_S26((target-p)+1);
}

/* Prepare tail of code. */
Expand Down Expand Up @@ -1982,40 +1982,50 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
{
MCode *p = T->mcode;
MCode *pe = (MCode *)((char *)p + T->szmcode);
MCode *cstart = NULL, *cend = p;
MCode *cstart = NULL;
MCode *mcarea = lj_mcode_patch(J, p, 0);
MCode *px = exitstub_trace_addr(T, exitno);
/* Note: this assumes a trace exit is only ever patched once. */
for (; p < pe; p++) {
/* Look for exitstub branch, replace with branch to target. */
ptrdiff_t delta = target - p;
MCode ins = A64I_LE(*p);
if ((ins & 0xff000000u) == 0x54000000u &&
((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) {
/* Patch bcc exitstub. */
*p = A64I_LE((ins & 0xff00001fu) | (((target-p)<<5) & 0x00ffffe0u));
cend = p+1;
if (!cstart) cstart = p;
/* Patch bcc, if within range. */
if (A64F_S_OK(delta, 19)) {
*p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta));
if (!cstart) cstart = p;
}
} else if ((ins & 0xfc000000u) == 0x14000000u &&
((ins ^ (px-p)) & 0x03ffffffu) == 0) {
/* Patch b exitstub. */
*p = A64I_LE((ins & 0xfc000000u) | ((target-p) & 0x03ffffffu));
cend = p+1;
/* Patch b. */
lua_assert(A64F_S_OK(delta, 26));
*p = A64I_LE((ins & 0xfc000000u) | A64F_S26(delta));
if (!cstart) cstart = p;
} else if ((ins & 0x7e000000u) == 0x34000000u &&
((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) {
/* Patch cbz/cbnz exitstub. */
*p = A64I_LE((ins & 0xff00001f) | (((target-p)<<5) & 0x00ffffe0u));
cend = p+1;
if (!cstart) cstart = p;
/* Patch cbz/cbnz, if within range. */
if (A64F_S_OK(delta, 19)) {
*p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta));
if (!cstart) cstart = p;
}
} else if ((ins & 0x7e000000u) == 0x36000000u &&
((ins ^ ((px-p)<<5)) & 0x0007ffe0u) == 0) {
/* Patch tbz/tbnz exitstub. */
*p = A64I_LE((ins & 0xfff8001fu) | (((target-p)<<5) & 0x0007ffe0u));
cend = p+1;
if (!cstart) cstart = p;
/* Patch tbz/tbnz, if within range. */
if (A64F_S_OK(delta, 14)) {
*p = A64I_LE((ins & 0xfff8001fu) | A64F_S14(delta));
if (!cstart) cstart = p;
}
}
}
lua_assert(cstart != NULL);
lj_mcode_sync(cstart, cend);
{ /* Always patch long-range branch in exit stub itself. */
ptrdiff_t delta = target - px;
lua_assert(A64F_S_OK(delta, 26));
*px = A64I_B | A64F_S26(delta);
if (!cstart) cstart = px;
}
lj_mcode_sync(cstart, px+1);
lj_mcode_patch(J, mcarea, 1);
}

18 changes: 9 additions & 9 deletions src/lj_emit_arm64.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@ static void emit_loadk(ASMState *as, Reg rd, uint64_t u64, int is64)
#define mcpofs(as, k) \
((intptr_t)((uintptr_t)(k) - (uintptr_t)(as->mcp - 1)))
#define checkmcpofs(as, k) \
((((mcpofs(as, k)>>2) + 0x00040000) >> 19) == 0)
(A64F_S_OK(mcpofs(as, k)>>2, 19))

static Reg ra_allock(ASMState *as, intptr_t k, RegSet allow);

Expand Down Expand Up @@ -312,32 +312,32 @@ static void emit_cond_branch(ASMState *as, A64CC cond, MCode *target)
{
MCode *p = --as->mcp;
ptrdiff_t delta = target - p;
lua_assert(((delta + 0x40000) >> 19) == 0);
lua_assert(A64F_S_OK(delta, 19));
*p = A64I_BCC | A64F_S19(delta) | cond;
}

static void emit_branch(ASMState *as, A64Ins ai, MCode *target)
{
MCode *p = --as->mcp;
ptrdiff_t delta = target - p;
lua_assert(((delta + 0x02000000) >> 26) == 0);
*p = ai | ((uint32_t)delta & 0x03ffffffu);
lua_assert(A64F_S_OK(delta, 26));
*p = ai | A64F_S26(delta);
}

static void emit_tnb(ASMState *as, A64Ins ai, Reg r, uint32_t bit, MCode *target)
{
MCode *p = --as->mcp;
ptrdiff_t delta = target - p;
lua_assert(bit < 63 && ((delta + 0x2000) >> 14) == 0);
lua_assert(bit < 63 && A64F_S_OK(delta, 14));
if (bit > 31) ai |= A64I_X;
*p = ai | A64F_BIT(bit & 31) | A64F_S14((uint32_t)delta & 0x3fffu) | r;
*p = ai | A64F_BIT(bit & 31) | A64F_S14(delta) | r;
}

static void emit_cnb(ASMState *as, A64Ins ai, Reg r, MCode *target)
{
MCode *p = --as->mcp;
ptrdiff_t delta = target - p;
lua_assert(((delta + 0x40000) >> 19) == 0);
lua_assert(A64F_S_OK(delta, 19));
*p = ai | A64F_S19(delta) | r;
}

Expand All @@ -347,8 +347,8 @@ static void emit_call(ASMState *as, void *target)
{
MCode *p = --as->mcp;
ptrdiff_t delta = (char *)target - (char *)p;
if ((((delta>>2) + 0x02000000) >> 26) == 0) {
*p = A64I_BL | ((uint32_t)(delta>>2) & 0x03ffffffu);
if (A64F_S_OK(delta>>2, 26)) {
*p = A64I_BL | A64F_S26(delta>>2);
} else { /* Target out of range: need indirect call. But don't use R0-R7. */
Reg r = ra_allock(as, i64ptr(target),
RSET_RANGE(RID_X8, RID_MAX_GPR)-RSET_FIXED);
Expand Down
7 changes: 5 additions & 2 deletions src/lj_target_arm64.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,9 +132,9 @@ static LJ_AINLINE uint32_t *exitstub_trace_addr_(uint32_t *p, uint32_t exitno)
#define A64F_IMMR(x) ((x) << 16)
#define A64F_U16(x) ((x) << 5)
#define A64F_U12(x) ((x) << 10)
#define A64F_S26(x) (x)
#define A64F_S26(x) (((uint32_t)(x) & 0x03ffffffu))
#define A64F_S19(x) (((uint32_t)(x) & 0x7ffffu) << 5)
#define A64F_S14(x) ((x) << 5)
#define A64F_S14(x) (((uint32_t)(x) & 0x3fffu) << 5)
#define A64F_S9(x) ((x) << 12)
#define A64F_BIT(x) ((x) << 19)
#define A64F_SH(sh, x) (((sh) << 22) | ((x) << 10))
Expand All @@ -145,6 +145,9 @@ static LJ_AINLINE uint32_t *exitstub_trace_addr_(uint32_t *p, uint32_t exitno)
#define A64F_LSL16(x) (((x) / 16) << 21)
#define A64F_BSH(sh) ((sh) << 10)

/* Check for valid field range. */
#define A64F_S_OK(x, b) ((((x) + (1 << (b-1))) >> (b)) == 0)

typedef enum A64Ins {
A64I_S = 0x20000000,
A64I_X = 0x80000000,
Expand Down
2 changes: 1 addition & 1 deletion src/vm_arm64.dasc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2780,7 +2780,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
|2: // Check if string is white and ensure upvalue is closed.
| ldrb TMP0w, UPVAL:CARG1->closed
| tst TMP1w, #LJ_GC_WHITES // iswhite(str)
| ccmp TMP0w, #0, #0, ne
| ccmp TMP0w, #0, #4, ne
| beq <1
| // Crossed a write barrier. Move the barrier forward.
| mov CARG1, GL
Expand Down

0 comments on commit 1d37c5b

Please sign in to comment.