Skip to content

gh-134584: Decref elimination for float ops in the JIT #134588

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Fidget-Spinner merged 27 commits into from
Jun 17, 2025
Merged

gh-134584: Decref elimination for float ops in the JIT #134588

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
9466417
Skip refcounting where possible for common float ops
Fidget-Spinner May 23, 2025
ad03b1e
Add for common list ops
Fidget-Spinner May 23, 2025
7f90d0c
Fix test, rename
Fidget-Spinner May 23, 2025
a42b434
Remove list optimizations to minimize PR
Fidget-Spinner May 23, 2025
f456740
📜🤖 Added by blurb_it.
blurb-it[bot] May 23, 2025
16f9dee
Merge remote-tracking branch 'upstream/main' into decref_elimination_…
Fidget-Spinner May 27, 2025
5c429b6
Rename things to make things clearer
Fidget-Spinner May 27, 2025
1535133
Revert "Rename things to make things clearer"
Fidget-Spinner May 27, 2025
8f62067
Massive refactor from JitOptSymbol to JitRef
Fidget-Spinner May 28, 2025
a158835
refactor more
Fidget-Spinner May 28, 2025
e77f842
fix debug build
Fidget-Spinner May 28, 2025
01004c2
lint
Fidget-Spinner May 28, 2025
24f98d5
Merge remote-tracking branch 'upstream/main' into decref_elimination_…
Fidget-Spinner May 28, 2025
0189413
fix upstream
Fidget-Spinner May 28, 2025
4a386bf
reduce diff
Fidget-Spinner May 28, 2025
ac034a0
fix for FT
Fidget-Spinner May 28, 2025
b6e467e
fix failing tests
Fidget-Spinner May 28, 2025
3a3fa9d
Fix for disabled GIL
Fidget-Spinner May 29, 2025
ab1ad9c
fix on FT again
Fidget-Spinner May 29, 2025
5d82489
Try fix windows
Fidget-Spinner May 29, 2025
4d9a68e
Apply code review suggestions from Tomas
Fidget-Spinner Jun 4, 2025
2bbd47a
call the functions sym instead of ref
Fidget-Spinner Jun 4, 2025
2d779c4
rename jitref functions
Fidget-Spinner Jun 4, 2025
b74e160
Address review
Fidget-Spinner Jun 4, 2025
3ebcc20
Update comment
Fidget-Spinner Jun 4, 2025
673d5c8
Merge remote-tracking branch 'upstream/main' into decref_elimination_…
Fidget-Spinner Jun 17, 2025
914f1ff
Fix changes from upstream (no more casts)
Fidget-Spinner Jun 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
rename jitref functions
  • Loading branch information
@Fidget-Spinner
Fidget-Spinner committed Jun 4, 2025
commit 2d779c4ded74b3287f6ea72ef8795e9dde126146
11 changes: 3 additions & 8 deletions Include/internal/pycore_optimizer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,24 +230,19 @@ typedef union {
#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~Py_TAG_REFCNT)))

static inline JitOptSymbol *
PyJitRef_AsSymbolBorrow(JitOptRef ref)
PyJitRef_Unwrap(JitOptRef ref)
{
return JIT_BITS_TO_PTR_MASKED(ref);
}

bool _Py_uop_symbol_is_immortal(JitOptSymbol *sym);

static inline JitOptRef
PyJitRef_FromSymbolBorrow(JitOptSymbol *sym)
{
return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT};
}

static inline JitOptRef
PyJitRef_FromSymbolSteal(JitOptSymbol *sym)
PyJitRef_Wrap(JitOptSymbol *sym)
{
if (sym == NULL || _Py_uop_symbol_is_immortal(sym)) {
return PyJitRef_FromSymbolBorrow(sym);
return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT};
}
return (JitOptRef){.bits=(uintptr_t)sym};
}
Expand Down
65 changes: 33 additions & 32 deletions Python/optimizer_symbols.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ out_of_space(JitOptContext *ctx)
JitOptRef
out_of_space_ref(JitOptContext *ctx)
{
return PyJitRef_FromSymbolSteal(out_of_space(ctx));
return PyJitRef_Wrap(out_of_space(ctx));
}

static JitOptSymbol *
Expand Down Expand Up @@ -125,26 +125,26 @@ sym_set_bottom(JitOptContext *ctx, JitOptSymbol *sym)
bool
_Py_uop_sym_is_bottom(JitOptRef ref)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
return sym->tag == JIT_SYM_BOTTOM_TAG;
}

bool
_Py_uop_sym_is_not_null(JitOptRef ref) {
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
return sym->tag == JIT_SYM_NON_NULL_TAG || sym->tag > JIT_SYM_BOTTOM_TAG;
}

bool
_Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef ref)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) {
return true;
}
if (sym->tag == JIT_SYM_TRUTHINESS_TAG) {
JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value;
int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_FromSymbolSteal(value));
int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_Wrap(value));
if (truthiness < 0) {
return false;
}
Expand All @@ -157,20 +157,20 @@ _Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef ref)
bool
_Py_uop_sym_is_null(JitOptRef ref)
{
return PyJitRef_AsSymbolBorrow(ref)->tag == JIT_SYM_NULL_TAG;
return PyJitRef_Unwrap(ref)->tag == JIT_SYM_NULL_TAG;
}


PyObject *
_Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef ref)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) {
return sym->value.value;
}
if (sym->tag == JIT_SYM_TRUTHINESS_TAG) {
JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value;
int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_FromSymbolSteal(value));
int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_Wrap(value));
if (truthiness < 0) {
return NULL;
}
Expand All @@ -184,7 +184,7 @@ _Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef ref)
void
_Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
JitSymType tag = sym->tag;
switch(tag) {
case JIT_SYM_NULL_TAG:
Expand Down Expand Up @@ -235,7 +235,7 @@ _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ)
bool
_Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int version)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
PyTypeObject *type = _PyType_LookupByVersion(version);
if (type) {
_Py_uop_sym_set_type(ctx, ref, type);
Expand Down Expand Up @@ -293,7 +293,7 @@ _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int ver
void
_Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
JitSymType tag = sym->tag;
switch(tag) {
case JIT_SYM_NULL_TAG:
Expand Down Expand Up @@ -348,7 +348,8 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val)
sym_set_bottom(ctx, sym);
return;
}
JitOptRef value = PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->truthiness.value);
JitOptRef value = PyJitRef_Wrap(
allocation_base(ctx) + sym->truthiness.value);
PyTypeObject *type = _Py_uop_sym_get_type(value);
if (const_val == (sym->truthiness.invert ? Py_False : Py_True)) {
// value is truthy. This is only useful for bool:
Expand All @@ -375,7 +376,7 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val)
void
_Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef ref)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
if (sym->tag == JIT_SYM_UNKNOWN_TAG) {
sym->tag = JIT_SYM_NULL_TAG;
}
Expand All @@ -387,7 +388,7 @@ _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef ref)
void
_Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef ref)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
if (sym->tag == JIT_SYM_UNKNOWN_TAG) {
sym->tag = JIT_SYM_NON_NULL_TAG;
}
Expand All @@ -403,7 +404,7 @@ _Py_uop_sym_new_unknown(JitOptContext *ctx)
if (res == NULL) {
return out_of_space_ref(ctx);
}
return PyJitRef_FromSymbolSteal(res);
return PyJitRef_Wrap(res);
}

JitOptRef
Expand All @@ -414,7 +415,7 @@ _Py_uop_sym_new_not_null(JitOptContext *ctx)
return out_of_space_ref(ctx);
}
res->tag = JIT_SYM_NON_NULL_TAG;
return PyJitRef_FromSymbolSteal(res);
return PyJitRef_Wrap(res);
}

JitOptRef
Expand All @@ -424,7 +425,7 @@ _Py_uop_sym_new_type(JitOptContext *ctx, PyTypeObject *typ)
if (res == NULL) {
return out_of_space_ref(ctx);
}
JitOptRef ref = PyJitRef_FromSymbolSteal(res);
JitOptRef ref = PyJitRef_Wrap(res);
_Py_uop_sym_set_type(ctx, ref, typ);
return ref;
}
Expand All @@ -438,7 +439,7 @@ _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val)
if (res == NULL) {
return out_of_space_ref(ctx);
}
JitOptRef ref = PyJitRef_FromSymbolSteal(res);
JitOptRef ref = PyJitRef_Wrap(res);
_Py_uop_sym_set_const(ctx, ref, const_val);
return ref;
}
Expand All @@ -450,15 +451,15 @@ _Py_uop_sym_new_null(JitOptContext *ctx)
if (null_sym == NULL) {
return out_of_space_ref(ctx);
}
JitOptRef ref = PyJitRef_FromSymbolSteal(null_sym);
JitOptRef ref = PyJitRef_Wrap(null_sym);
_Py_uop_sym_set_null(ctx, ref);
return ref;
}

PyTypeObject *
_Py_uop_sym_get_type(JitOptRef ref)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
JitSymType tag = sym->tag;
switch(tag) {
case JIT_SYM_NULL_TAG:
Expand All @@ -483,7 +484,7 @@ _Py_uop_sym_get_type(JitOptRef ref)
unsigned int
_Py_uop_sym_get_type_version(JitOptRef ref)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
JitSymType tag = sym->tag;
switch(tag) {
case JIT_SYM_NULL_TAG:
Expand Down Expand Up @@ -527,7 +528,7 @@ _Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version)
int
_Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef ref)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
switch(sym->tag) {
case JIT_SYM_NULL_TAG:
case JIT_SYM_TYPE_VERSION_TAG:
Expand All @@ -548,7 +549,7 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef ref)
;
JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value;
int truthiness = _Py_uop_sym_truthiness(ctx,
PyJitRef_FromSymbolBorrow(value));
PyJitRef_Wrap(value));
if (truthiness < 0) {
return truthiness;
}
Expand Down Expand Up @@ -589,16 +590,16 @@ _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args)
res->tag = JIT_SYM_TUPLE_TAG;
res->tuple.length = size;
for (int i = 0; i < size; i++) {
res->tuple.items[i] = (uint16_t)(PyJitRef_AsSymbolBorrow(args[i]) - allocation_base(ctx));
res->tuple.items[i] = (uint16_t)(PyJitRef_Unwrap(args[i]) - allocation_base(ctx));
}
}
return PyJitRef_FromSymbolSteal(res);
return PyJitRef_Wrap(res);
}

JitOptRef
_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
assert(item >= 0);
if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) {
PyObject *tuple = sym->value.value;
Expand All @@ -607,15 +608,15 @@ _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item)
}
}
else if (sym->tag == JIT_SYM_TUPLE_TAG && item < sym->tuple.length) {
return PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->tuple.items[item]);
return PyJitRef_Wrap(allocation_base(ctx) + sym->tuple.items[item]);
}
return _Py_uop_sym_new_not_null(ctx);
}

int
_Py_uop_sym_tuple_length(JitOptRef ref)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) {
PyObject *tuple = sym->value.value;
if (PyTuple_CheckExact(tuple)) {
Expand Down Expand Up @@ -647,14 +648,14 @@ _Py_uop_symbol_is_immortal(JitOptSymbol *sym)
bool
_Py_uop_sym_is_immortal(JitOptRef ref)
{
JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *sym = PyJitRef_Unwrap(ref);
return _Py_uop_symbol_is_immortal(sym);
}

JitOptRef
_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy)
{
JitOptSymbol *value = PyJitRef_AsSymbolBorrow(ref);
JitOptSymbol *value = PyJitRef_Unwrap(ref);
// It's clearer to invert this in the signature:
bool invert = !truthy;
if (value->tag == JIT_SYM_TRUTHINESS_TAG && value->truthiness.invert == invert) {
Expand All @@ -673,7 +674,7 @@ _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy)
else {
make_const(res, (truthiness ^ invert) ? Py_True : Py_False);
}
return PyJitRef_FromSymbolSteal(res);
return PyJitRef_Wrap(res);
}

// 0 on success, -1 on error.
Expand Down Expand Up @@ -809,7 +810,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored))
TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "top as constant is not NULL");
TEST_PREDICATE(!_Py_uop_sym_is_bottom(ref), "top is bottom");

ref = PyJitRef_FromSymbolSteal(make_bottom(ctx));
ref = PyJitRef_Wrap(make_bottom(ctx));
if (PyJitRef_IsNull(ref)) {
goto fail;
}
Expand Down
Loading