Skip to content

Commit

Permalink
improvements to constant values in IR (#45173)
Browse files Browse the repository at this point in the history 
- serialize bits values and QuoteNodes in IR encoding
  Using this instead of the `roots` array avoids scaling problems in
  functions with many constants.
- move roots created by codegen to a single global table, allowing
  reuse of egal values and keeping method roots lists shorter
  • Loading branch information
@JeffBezanson
JeffBezanson committed Aug 17, 2022
1 parent 0c49e42 commit 425f6ff
Show file tree
Hide file tree
Showing 10 changed files with 128 additions and 108 deletions.
4 changes: 2 additions & 2 deletions src/ccall.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1451,7 +1451,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
return jl_cgval_t();
}
if (rt != args[2] && rt != (jl_value_t*)jl_any_type)
jl_add_method_root(ctx, rt);
rt = jl_ensure_rooted(ctx, rt);
function_sig_t sig("ccall", lrt, rt, retboxed,
(jl_svec_t*)at, unionall, nreqargs,
cc, llvmcall, &ctx.emission_context);
Expand Down Expand Up @@ -1927,7 +1927,7 @@ jl_cgval_t function_sig_t::emit_a_ccall(
jl_svec_len(ctx.linfo->sparam_vals) > 0) {
jargty_in_env = jl_instantiate_type_in_env(jargty_in_env, unionall_env, jl_svec_data(ctx.linfo->sparam_vals));
if (jargty_in_env != jargty)
jl_add_method_root(ctx, jargty_in_env);
jargty_in_env = jl_ensure_rooted(ctx, jargty_in_env);
}

Value *v;
Expand Down
4 changes: 2 additions & 2 deletions src/cgutils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2978,7 +2978,7 @@ static Value *call_with_attrs(jl_codectx_t &ctx, JuliaFunction *intr, Value *v)
return Call;
}

static void jl_add_method_root(jl_codectx_t &ctx, jl_value_t *val);
static jl_value_t *jl_ensure_rooted(jl_codectx_t &ctx, jl_value_t *val);

static Value *as_value(jl_codectx_t &ctx, Type *to, const jl_cgval_t &v)
{
Expand Down Expand Up @@ -3011,7 +3011,7 @@ static Value *_boxed_special(jl_codectx_t &ctx, const jl_cgval_t &vinfo, Type *t
if (Constant *c = dyn_cast<Constant>(vinfo.V)) {
jl_value_t *s = static_constant_instance(jl_Module->getDataLayout(), c, jt);
if (s) {
jl_add_method_root(ctx, s);
s = jl_ensure_rooted(ctx, s);
return track_pjlvalue(ctx, literal_pointer_val(ctx, s));
}
}
Expand Down
107 changes: 27 additions & 80 deletions src/codegen.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1419,7 +1419,6 @@ class jl_codectx_t {
jl_code_info_t *source = NULL;
jl_array_t *code = NULL;
size_t world = 0;
jl_array_t *roots = NULL;
const char *name = NULL;
StringRef file{};
ssize_t *line = NULL;
Expand Down Expand Up @@ -1463,7 +1462,6 @@ class jl_codectx_t {
}

~jl_codectx_t() {
assert(this->roots == NULL);
// Transfer local delayed calls to the global queue
for (auto call_target : call_targets)
emission_context.workqueue.push_back(call_target);
Expand Down Expand Up @@ -2532,27 +2530,27 @@ static void simple_use_analysis(jl_codectx_t &ctx, jl_value_t *expr)

// ---- Get Element Pointer (GEP) instructions within the GC frame ----

static void jl_add_method_root(jl_codectx_t &ctx, jl_value_t *val)
static jl_value_t *jl_ensure_rooted(jl_codectx_t &ctx, jl_value_t *val)
{
if (jl_is_concrete_type(val) || jl_is_bool(val) || jl_is_symbol(val) || val == jl_nothing ||
val == (jl_value_t*)jl_any_type || val == (jl_value_t*)jl_bottom_type || val == (jl_value_t*)jl_core_module)
return;
JL_GC_PUSH1(&val);
if (ctx.roots == NULL) {
ctx.roots = jl_alloc_vec_any(1);
jl_array_ptr_set(ctx.roots, 0, val);
}
else {
size_t rlen = jl_array_dim0(ctx.roots);
for (size_t i = 0; i < rlen; i++) {
if (jl_array_ptr_ref(ctx.roots,i) == val) {
JL_GC_POP();
return;
if (jl_is_globally_rooted(val))
return val;
jl_method_t *m = ctx.linfo->def.method;
if (jl_is_method(m)) {
// the method might have a root for this already; use it if so
JL_LOCK(&m->writelock);
if (m->roots) {
size_t i, len = jl_array_dim0(m->roots);
for (i = 0; i < len; i++) {
jl_value_t *mval = jl_array_ptr_ref(m->roots, i);
if (mval == val || jl_egal(mval, val)) {
JL_UNLOCK(&m->writelock);
return mval;
}
}
}
jl_array_ptr_1d_push(ctx.roots, val);
JL_UNLOCK(&m->writelock);
}
JL_GC_POP();
return jl_as_global_root(val);
}

// --- generating function calls ---
Expand Down Expand Up @@ -3640,7 +3638,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f,
// don't bother codegen constant-folding for toplevel.
jl_value_t *ty = static_apply_type(ctx, argv, nargs + 1);
if (ty != NULL) {
jl_add_method_root(ctx, ty);
ty = jl_ensure_rooted(ctx, ty);
*ret = mark_julia_const(ctx, ty);
return true;
}
Expand Down Expand Up @@ -4934,35 +4932,12 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_
return convert_julia_type(ctx, emit_expr(ctx, jl_fieldref_noalloc(expr, 0)), jl_fieldref_noalloc(expr, 1), &skip);
}
if (!jl_is_expr(expr)) {
int needroot = true;
if (jl_is_quotenode(expr)) {
expr = jl_fieldref_noalloc(expr,0);
}
// numeric literals
if (jl_is_int32(expr)) {
int32_t val = jl_unbox_int32(expr);
if ((uint32_t)(val+512) < 1024) {
// this can be gotten from the box cache
needroot = false;
expr = jl_box_int32(val);
}
}
else if (jl_is_int64(expr)) {
uint64_t val = jl_unbox_uint64(expr);
if ((uint64_t)(val+512) < 1024) {
// this can be gotten from the box cache
needroot = false;
expr = jl_box_int64(val);
}
}
else if (jl_is_uint8(expr)) {
expr = jl_box_uint8(jl_unbox_uint8(expr));
needroot = false;
}
if (needroot && jl_is_method(ctx.linfo->def.method)) { // toplevel exprs and some integers are already rooted
jl_add_method_root(ctx, expr);
}
return mark_julia_const(ctx, expr);
jl_value_t *val = expr;
if (jl_is_quotenode(expr))
val = jl_fieldref_noalloc(expr, 0);
if (jl_is_method(ctx.linfo->def.method)) // toplevel exprs are already rooted
val = jl_ensure_rooted(ctx, val);
return mark_julia_const(ctx, val);
}

jl_expr_t *ex = (jl_expr_t*)expr;
Expand Down Expand Up @@ -6141,7 +6116,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con
return jl_cgval_t();
}
if (rt != declrt && rt != (jl_value_t*)jl_any_type)
jl_add_method_root(ctx, rt);
rt = jl_ensure_rooted(ctx, rt);

function_sig_t sig("cfunction", lrt, rt, retboxed, argt, unionall_env, false, CallingConv::C, false, &ctx.emission_context);
assert(sig.fargt.size() + sig.sret == sig.fargt_sig.size());
Expand Down Expand Up @@ -6214,7 +6189,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con
for (size_t i = 0; i < n; i++) {
jl_svecset(fill, i, jl_array_ptr_ref(closure_types, i));
}
jl_add_method_root(ctx, (jl_value_t*)fill);
fill = (jl_svec_t*)jl_ensure_rooted(ctx, (jl_value_t*)fill);
}
Type *T_htable = ArrayType::get(getSizeTy(ctx.builder.getContext()), sizeof(htable_t) / sizeof(void*));
Value *cache = new GlobalVariable(*jl_Module, T_htable, false,
Expand Down Expand Up @@ -6435,7 +6410,6 @@ static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlret
}
}
ctx.builder.CreateRet(boxed(ctx, retval));
assert(!ctx.roots);
return w;
}

Expand Down Expand Up @@ -6640,7 +6614,7 @@ static jl_llvm_functions_t
jl_llvm_functions_t declarations;
jl_codectx_t ctx(*params.tsctx.getContext(), params);
jl_datatype_t *vatyp = NULL;
JL_GC_PUSH3(&ctx.code, &ctx.roots, &vatyp);
JL_GC_PUSH2(&ctx.code, &vatyp);
ctx.code = src->code;

std::map<int, BasicBlock*> labels;
Expand Down Expand Up @@ -8116,33 +8090,6 @@ static jl_llvm_functions_t
}
}

// copy ctx.roots into m->roots
// if we created any new roots during codegen
if (ctx.roots) {
jl_method_t *m = lam->def.method;
JL_LOCK(&m->writelock);
if (m->roots == NULL) {
m->roots = ctx.roots;
jl_gc_wb(m, m->roots);
}
else {
size_t i, ilen = jl_array_dim0(ctx.roots);
size_t j, jlen = jl_array_dim0(m->roots);
for (i = 0; i < ilen; i++) {
jl_value_t *ival = jl_array_ptr_ref(ctx.roots, i);
for (j = 0; j < jlen; j++) {
jl_value_t *jval = jl_array_ptr_ref(m->roots, j);
if (ival == jval)
break;
}
if (j == jlen) // not found - add to array
jl_add_method_root(m, jl_precompile_toplevel_module, ival);
}
}
ctx.roots = NULL;
JL_UNLOCK(&m->writelock);
}

// link the dependent llvmcall modules, but switch their function's linkage to internal
// so that they don't conflict when they show up in the execution engine.
for (auto &TSMod : ctx.llvmcall_modules) {
Expand Down
1 change: 1 addition & 0 deletions src/gc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2886,6 +2886,7 @@ static void mark_roots(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp)
gc_mark_queue_obj(gc_cache, sp, jl_emptytuple_type);
if (cmpswap_names != NULL)
gc_mark_queue_obj(gc_cache, sp, cmpswap_names);
gc_mark_queue_obj(gc_cache, sp, jl_global_roots_table);
}

// find unmarked objects that need to be finalized from the finalizer list "list".
Expand Down
6 changes: 6 additions & 0 deletions src/iddict.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,12 @@ jl_value_t *jl_eqtable_get(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL
return (bp == NULL) ? deflt : jl_atomic_load_relaxed(bp);
}

jl_value_t *jl_eqtable_getkey(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT
{
_Atomic(jl_value_t*) *bp = jl_table_peek_bp(h, key);
return (bp == NULL) ? deflt : jl_atomic_load_relaxed(bp - 1);
}

JL_DLLEXPORT
jl_value_t *jl_eqtable_pop(jl_array_t *h, jl_value_t *key, jl_value_t *deflt, int *found)
{
Expand Down
1 change: 1 addition & 0 deletions src/init.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -744,6 +744,7 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_
jl_restore_system_image(jl_options.image_file);
} else {
jl_init_types();
jl_global_roots_table = jl_alloc_vec_any(16);
jl_init_codegen();
}

Expand Down
61 changes: 38 additions & 23 deletions src/ircode.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,31 @@ static void jl_encode_int32(jl_ircode_state *s, int32_t x)
}
}

static void jl_encode_as_indexed_root(jl_ircode_state *s, jl_value_t *v)
{
rle_reference rr;

literal_val_id(&rr, s, v);
int id = rr.index;
assert(id >= 0);
if (rr.key) {
write_uint8(s->s, TAG_RELOC_METHODROOT);
write_int64(s->s, rr.key);
}
if (id < 256) {
write_uint8(s->s, TAG_METHODROOT);
write_uint8(s->s, id);
}
else {
assert(id <= UINT16_MAX);
write_uint8(s->s, TAG_LONG_METHODROOT);
write_uint16(s->s, id);
}
}

static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) JL_GC_DISABLED
{
size_t i;
rle_reference rr;

if (v == NULL) {
write_uint8(s->s, TAG_NULL);
Expand Down Expand Up @@ -240,6 +261,16 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal)
write_uint8(s->s, TAG_RETURNNODE);
jl_encode_value(s, jl_get_nth_field(v, 0));
}
else if (jl_is_quotenode(v)) {
write_uint8(s->s, TAG_QUOTENODE);
jl_value_t *inner = jl_quotenode_value(v);
// we might need to return this exact object at run time, therefore codegen might
// need to reference it as well, so it is more likely useful to give it a root
if (jl_is_expr(inner) || jl_is_phinode(inner) || jl_is_phicnode(inner))
jl_encode_as_indexed_root(s, inner);
else
jl_encode_value(s, inner);
}
else if (jl_typeis(v, jl_int64_type)) {
void *data = jl_data_ptr(v);
if (*(int64_t*)data >= INT16_MIN && *(int64_t*)data <= INT16_MAX) {
Expand Down Expand Up @@ -325,28 +356,9 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal)
ios_write(s->s, jl_array_typetagdata(ar), l);
}
}
else {
if (!as_literal && !(jl_is_uniontype(v) || jl_is_newvarnode(v) || jl_is_tuple(v) ||
jl_is_linenode(v) || jl_is_upsilonnode(v) || jl_is_pinode(v) ||
jl_is_slot(v) || jl_is_ssavalue(v))) {
literal_val_id(&rr, s, v);
int id = rr.index;
assert(id >= 0);
if (rr.key) {
write_uint8(s->s, TAG_RELOC_METHODROOT);
write_int64(s->s, rr.key);
}
if (id < 256) {
write_uint8(s->s, TAG_METHODROOT);
write_uint8(s->s, id);
}
else {
assert(id <= UINT16_MAX);
write_uint8(s->s, TAG_LONG_METHODROOT);
write_uint16(s->s, id);
}
return;
}
else if (as_literal || jl_is_uniontype(v) || jl_is_newvarnode(v) || jl_is_linenode(v) ||
jl_is_upsilonnode(v) || jl_is_pinode(v) || jl_is_slot(v) || jl_is_ssavalue(v) ||
(jl_isbits(jl_typeof(v)) && jl_datatype_size(jl_typeof(v)) <= 64)) {
jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v);
if (t->size <= 255) {
write_uint8(s->s, TAG_SHORT_GENERAL);
Expand Down Expand Up @@ -388,6 +400,9 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal)
if (ptr > last)
ios_write(s->s, last, ptr - last);
}
else {
jl_encode_as_indexed_root(s, v);
}
}

static jl_code_info_flags_t code_info_flags(uint8_t pure, uint8_t propagate_inbounds, uint8_t inferred, uint8_t constprop)
Expand Down
3 changes: 2 additions & 1 deletion src/julia.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1280,7 +1280,7 @@ STATIC_INLINE int jl_is_structtype(void *v) JL_NOTSAFEPOINT
!jl_is_primitivetype(v));
}

STATIC_INLINE int jl_isbits(void *t) JL_NOTSAFEPOINT // corresponding to isbits() in julia
STATIC_INLINE int jl_isbits(void *t) JL_NOTSAFEPOINT // corresponding to isbitstype() in julia
{
return (jl_is_datatype(t) && ((jl_datatype_t*)t)->isbitstype);
}
Expand Down Expand Up @@ -1645,6 +1645,7 @@ STATIC_INLINE jl_function_t *jl_get_function(jl_module_t *m, const char *name)
// eq hash tables
JL_DLLEXPORT jl_array_t *jl_eqtable_put(jl_array_t *h, jl_value_t *key, jl_value_t *val, int *inserted);
JL_DLLEXPORT jl_value_t *jl_eqtable_get(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT;
jl_value_t *jl_eqtable_getkey(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT;

// system information
JL_DLLEXPORT int jl_errno(void) JL_NOTSAFEPOINT;
Expand Down
3 changes: 3 additions & 0 deletions src/julia_internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -713,6 +713,9 @@ JL_DLLEXPORT void jl_binding_deprecation_warning(jl_module_t *m, jl_binding_t *b
extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED;
extern htable_t jl_current_modules JL_GLOBALLY_ROOTED;
extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED;
extern jl_array_t *jl_global_roots_table JL_GLOBALLY_ROOTED;
JL_DLLEXPORT int jl_is_globally_rooted(jl_value_t *val JL_MAYBE_UNROOTED) JL_NOTSAFEPOINT;
JL_DLLEXPORT jl_value_t *jl_as_global_root(jl_value_t *val JL_MAYBE_UNROOTED);
int jl_compile_extern_c(LLVMOrcThreadSafeModuleRef llvmmod, void *params, void *sysimg, jl_value_t *declrt, jl_value_t *sigt);

jl_opaque_closure_t *jl_new_opaque_closure(jl_tupletype_t *argt, jl_value_t *rt_lb, jl_value_t *rt_ub,
Expand Down
Loading

2 comments on commit 425f6ff

@nanosoldier
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Executing the daily package evaluation, I will reply here when finished:

@nanosoldier runtests(ALL, isdaily = true)

@nanosoldier
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Your package evaluation job has completed - possible new issues were detected. A full report can be found here.

Please sign in to comment.