diff --git a/src/ccall.cpp b/src/ccall.cpp index 0fddf5425acc0..99cf6a1f67f96 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1451,7 +1451,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) return jl_cgval_t(); } if (rt != args[2] && rt != (jl_value_t*)jl_any_type) - jl_add_method_root(ctx, rt); + rt = jl_ensure_rooted(ctx, rt); function_sig_t sig("ccall", lrt, rt, retboxed, (jl_svec_t*)at, unionall, nreqargs, cc, llvmcall, &ctx.emission_context); @@ -1927,7 +1927,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( jl_svec_len(ctx.linfo->sparam_vals) > 0) { jargty_in_env = jl_instantiate_type_in_env(jargty_in_env, unionall_env, jl_svec_data(ctx.linfo->sparam_vals)); if (jargty_in_env != jargty) - jl_add_method_root(ctx, jargty_in_env); + jargty_in_env = jl_ensure_rooted(ctx, jargty_in_env); } Value *v; diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 2f1803ef91051..5b03dbf474f90 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -2978,7 +2978,7 @@ static Value *call_with_attrs(jl_codectx_t &ctx, JuliaFunction *intr, Value *v) return Call; } -static void jl_add_method_root(jl_codectx_t &ctx, jl_value_t *val); +static jl_value_t *jl_ensure_rooted(jl_codectx_t &ctx, jl_value_t *val); static Value *as_value(jl_codectx_t &ctx, Type *to, const jl_cgval_t &v) { @@ -3011,7 +3011,7 @@ static Value *_boxed_special(jl_codectx_t &ctx, const jl_cgval_t &vinfo, Type *t if (Constant *c = dyn_cast(vinfo.V)) { jl_value_t *s = static_constant_instance(jl_Module->getDataLayout(), c, jt); if (s) { - jl_add_method_root(ctx, s); + s = jl_ensure_rooted(ctx, s); return track_pjlvalue(ctx, literal_pointer_val(ctx, s)); } } diff --git a/src/codegen.cpp b/src/codegen.cpp index 8652536656f3d..645a92cb92203 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1419,7 +1419,6 @@ class jl_codectx_t { jl_code_info_t *source = NULL; jl_array_t *code = NULL; size_t world = 0; - jl_array_t *roots = NULL; const char *name = NULL; StringRef file{}; ssize_t *line = NULL; @@ -1463,7 +1462,6 @@ class jl_codectx_t { } ~jl_codectx_t() { - assert(this->roots == NULL); // Transfer local delayed calls to the global queue for (auto call_target : call_targets) emission_context.workqueue.push_back(call_target); @@ -2532,27 +2530,27 @@ static void simple_use_analysis(jl_codectx_t &ctx, jl_value_t *expr) // ---- Get Element Pointer (GEP) instructions within the GC frame ---- -static void jl_add_method_root(jl_codectx_t &ctx, jl_value_t *val) +static jl_value_t *jl_ensure_rooted(jl_codectx_t &ctx, jl_value_t *val) { - if (jl_is_concrete_type(val) || jl_is_bool(val) || jl_is_symbol(val) || val == jl_nothing || - val == (jl_value_t*)jl_any_type || val == (jl_value_t*)jl_bottom_type || val == (jl_value_t*)jl_core_module) - return; - JL_GC_PUSH1(&val); - if (ctx.roots == NULL) { - ctx.roots = jl_alloc_vec_any(1); - jl_array_ptr_set(ctx.roots, 0, val); - } - else { - size_t rlen = jl_array_dim0(ctx.roots); - for (size_t i = 0; i < rlen; i++) { - if (jl_array_ptr_ref(ctx.roots,i) == val) { - JL_GC_POP(); - return; + if (jl_is_globally_rooted(val)) + return val; + jl_method_t *m = ctx.linfo->def.method; + if (jl_is_method(m)) { + // the method might have a root for this already; use it if so + JL_LOCK(&m->writelock); + if (m->roots) { + size_t i, len = jl_array_dim0(m->roots); + for (i = 0; i < len; i++) { + jl_value_t *mval = jl_array_ptr_ref(m->roots, i); + if (mval == val || jl_egal(mval, val)) { + JL_UNLOCK(&m->writelock); + return mval; + } } } - jl_array_ptr_1d_push(ctx.roots, val); + JL_UNLOCK(&m->writelock); } - JL_GC_POP(); + return jl_as_global_root(val); } // --- generating function calls --- @@ -3640,7 +3638,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, // don't bother codegen constant-folding for toplevel. jl_value_t *ty = static_apply_type(ctx, argv, nargs + 1); if (ty != NULL) { - jl_add_method_root(ctx, ty); + ty = jl_ensure_rooted(ctx, ty); *ret = mark_julia_const(ctx, ty); return true; } @@ -4934,35 +4932,12 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ return convert_julia_type(ctx, emit_expr(ctx, jl_fieldref_noalloc(expr, 0)), jl_fieldref_noalloc(expr, 1), &skip); } if (!jl_is_expr(expr)) { - int needroot = true; - if (jl_is_quotenode(expr)) { - expr = jl_fieldref_noalloc(expr,0); - } - // numeric literals - if (jl_is_int32(expr)) { - int32_t val = jl_unbox_int32(expr); - if ((uint32_t)(val+512) < 1024) { - // this can be gotten from the box cache - needroot = false; - expr = jl_box_int32(val); - } - } - else if (jl_is_int64(expr)) { - uint64_t val = jl_unbox_uint64(expr); - if ((uint64_t)(val+512) < 1024) { - // this can be gotten from the box cache - needroot = false; - expr = jl_box_int64(val); - } - } - else if (jl_is_uint8(expr)) { - expr = jl_box_uint8(jl_unbox_uint8(expr)); - needroot = false; - } - if (needroot && jl_is_method(ctx.linfo->def.method)) { // toplevel exprs and some integers are already rooted - jl_add_method_root(ctx, expr); - } - return mark_julia_const(ctx, expr); + jl_value_t *val = expr; + if (jl_is_quotenode(expr)) + val = jl_fieldref_noalloc(expr, 0); + if (jl_is_method(ctx.linfo->def.method)) // toplevel exprs are already rooted + val = jl_ensure_rooted(ctx, val); + return mark_julia_const(ctx, val); } jl_expr_t *ex = (jl_expr_t*)expr; @@ -6141,7 +6116,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con return jl_cgval_t(); } if (rt != declrt && rt != (jl_value_t*)jl_any_type) - jl_add_method_root(ctx, rt); + rt = jl_ensure_rooted(ctx, rt); function_sig_t sig("cfunction", lrt, rt, retboxed, argt, unionall_env, false, CallingConv::C, false, &ctx.emission_context); assert(sig.fargt.size() + sig.sret == sig.fargt_sig.size()); @@ -6214,7 +6189,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con for (size_t i = 0; i < n; i++) { jl_svecset(fill, i, jl_array_ptr_ref(closure_types, i)); } - jl_add_method_root(ctx, (jl_value_t*)fill); + fill = (jl_svec_t*)jl_ensure_rooted(ctx, (jl_value_t*)fill); } Type *T_htable = ArrayType::get(getSizeTy(ctx.builder.getContext()), sizeof(htable_t) / sizeof(void*)); Value *cache = new GlobalVariable(*jl_Module, T_htable, false, @@ -6435,7 +6410,6 @@ static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlret } } ctx.builder.CreateRet(boxed(ctx, retval)); - assert(!ctx.roots); return w; } @@ -6640,7 +6614,7 @@ static jl_llvm_functions_t jl_llvm_functions_t declarations; jl_codectx_t ctx(*params.tsctx.getContext(), params); jl_datatype_t *vatyp = NULL; - JL_GC_PUSH3(&ctx.code, &ctx.roots, &vatyp); + JL_GC_PUSH2(&ctx.code, &vatyp); ctx.code = src->code; std::map labels; @@ -8116,33 +8090,6 @@ static jl_llvm_functions_t } } - // copy ctx.roots into m->roots - // if we created any new roots during codegen - if (ctx.roots) { - jl_method_t *m = lam->def.method; - JL_LOCK(&m->writelock); - if (m->roots == NULL) { - m->roots = ctx.roots; - jl_gc_wb(m, m->roots); - } - else { - size_t i, ilen = jl_array_dim0(ctx.roots); - size_t j, jlen = jl_array_dim0(m->roots); - for (i = 0; i < ilen; i++) { - jl_value_t *ival = jl_array_ptr_ref(ctx.roots, i); - for (j = 0; j < jlen; j++) { - jl_value_t *jval = jl_array_ptr_ref(m->roots, j); - if (ival == jval) - break; - } - if (j == jlen) // not found - add to array - jl_add_method_root(m, jl_precompile_toplevel_module, ival); - } - } - ctx.roots = NULL; - JL_UNLOCK(&m->writelock); - } - // link the dependent llvmcall modules, but switch their function's linkage to internal // so that they don't conflict when they show up in the execution engine. for (auto &TSMod : ctx.llvmcall_modules) { diff --git a/src/gc.c b/src/gc.c index c45ff8206ca67..7c577cccafff8 100644 --- a/src/gc.c +++ b/src/gc.c @@ -2886,6 +2886,7 @@ static void mark_roots(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp_t *sp) gc_mark_queue_obj(gc_cache, sp, jl_emptytuple_type); if (cmpswap_names != NULL) gc_mark_queue_obj(gc_cache, sp, cmpswap_names); + gc_mark_queue_obj(gc_cache, sp, jl_global_roots_table); } // find unmarked objects that need to be finalized from the finalizer list "list". diff --git a/src/iddict.c b/src/iddict.c index da2c36d97d2e4..1fa8a67d1ae96 100644 --- a/src/iddict.c +++ b/src/iddict.c @@ -159,6 +159,12 @@ jl_value_t *jl_eqtable_get(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL return (bp == NULL) ? deflt : jl_atomic_load_relaxed(bp); } +jl_value_t *jl_eqtable_getkey(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT +{ + _Atomic(jl_value_t*) *bp = jl_table_peek_bp(h, key); + return (bp == NULL) ? deflt : jl_atomic_load_relaxed(bp - 1); +} + JL_DLLEXPORT jl_value_t *jl_eqtable_pop(jl_array_t *h, jl_value_t *key, jl_value_t *deflt, int *found) { diff --git a/src/init.c b/src/init.c index f7f33a1c38578..565f3d55f0f78 100644 --- a/src/init.c +++ b/src/init.c @@ -744,6 +744,7 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ jl_restore_system_image(jl_options.image_file); } else { jl_init_types(); + jl_global_roots_table = jl_alloc_vec_any(16); jl_init_codegen(); } diff --git a/src/ircode.c b/src/ircode.c index 2116022eae3b9..5dae26cfadf8c 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -71,10 +71,31 @@ static void jl_encode_int32(jl_ircode_state *s, int32_t x) } } +static void jl_encode_as_indexed_root(jl_ircode_state *s, jl_value_t *v) +{ + rle_reference rr; + + literal_val_id(&rr, s, v); + int id = rr.index; + assert(id >= 0); + if (rr.key) { + write_uint8(s->s, TAG_RELOC_METHODROOT); + write_int64(s->s, rr.key); + } + if (id < 256) { + write_uint8(s->s, TAG_METHODROOT); + write_uint8(s->s, id); + } + else { + assert(id <= UINT16_MAX); + write_uint8(s->s, TAG_LONG_METHODROOT); + write_uint16(s->s, id); + } +} + static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) JL_GC_DISABLED { size_t i; - rle_reference rr; if (v == NULL) { write_uint8(s->s, TAG_NULL); @@ -240,6 +261,16 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) write_uint8(s->s, TAG_RETURNNODE); jl_encode_value(s, jl_get_nth_field(v, 0)); } + else if (jl_is_quotenode(v)) { + write_uint8(s->s, TAG_QUOTENODE); + jl_value_t *inner = jl_quotenode_value(v); + // we might need to return this exact object at run time, therefore codegen might + // need to reference it as well, so it is more likely useful to give it a root + if (jl_is_expr(inner) || jl_is_phinode(inner) || jl_is_phicnode(inner)) + jl_encode_as_indexed_root(s, inner); + else + jl_encode_value(s, inner); + } else if (jl_typeis(v, jl_int64_type)) { void *data = jl_data_ptr(v); if (*(int64_t*)data >= INT16_MIN && *(int64_t*)data <= INT16_MAX) { @@ -325,28 +356,9 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) ios_write(s->s, jl_array_typetagdata(ar), l); } } - else { - if (!as_literal && !(jl_is_uniontype(v) || jl_is_newvarnode(v) || jl_is_tuple(v) || - jl_is_linenode(v) || jl_is_upsilonnode(v) || jl_is_pinode(v) || - jl_is_slot(v) || jl_is_ssavalue(v))) { - literal_val_id(&rr, s, v); - int id = rr.index; - assert(id >= 0); - if (rr.key) { - write_uint8(s->s, TAG_RELOC_METHODROOT); - write_int64(s->s, rr.key); - } - if (id < 256) { - write_uint8(s->s, TAG_METHODROOT); - write_uint8(s->s, id); - } - else { - assert(id <= UINT16_MAX); - write_uint8(s->s, TAG_LONG_METHODROOT); - write_uint16(s->s, id); - } - return; - } + else if (as_literal || jl_is_uniontype(v) || jl_is_newvarnode(v) || jl_is_linenode(v) || + jl_is_upsilonnode(v) || jl_is_pinode(v) || jl_is_slot(v) || jl_is_ssavalue(v) || + (jl_isbits(jl_typeof(v)) && jl_datatype_size(jl_typeof(v)) <= 64)) { jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v); if (t->size <= 255) { write_uint8(s->s, TAG_SHORT_GENERAL); @@ -388,6 +400,9 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) if (ptr > last) ios_write(s->s, last, ptr - last); } + else { + jl_encode_as_indexed_root(s, v); + } } static jl_code_info_flags_t code_info_flags(uint8_t pure, uint8_t propagate_inbounds, uint8_t inferred, uint8_t constprop) diff --git a/src/julia.h b/src/julia.h index 1fc293dc6dd16..424c6e0cdc2c5 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1280,7 +1280,7 @@ STATIC_INLINE int jl_is_structtype(void *v) JL_NOTSAFEPOINT !jl_is_primitivetype(v)); } -STATIC_INLINE int jl_isbits(void *t) JL_NOTSAFEPOINT // corresponding to isbits() in julia +STATIC_INLINE int jl_isbits(void *t) JL_NOTSAFEPOINT // corresponding to isbitstype() in julia { return (jl_is_datatype(t) && ((jl_datatype_t*)t)->isbitstype); } @@ -1645,6 +1645,7 @@ STATIC_INLINE jl_function_t *jl_get_function(jl_module_t *m, const char *name) // eq hash tables JL_DLLEXPORT jl_array_t *jl_eqtable_put(jl_array_t *h, jl_value_t *key, jl_value_t *val, int *inserted); JL_DLLEXPORT jl_value_t *jl_eqtable_get(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT; +jl_value_t *jl_eqtable_getkey(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT; // system information JL_DLLEXPORT int jl_errno(void) JL_NOTSAFEPOINT; diff --git a/src/julia_internal.h b/src/julia_internal.h index 76f16090e64e7..5829bd2dc742d 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -713,6 +713,9 @@ JL_DLLEXPORT void jl_binding_deprecation_warning(jl_module_t *m, jl_binding_t *b extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED; extern htable_t jl_current_modules JL_GLOBALLY_ROOTED; extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; +extern jl_array_t *jl_global_roots_table JL_GLOBALLY_ROOTED; +JL_DLLEXPORT int jl_is_globally_rooted(jl_value_t *val JL_MAYBE_UNROOTED) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_as_global_root(jl_value_t *val JL_MAYBE_UNROOTED); int jl_compile_extern_c(LLVMOrcThreadSafeModuleRef llvmmod, void *params, void *sysimg, jl_value_t *declrt, jl_value_t *sigt); jl_opaque_closure_t *jl_new_opaque_closure(jl_tupletype_t *argt, jl_value_t *rt_lb, jl_value_t *rt_ub, diff --git a/src/staticdata.c b/src/staticdata.c index 9f5c1e64ba928..af3e63c10123c 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1802,6 +1802,49 @@ static void jl_set_nroots_sysimg(void) static void jl_init_serializer2(int); static void jl_cleanup_serializer2(void); +jl_array_t *jl_global_roots_table; +static jl_mutex_t global_roots_lock; + +JL_DLLEXPORT int jl_is_globally_rooted(jl_value_t *val JL_MAYBE_UNROOTED) JL_NOTSAFEPOINT +{ + if (jl_is_concrete_type(val) || jl_is_bool(val) || jl_is_symbol(val) || + val == (jl_value_t*)jl_any_type || val == (jl_value_t*)jl_bottom_type || val == (jl_value_t*)jl_core_module) + return 1; + if (val == ((jl_datatype_t*)jl_typeof(val))->instance) + return 1; + return 0; +} + +JL_DLLEXPORT jl_value_t *jl_as_global_root(jl_value_t *val JL_MAYBE_UNROOTED) +{ + if (jl_is_globally_rooted(val)) + return val; + if (jl_is_uint8(val)) + return jl_box_uint8(jl_unbox_uint8(val)); + if (jl_is_int32(val)) { + int32_t n = jl_unbox_int32(val); + if ((uint32_t)(n+512) < 1024) + return jl_box_int32(n); + } + else if (jl_is_int64(val)) { + uint64_t n = jl_unbox_uint64(val); + if ((uint64_t)(n+512) < 1024) + return jl_box_int64(n); + } + JL_GC_PUSH1(&val); + JL_LOCK(&global_roots_lock); + jl_value_t *rval = jl_eqtable_getkey(jl_global_roots_table, val, NULL); + if (rval) { + val = rval; + } + else { + jl_global_roots_table = jl_eqtable_put(jl_global_roots_table, val, jl_nothing, NULL); + } + JL_UNLOCK(&global_roots_lock); + JL_GC_POP(); + return val; +} + static void jl_save_system_image_to_stream(ios_t *f) JL_GC_DISABLED { jl_gc_collect(JL_GC_FULL); @@ -1868,6 +1911,7 @@ static void jl_save_system_image_to_stream(ios_t *f) JL_GC_DISABLED jl_value_t *tag = *tags[i]; jl_serialize_value(&s, tag); } + jl_serialize_value(&s, jl_global_roots_table); jl_serialize_reachable(&s); // step 1.1: check for values only found in the generated code arraylist_t typenames; @@ -1948,6 +1992,7 @@ static void jl_save_system_image_to_stream(ios_t *f) JL_GC_DISABLED jl_value_t *tag = *tags[i]; jl_write_value(&s, tag); } + jl_write_value(&s, jl_global_roots_table); jl_write_value(&s, s.ptls->root_task->tls); write_uint32(f, jl_get_gs_ctr()); write_uint32(f, jl_atomic_load_acquire(&jl_world_counter)); @@ -2074,6 +2119,7 @@ static void jl_restore_system_image_from_stream(ios_t *f) JL_GC_DISABLED jl_value_t **tag = tags[i]; *tag = jl_read_value(&s); } + jl_global_roots_table = (jl_array_t*)jl_read_value(&s); // set typeof extra-special values now that we have the type set by tags above jl_astaggedvalue(jl_current_task)->header = (uintptr_t)jl_task_type | jl_astaggedvalue(jl_current_task)->header; jl_astaggedvalue(jl_nothing)->header = (uintptr_t)jl_nothing_type | jl_astaggedvalue(jl_nothing)->header;