Skip to content

Commit 2fac8e0

Browse files
vtjnashvtjnash
committed
intrinsics: optimize several atomic intrinsics
1 parent 468b157 commit 2fac8e0

File tree

4 files changed

+314
-93
lines changed

4 files changed

+314
-93
lines changed

‎src/intrinsics.cpp‎

Lines changed: 177 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -584,25 +584,21 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv)
584584
jl_value_t *ety = jl_tparam0(aty);
585585
if (jl_is_typevar(ety))
586586
return emit_runtime_pointerref(ctx, argv);
587-
if (!jl_is_datatype(ety))
588-
ety = (jl_value_t*)jl_any_type;
587+
if (!is_valid_intrinsic_elptr(ety)) {
588+
emit_error(ctx, "pointerref: invalid pointer type");
589+
return jl_cgval_t();
590+
}
589591

590592
Value *idx = emit_unbox(ctx, T_size, i, (jl_value_t*)jl_long_type);
591593
Value *im1 = ctx.builder.CreateSub(idx, ConstantInt::get(T_size, 1));
592594

593595
if (ety == (jl_value_t*)jl_any_type) {
594596
Value *thePtr = emit_unbox(ctx, T_pprjlvalue, e, e.typ);
595-
return mark_julia_type(
596-
ctx,
597-
ctx.builder.CreateAlignedLoad(ctx.builder.CreateInBoundsGEP(T_prjlvalue, thePtr, im1), Align(align_nb)),
598-
true,
599-
ety);
597+
LoadInst *load = ctx.builder.CreateAlignedLoad(ctx.builder.CreateInBoundsGEP(T_prjlvalue, thePtr, im1), Align(align_nb));
598+
tbaa_decorate(tbaa_data, load);
599+
return mark_julia_type(ctx, load, true, ety);
600600
}
601601
else if (!jl_isbits(ety)) {
602-
if (!jl_is_structtype(ety) || jl_is_array_type(ety) || !jl_is_concrete_type(ety)) {
603-
emit_error(ctx, "pointerref: invalid pointer type");
604-
return jl_cgval_t();
605-
}
606602
assert(jl_is_datatype(ety));
607603
uint64_t size = jl_datatype_size(ety);
608604
Value *strct = emit_allocobj(ctx, size,
@@ -656,8 +652,8 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, jl_cgval_t *argv)
656652
return emit_runtime_pointerset(ctx, argv);
657653
if (align.constant == NULL || !jl_is_long(align.constant))
658654
return emit_runtime_pointerset(ctx, argv);
659-
if (!jl_is_datatype(ety))
660-
ety = (jl_value_t*)jl_any_type;
655+
if (!is_valid_intrinsic_elptr(ety))
656+
emit_error(ctx, "pointerset: invalid pointer type");
661657
emit_typecheck(ctx, x, ety, "pointerset");
662658

663659
Value *idx = emit_unbox(ctx, T_size, i, (jl_value_t*)jl_long_type);
@@ -673,10 +669,6 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, jl_cgval_t *argv)
673669
tbaa_decorate(tbaa_data, store);
674670
}
675671
else if (!jl_isbits(ety)) {
676-
if (!jl_is_structtype(ety) || jl_is_array_type(ety) || !jl_is_concrete_type(ety)) {
677-
emit_error(ctx, "pointerset: invalid pointer type");
678-
return jl_cgval_t();
679-
}
680672
thePtr = emit_unbox(ctx, T_pint8, e, e.typ);
681673
uint64_t size = jl_datatype_size(ety);
682674
im1 = ctx.builder.CreateMul(im1, ConstantInt::get(T_size,
@@ -696,6 +688,170 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, jl_cgval_t *argv)
696688
return e;
697689
}
698690

691+
static jl_cgval_t emit_atomicfence(jl_codectx_t &ctx, jl_cgval_t *argv)
692+
{
693+
const jl_cgval_t &ord = argv[0];
694+
if (ord.constant && jl_is_symbol(ord.constant)) {
695+
enum jl_memory_order order = jl_get_atomic_order((jl_sym_t*)ord.constant, false, false);
696+
if (order == jl_memory_order_invalid) {
697+
emit_atomic_error(ctx, "invalid atomic ordering");
698+
return jl_cgval_t(); // unreachable
699+
}
700+
if (order > jl_memory_order_monotonic)
701+
ctx.builder.CreateFence(get_llvm_atomic_order(order));
702+
return ghostValue(jl_nothing_type);
703+
}
704+
return emit_runtime_call(ctx, atomic_fence, argv, 1);
705+
}
706+
707+
static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv)
708+
{
709+
const jl_cgval_t &e = argv[0];
710+
const jl_cgval_t &ord = argv[1];
711+
jl_value_t *aty = e.typ;
712+
if (!jl_is_cpointer_type(aty) || !ord.constant || !jl_is_symbol(ord.constant))
713+
return emit_runtime_call(ctx, atomic_pointerref, argv, 2);
714+
jl_value_t *ety = jl_tparam0(aty);
715+
if (jl_is_typevar(ety))
716+
return emit_runtime_call(ctx, atomic_pointerref, argv, 2);
717+
enum jl_memory_order order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false);
718+
if (order == jl_memory_order_invalid) {
719+
emit_atomic_error(ctx, "invalid atomic ordering");
720+
return jl_cgval_t(); // unreachable
721+
}
722+
AtomicOrdering llvm_order = get_llvm_atomic_order(order);
723+
724+
if (ety == (jl_value_t*)jl_any_type) {
725+
Value *thePtr = emit_unbox(ctx, T_pprjlvalue, e, e.typ);
726+
LoadInst *load = ctx.builder.CreateAlignedLoad(thePtr, Align(sizeof(jl_value_t*)));
727+
tbaa_decorate(tbaa_data, load);
728+
load->setOrdering(llvm_order);
729+
return mark_julia_type(ctx, load, true, ety);
730+
}
731+
732+
if (!is_valid_intrinsic_elptr(ety)) {
733+
emit_error(ctx, "atomic_pointerref: invalid pointer type");
734+
return jl_cgval_t();
735+
}
736+
737+
size_t nb = jl_datatype_size(ety);
738+
if ((nb & (nb - 1)) != 0 || nb > MAX_POINTERATOMIC_SIZE) {
739+
emit_error(ctx, "atomic_pointerref: invalid pointer for atomic operation");
740+
return jl_cgval_t();
741+
}
742+
743+
if (!jl_isbits(ety)) {
744+
assert(jl_is_datatype(ety));
745+
uint64_t size = jl_datatype_size(ety);
746+
Value *strct = emit_allocobj(ctx, size,
747+
literal_pointer_val(ctx, ety));
748+
Value *thePtr = emit_unbox(ctx, T_pint8, e, e.typ);
749+
Type *loadT = Type::getIntNTy(jl_LLVMContext, nb * 8);
750+
thePtr = emit_bitcast(ctx, thePtr, loadT->getPointerTo());
751+
MDNode *tbaa = best_tbaa(ety);
752+
LoadInst *load = ctx.builder.CreateAlignedLoad(loadT, thePtr, Align(nb));
753+
tbaa_decorate(tbaa, load);
754+
load->setOrdering(llvm_order);
755+
thePtr = emit_bitcast(ctx, strct, thePtr->getType());
756+
StoreInst *store = ctx.builder.CreateAlignedStore(load, thePtr, Align(julia_alignment(ety)));
757+
tbaa_decorate(tbaa, store);
758+
return mark_julia_type(ctx, strct, true, ety);
759+
}
760+
else {
761+
bool isboxed;
762+
Type *ptrty = julia_type_to_llvm(ctx, ety, &isboxed);
763+
assert(!isboxed);
764+
if (!type_is_ghost(ptrty)) {
765+
Value *thePtr = emit_unbox(ctx, ptrty->getPointerTo(), e, e.typ);
766+
return typed_load(ctx, thePtr, nullptr, ety, tbaa_data, nullptr, isboxed, llvm_order, true, nb);
767+
}
768+
else {
769+
if (order > jl_memory_order_monotonic)
770+
ctx.builder.CreateFence(llvm_order);
771+
return ghostValue(ety);
772+
}
773+
}
774+
}
775+
776+
// e[i] = x (set)
777+
// e[i] <= x (swap)
778+
// e[i] y => x (replace)
779+
static jl_cgval_t emit_atomic_pointerset(jl_codectx_t &ctx, intrinsic f, const jl_cgval_t *argv, int nargs)
780+
{
781+
bool issetfield = f == atomic_pointerset;
782+
bool isreplacefield = f == atomic_pointerreplace;
783+
const jl_cgval_t undefval;
784+
const jl_cgval_t &e = argv[0];
785+
const jl_cgval_t &x = isreplacefield ? argv[2] : argv[1];
786+
const jl_cgval_t &y = isreplacefield ? argv[1] : undefval;
787+
const jl_cgval_t &ord = isreplacefield ? argv[3] : argv[2];
788+
const jl_cgval_t &failord = isreplacefield ? argv[4] : undefval;
789+
790+
jl_value_t *aty = e.typ;
791+
if (!jl_is_cpointer_type(aty) || !ord.constant || !jl_is_symbol(ord.constant))
792+
return emit_runtime_call(ctx, f, argv, nargs);
793+
if (isreplacefield) {
794+
if (!failord.constant || !jl_is_symbol(failord.constant))
795+
return emit_runtime_call(ctx, f, argv, nargs);
796+
}
797+
jl_value_t *ety = jl_tparam0(aty);
798+
if (jl_is_typevar(ety))
799+
return emit_runtime_call(ctx, f, argv, nargs);
800+
enum jl_memory_order order = jl_get_atomic_order((jl_sym_t*)ord.constant, !issetfield, true);
801+
enum jl_memory_order failorder = isreplacefield ? jl_get_atomic_order((jl_sym_t*)failord.constant, true, false) : order;
802+
if (order == jl_memory_order_invalid || failorder == jl_memory_order_invalid || failorder > order) {
803+
emit_atomic_error(ctx, "invalid atomic ordering");
804+
return jl_cgval_t(); // unreachable
805+
}
806+
AtomicOrdering llvm_order = get_llvm_atomic_order(order);
807+
AtomicOrdering llvm_failorder = get_llvm_atomic_order(failorder);
808+
809+
if (ety == (jl_value_t*)jl_any_type) {
810+
// unsafe_store to Ptr{Any} is allowed to implicitly drop GC roots.
811+
// n.b.: the expected value (y) must be rooted, but not the others
812+
Value *thePtr = emit_unbox(ctx, T_pprjlvalue, e, e.typ);
813+
bool isboxed = true;
814+
jl_cgval_t ret = typed_store(ctx, thePtr, nullptr, x, y, ety, tbaa_data, nullptr, nullptr, isboxed,
815+
llvm_order, llvm_failorder, sizeof(jl_value_t*), false, issetfield, isreplacefield, false);
816+
if (issetfield)
817+
ret = e;
818+
return ret;
819+
}
820+
821+
if (!is_valid_intrinsic_elptr(ety)) {
822+
std::string msg(StringRef(jl_intrinsic_name((int)f)));
823+
msg += ": invalid pointer type";
824+
emit_error(ctx, msg);
825+
return jl_cgval_t();
826+
}
827+
emit_typecheck(ctx, x, ety, std::string(jl_intrinsic_name((int)f)));
828+
829+
size_t nb = jl_datatype_size(ety);
830+
if ((nb & (nb - 1)) != 0 || nb > MAX_POINTERATOMIC_SIZE) {
831+
std::string msg(StringRef(jl_intrinsic_name((int)f)));
832+
msg += ": invalid pointer for atomic operation";
833+
emit_error(ctx, msg);
834+
return jl_cgval_t();
835+
}
836+
837+
if (!jl_isbits(ety)) {
838+
//Value *thePtr = emit_unbox(ctx, T_pint8, e, e.typ);
839+
//uint64_t size = jl_datatype_size(ety);
840+
return emit_runtime_call(ctx, f, argv, nargs); // TODO: optimizations
841+
}
842+
else {
843+
bool isboxed;
844+
Type *ptrty = julia_type_to_llvm(ctx, ety, &isboxed);
845+
assert(!isboxed);
846+
Value *thePtr = emit_unbox(ctx, ptrty->getPointerTo(), e, e.typ);
847+
jl_cgval_t ret = typed_store(ctx, thePtr, nullptr, x, y, ety, tbaa_data, nullptr, nullptr, isboxed,
848+
llvm_order, llvm_failorder, nb, false, issetfield, isreplacefield, false);
849+
if (issetfield)
850+
ret = e;
851+
return ret;
852+
}
853+
}
854+
699855
static Value *emit_checked_srem_int(jl_codectx_t &ctx, Value *x, Value *den)
700856
{
701857
Type *t = den->getType();
@@ -924,11 +1080,14 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar
9241080
case pointerset:
9251081
return emit_pointerset(ctx, argv);
9261082
case atomic_fence:
1083+
return emit_atomicfence(ctx, argv);
9271084
case atomic_pointerref:
1085+
return emit_atomic_pointerref(ctx, argv);
9281086
case atomic_pointerset:
9291087
case atomic_pointerswap:
930-
case atomic_pointermodify:
9311088
case atomic_pointerreplace:
1089+
return emit_atomic_pointerset(ctx, f, argv, nargs);
1090+
case atomic_pointermodify:
9321091
return emit_runtime_call(ctx, f, argv, nargs);
9331092
case bitcast:
9341093
return generic_bitcast(ctx, argv);

‎src/julia_internal.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1128,6 +1128,10 @@ extern JL_DLLEXPORT jl_value_t *jl_segv_exception;
11281128
JL_DLLEXPORT const char *jl_intrinsic_name(int f) JL_NOTSAFEPOINT;
11291129
unsigned jl_intrinsic_nargs(int f) JL_NOTSAFEPOINT;
11301130

1131+
STATIC_INLINE int is_valid_intrinsic_elptr(jl_value_t *ety)
1132+
{
1133+
return ety == (jl_value_t*)jl_any_type || (jl_is_concrete_type(ety) && !jl_is_layout_opaque(((jl_datatype_t*)ety)->layout));
1134+
}
11311135
JL_DLLEXPORT jl_value_t *jl_bitcast(jl_value_t *ty, jl_value_t *v);
11321136
JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i, jl_value_t *align);
11331137
JL_DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *align, jl_value_t *i);

0 commit comments

Comments
 (0)