From 914c502133026cd7b8d6ac726308f8805e3efdef Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 25 Sep 2025 15:21:14 +0200 Subject: [PATCH 1/2] [IR] Introduce !captures metadata This introduces `!captures` metadata on stores, which looks like this: ``` store ptr %x, ptr %y, !captures !{!"address", !"read_provenance"} ``` The semantics are equivalent to replacing the store with a call like this: ``` call void @llvm.store(ptr captures(address, read_provenance) %x, ptr %y) ``` This metadata is intended for annotation by frontends -- it's not something we can feasibly infer at this pointer, as it would require analyzing uses of the pointer stored in memory. The motivating use case for this is Rust's `println!()` machinery, which involves storing a reference to the value inside a structure. This means that printing code (including conditional debugging code), can inhibit optimizations because the pointer escapes. With the new metadata we could annotate this as a read-only capture, which would have less impact on optimizations. --- llvm/docs/LangRef.rst | 29 ++++ llvm/include/llvm/IR/FixedMetadataKinds.def | 1 + llvm/include/llvm/IR/Instructions.h | 3 + llvm/lib/Analysis/CaptureTracking.cpp | 5 +- llvm/lib/IR/Instructions.cpp | 18 +++ llvm/lib/IR/Verifier.cpp | 25 ++++ llvm/lib/Transforms/Utils/Local.cpp | 3 + .../Transforms/FunctionAttrs/nocapture.ll | 68 +++++++++ .../SimplifyCFG/hoist-with-metadata.ll | 132 ++++++++++++++++++ llvm/test/Verifier/captures-metadata.ll | 37 +++++ 10 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Verifier/captures-metadata.ll diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 8e863939781a2..22b58bf0f5735 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -1489,6 +1489,8 @@ Currently, only the following parameter attributes are defined: function, returning a pointer to allocated storage disjoint from the storage for any other object accessible to the caller. +.. _captures_attr: + ``captures(...)`` This attribute restricts the ways in which the callee may capture the pointer. This is not a valid attribute for return values. This attribute @@ -7543,6 +7545,33 @@ The number of bytes known to be dereferenceable is specified by the integer value in the metadata node. This is analogous to the ''dereferenceable_or_null'' attribute on parameters and return values. +'``captures``' Metadata +^^^^^^^^^^^^^^^^^^^^^^^ + +The ``!captures`` metadata can only be applied to ``store`` instructions with +a pointer-typed value operand. It restricts the capturing behavior of the store +value operand in the same way the ``captures(...)`` attribute would do on a +call. See the :ref:`pointer capture section ` for a detailed +discussion of capture semantics. + +The ``!captures`` metadata accepts a non-empty list of strings from the same +set as the :ref:`captures attribute `: +``!"address"``, ``!"address_is_null"``, ``!"provenance"`` and +``!"read_provenance"``. ``!"none"`` is not supported. + +For example ``store ptr %x, ptr %y, !captures !{!"address"}`` indicates that +the copy of pointer ``%x`` stored to location ``%y`` will only be used to +inspect its integral address value, and not dereferenced. Dereferencing the +pointer would result in undefined behavior. + +Similarly ``store ptr %x, ptr %y, !captures !{!"address", !"read_provenance"}`` +indicates that while reads through the stored pointer are allowed, writes would +result in undefined behavior. + +The ``!captures`` attribute makes no statement about other uses of ``%x``, or +uses of the stored-to memory location after it has been overwritten with a +different value. + .. _llvm.loop: '``llvm.loop``' diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def index d09cc15d65ff6..0603abcd6a4da 100644 --- a/llvm/include/llvm/IR/FixedMetadataKinds.def +++ b/llvm/include/llvm/IR/FixedMetadataKinds.def @@ -55,3 +55,4 @@ LLVM_FIXED_MD_KIND(MD_mmra, "mmra", 40) LLVM_FIXED_MD_KIND(MD_noalias_addrspace, "noalias.addrspace", 41) LLVM_FIXED_MD_KIND(MD_callee_type, "callee_type", 42) LLVM_FIXED_MD_KIND(MD_nofree, "nofree", 43) +LLVM_FIXED_MD_KIND(MD_captures, "captures", 44) diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h index 95a0a7fd2f97e..43e9b7b338589 100644 --- a/llvm/include/llvm/IR/Instructions.h +++ b/llvm/include/llvm/IR/Instructions.h @@ -393,6 +393,9 @@ class StoreInst : public Instruction { return getPointerOperandType()->getPointerAddressSpace(); } + /// Get capturing behavior of the value operand, based on !captures metadata. + CaptureComponents getCaptureComponents() const; + // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Store; diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp index a0fe7f9037e47..0d4ed2dd4f2a7 100644 --- a/llvm/lib/Analysis/CaptureTracking.cpp +++ b/llvm/lib/Analysis/CaptureTracking.cpp @@ -320,8 +320,11 @@ UseCaptureInfo llvm::DetermineUseCaptureKind(const Use &U, const Value *Base) { return CaptureComponents::None; case Instruction::Store: // Stored the pointer - conservatively assume it may be captured. + if (U.getOperandNo() == 0) + return cast(I)->getCaptureComponents(); + // Volatile stores make the address observable. - if (U.getOperandNo() == 0 || cast(I)->isVolatile()) + if (cast(I)->isVolatile()) return CaptureComponents::All; return CaptureComponents::None; case Instruction::AtomicRMW: { diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index dd83168ab3c6e..4e1b01bf6826c 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -1390,6 +1390,24 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Align Align, AssertOK(); } +CaptureComponents StoreInst::getCaptureComponents() const { + const MDNode *MD = getMetadata(LLVMContext::MD_captures); + if (!MD) + return CaptureComponents::All; + + CaptureComponents CC = CaptureComponents::None; + for (Metadata *Op : MD->operands()) { + CaptureComponents Component = + StringSwitch(cast(Op)->getString()) + .Case("address", CaptureComponents::Address) + .Case("address_is_null", CaptureComponents::AddressIsNull) + .Case("provenance", CaptureComponents::Provenance) + .Case("read_provenance", CaptureComponents::ReadProvenance); + CC |= Component; + } + return CC; +} + //===----------------------------------------------------------------------===// // AtomicCmpXchgInst Implementation //===----------------------------------------------------------------------===// diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 8c03d6f809d50..6b3cd27b77a7a 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -542,6 +542,7 @@ class Verifier : public InstVisitor, VerifierSupport { void visitAliasScopeMetadata(const MDNode *MD); void visitAliasScopeListMetadata(const MDNode *MD); void visitAccessGroupMetadata(const MDNode *MD); + void visitCapturesMetadata(Instruction &I, const MDNode *Captures); template bool isValidMetadataArray(const MDTuple &N); #define HANDLE_SPECIALIZED_MDNODE_LEAF(CLASS) void visit##CLASS(const CLASS &N); @@ -5373,6 +5374,27 @@ void Verifier::visitAccessGroupMetadata(const MDNode *MD) { } } +void Verifier::visitCapturesMetadata(Instruction &I, const MDNode *Captures) { + static const char *ValidArgs[] = {"address_is_null", "address", + "read_provenance", "provenance"}; + + auto *SI = dyn_cast(&I); + Check(SI, "!captures metadata can only be applied to store instructions", &I); + Check(SI->getValueOperand()->getType()->isPointerTy(), + "!captures metadata can only be applied to store with value operand of " + "pointer type", + &I); + Check(Captures->getNumOperands() != 0, "!captures metadata cannot be empty", + &I); + + for (Metadata *Op : Captures->operands()) { + auto *Str = dyn_cast(Op); + Check(Str, "!captures metadata must be a list of strings", &I); + Check(is_contained(ValidArgs, Str->getString()), + "invalid entry in !captures metadata", &I, Str); + } +} + /// verifyInstruction - Verify that an instruction is well formed. /// void Verifier::visitInstruction(Instruction &I) { @@ -5600,6 +5622,9 @@ void Verifier::visitInstruction(Instruction &I) { if (MDNode *Annotation = I.getMetadata(LLVMContext::MD_annotation)) visitAnnotationMetadata(Annotation); + if (MDNode *Captures = I.getMetadata(LLVMContext::MD_captures)) + visitCapturesMetadata(I, Captures); + if (MDNode *N = I.getDebugLoc().getAsMDNode()) { CheckDI(isa(N), "invalid !dbg metadata attachment", &I, N); visitMDNode(*N, AreDebugLocsAllowed::Yes); diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index 123881e276584..c1ae0e4a19b41 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -3025,6 +3025,9 @@ static void combineMetadata(Instruction *K, const Instruction *J, // Preserve !nosanitize if both K and J have it. K->setMetadata(Kind, JMD); break; + case LLVMContext::MD_captures: + K->setMetadata(Kind, JMD ? MDNode::concatenate(JMD, KMD) : nullptr); + break; } } // Set !invariant.group from J if J has it. If both instructions have it diff --git a/llvm/test/Transforms/FunctionAttrs/nocapture.ll b/llvm/test/Transforms/FunctionAttrs/nocapture.ll index 60a4214548a72..8113ba65fe422 100644 --- a/llvm/test/Transforms/FunctionAttrs/nocapture.ll +++ b/llvm/test/Transforms/FunctionAttrs/nocapture.ll @@ -1398,5 +1398,73 @@ define void @assume_nonnull(ptr %p) { ret void } +define void @captures_metadata_address_is_null(ptr %x, ptr %y) { +; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write) +; FNATTRS-LABEL: define void @captures_metadata_address_is_null +; FNATTRS-SAME: (ptr captures(address_is_null) [[X:%.*]], ptr writeonly captures(none) initializes((0, 8)) [[Y:%.*]]) #[[ATTR17]] { +; FNATTRS-NEXT: store ptr [[X]], ptr [[Y]], align 8, !captures [[META0:![0-9]+]] +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write) +; ATTRIBUTOR-LABEL: define void @captures_metadata_address_is_null +; ATTRIBUTOR-SAME: (ptr nofree writeonly [[X:%.*]], ptr nofree nonnull writeonly captures(none) [[Y:%.*]]) #[[ATTR13]] { +; ATTRIBUTOR-NEXT: store ptr [[X]], ptr [[Y]], align 8, !captures [[META0:![0-9]+]] +; ATTRIBUTOR-NEXT: ret void +; + store ptr %x, ptr %y, !captures !{!"address_is_null"} + ret void +} + +define void @captures_metadata_address(ptr %x, ptr %y) { +; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write) +; FNATTRS-LABEL: define void @captures_metadata_address +; FNATTRS-SAME: (ptr captures(address) [[X:%.*]], ptr writeonly captures(none) initializes((0, 8)) [[Y:%.*]]) #[[ATTR17]] { +; FNATTRS-NEXT: store ptr [[X]], ptr [[Y]], align 8, !captures [[META1:![0-9]+]] +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write) +; ATTRIBUTOR-LABEL: define void @captures_metadata_address +; ATTRIBUTOR-SAME: (ptr nofree writeonly [[X:%.*]], ptr nofree nonnull writeonly captures(none) [[Y:%.*]]) #[[ATTR13]] { +; ATTRIBUTOR-NEXT: store ptr [[X]], ptr [[Y]], align 8, !captures [[META1:![0-9]+]] +; ATTRIBUTOR-NEXT: ret void +; + store ptr %x, ptr %y, !captures !{!"address"} + ret void +} + +define void @captures_metadata_address_read_provenance(ptr %x, ptr %y) { +; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write) +; FNATTRS-LABEL: define void @captures_metadata_address_read_provenance +; FNATTRS-SAME: (ptr captures(address, read_provenance) [[X:%.*]], ptr writeonly captures(none) initializes((0, 8)) [[Y:%.*]]) #[[ATTR17]] { +; FNATTRS-NEXT: store ptr [[X]], ptr [[Y]], align 8, !captures [[META2:![0-9]+]] +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write) +; ATTRIBUTOR-LABEL: define void @captures_metadata_address_read_provenance +; ATTRIBUTOR-SAME: (ptr nofree writeonly [[X:%.*]], ptr nofree nonnull writeonly captures(none) [[Y:%.*]]) #[[ATTR13]] { +; ATTRIBUTOR-NEXT: store ptr [[X]], ptr [[Y]], align 8, !captures [[META2:![0-9]+]] +; ATTRIBUTOR-NEXT: ret void +; + store ptr %x, ptr %y, !captures !{!"address", !"read_provenance"} + ret void +} + +define void @captures_metadata_provenance(ptr %x, ptr %y) { +; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write) +; FNATTRS-LABEL: define void @captures_metadata_provenance +; FNATTRS-SAME: (ptr captures(provenance) [[X:%.*]], ptr writeonly captures(none) initializes((0, 8)) [[Y:%.*]]) #[[ATTR17]] { +; FNATTRS-NEXT: store ptr [[X]], ptr [[Y]], align 8, !captures [[META3:![0-9]+]] +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write) +; ATTRIBUTOR-LABEL: define void @captures_metadata_provenance +; ATTRIBUTOR-SAME: (ptr nofree writeonly [[X:%.*]], ptr nofree nonnull writeonly captures(none) [[Y:%.*]]) #[[ATTR13]] { +; ATTRIBUTOR-NEXT: store ptr [[X]], ptr [[Y]], align 8, !captures [[META3:![0-9]+]] +; ATTRIBUTOR-NEXT: ret void +; + store ptr %x, ptr %y, !captures !{!"provenance"} + ret void +} + declare ptr @llvm.launder.invariant.group.p0(ptr) declare ptr @llvm.strip.invariant.group.p0(ptr) diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-with-metadata.ll b/llvm/test/Transforms/SimplifyCFG/hoist-with-metadata.ll index d34ac2bb30040..cba0251d1e18d 100644 --- a/llvm/test/Transforms/SimplifyCFG/hoist-with-metadata.ll +++ b/llvm/test/Transforms/SimplifyCFG/hoist-with-metadata.ll @@ -424,6 +424,134 @@ join: ret ptr %phi } +define void @hoist_captures_same(i1 %c, ptr %x, ptr %y) { +; CHECK-LABEL: @hoist_captures_same( +; CHECK-NEXT: if: +; CHECK-NEXT: store ptr [[X:%.*]], ptr [[Y:%.*]], align 8, !captures [[META9:![0-9]+]] +; CHECK-NEXT: ret void +; +if: + br i1 %c, label %then, label %else + +then: + store ptr %x, ptr %y, !captures !{!"address"} + br label %out + +else: + store ptr %x, ptr %y, !captures !{!"address"} + br label %out + +out: + ret void +} + +define void @hoist_captures_different(i1 %c, ptr %x, ptr %y) { +; CHECK-LABEL: @hoist_captures_different( +; CHECK-NEXT: if: +; CHECK-NEXT: store ptr [[X:%.*]], ptr [[Y:%.*]], align 8, !captures [[META10:![0-9]+]] +; CHECK-NEXT: ret void +; +if: + br i1 %c, label %then, label %else + +then: + store ptr %x, ptr %y, !captures !{!"address"} + br label %out + +else: + store ptr %x, ptr %y, !captures !{!"read_provenance"} + br label %out + +out: + ret void +} + +define void @hoist_captures_overlap(i1 %c, ptr %x, ptr %y) { +; CHECK-LABEL: @hoist_captures_overlap( +; CHECK-NEXT: if: +; CHECK-NEXT: store ptr [[X:%.*]], ptr [[Y:%.*]], align 8, !captures [[META11:![0-9]+]] +; CHECK-NEXT: ret void +; +if: + br i1 %c, label %then, label %else + +then: + store ptr %x, ptr %y, !captures !{!"address"} + br label %out + +else: + store ptr %x, ptr %y, !captures !{!"address", !"read_provenance"} + br label %out + +out: + ret void +} + +; We could also omit the attribute in this case, as it provides no additional +; information. +define void @hoist_captures_full_set(i1 %c, ptr %x, ptr %y) { +; CHECK-LABEL: @hoist_captures_full_set( +; CHECK-NEXT: if: +; CHECK-NEXT: store ptr [[X:%.*]], ptr [[Y:%.*]], align 8, !captures [[META12:![0-9]+]] +; CHECK-NEXT: ret void +; +if: + br i1 %c, label %then, label %else + +then: + store ptr %x, ptr %y, !captures !{!"address"} + br label %out + +else: + store ptr %x, ptr %y, !captures !{!"provenance"} + br label %out + +out: + ret void +} + +define void @hoist_captures_only_one1(i1 %c, ptr %x, ptr %y) { +; CHECK-LABEL: @hoist_captures_only_one1( +; CHECK-NEXT: if: +; CHECK-NEXT: store ptr [[X:%.*]], ptr [[Y:%.*]], align 8 +; CHECK-NEXT: ret void +; +if: + br i1 %c, label %then, label %else + +then: + store ptr %x, ptr %y, !captures !{!"address"} + br label %out + +else: + store ptr %x, ptr %y + br label %out + +out: + ret void +} + +define void @hoist_captures_only_one2(i1 %c, ptr %x, ptr %y) { +; CHECK-LABEL: @hoist_captures_only_one2( +; CHECK-NEXT: if: +; CHECK-NEXT: store ptr [[X:%.*]], ptr [[Y:%.*]], align 8 +; CHECK-NEXT: ret void +; +if: + br i1 %c, label %then, label %else + +then: + store ptr %x, ptr %y + br label %out + +else: + store ptr %x, ptr %y, !captures !{!"address"} + br label %out + +out: + ret void +} + !0 = !{ i8 0, i8 1 } !1 = !{ i8 3, i8 5 } !2 = !{} @@ -445,4 +573,8 @@ join: ; CHECK: [[META6]] = !{float 2.500000e+00} ; CHECK: [[META7]] = !{i32 5, i32 6} ; CHECK: [[META8]] = !{i32 4, i32 5} +; CHECK: [[META9]] = !{!"address"} +; CHECK: [[META10]] = !{!"read_provenance", !"address"} +; CHECK: [[META11]] = !{!"address", !"read_provenance"} +; CHECK: [[META12]] = !{!"provenance", !"address"} ;. diff --git a/llvm/test/Verifier/captures-metadata.ll b/llvm/test/Verifier/captures-metadata.ll new file mode 100644 index 0000000000000..ae08ddd036f16 --- /dev/null +++ b/llvm/test/Verifier/captures-metadata.ll @@ -0,0 +1,37 @@ +; RUN: not opt -passes=verify < %s 2>&1 | FileCheck %s + +; CHECK: !captures metadata can only be applied to store instructions +define void @wrong_instr_type(ptr %x) { + load ptr, ptr %x, !captures !{!"address"} + ret void +} + +; CHECK: captures metadata can only be applied to store with value operand of pointer type +define void @wrong_op_type(i32 %x, ptr %y) { + store i32 %x, ptr %y, !captures !{!"address"} + ret void +} + +; CHECK: !captures metadata cannot be empty +define void @empty(ptr %x, ptr %y) { + store ptr %x, ptr %y, !captures !{} + ret void +} + +; CHECK: !captures metadata must be a list of strings +define void @not_string(ptr %x, ptr %y) { + store ptr %x, ptr %y, !captures !{!{}} + ret void +} + +; CHECK: invalid entry in !captures metadata +define void @invalid_str(ptr %x, ptr %y) { + store ptr %x, ptr %y, !captures !{!"foo"} + ret void +} + +; CHECK: invalid entry in !captures metadata +define void @invalid_none(ptr %x, ptr %y) { + store ptr %x, ptr %y, !captures !{!"none"} + ret void +} From 36fe181849bc4b5d9bc9af06958df6374327b6c6 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 30 Sep 2025 12:57:46 +0200 Subject: [PATCH 2/2] Always produce canonical form when combining metadata --- llvm/include/llvm/IR/Instructions.h | 3 -- llvm/include/llvm/IR/Metadata.h | 8 +++ llvm/lib/Analysis/CaptureTracking.cpp | 3 +- llvm/lib/IR/Instructions.cpp | 18 ------- llvm/lib/IR/Metadata.cpp | 35 ++++++++++++ llvm/lib/Transforms/Utils/Local.cpp | 5 +- .../SimplifyCFG/hoist-with-metadata.ll | 53 ++++++++++++++++--- 7 files changed, 95 insertions(+), 30 deletions(-) diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h index 43e9b7b338589..95a0a7fd2f97e 100644 --- a/llvm/include/llvm/IR/Instructions.h +++ b/llvm/include/llvm/IR/Instructions.h @@ -393,9 +393,6 @@ class StoreInst : public Instruction { return getPointerOperandType()->getPointerAddressSpace(); } - /// Get capturing behavior of the value operand, based on !captures metadata. - CaptureComponents getCaptureComponents() const; - // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Store; diff --git a/llvm/include/llvm/IR/Metadata.h b/llvm/include/llvm/IR/Metadata.h index 990bdc618f240..85a7f8fd373c0 100644 --- a/llvm/include/llvm/IR/Metadata.h +++ b/llvm/include/llvm/IR/Metadata.h @@ -41,6 +41,7 @@ namespace llvm { +enum class CaptureComponents : uint8_t; class Module; class ModuleSlotTracker; class raw_ostream; @@ -1480,6 +1481,13 @@ class MDNode : public Metadata { LLVM_ABI static MDNode *getMergedCallsiteMetadata(MDNode *A, MDNode *B); LLVM_ABI static MDNode *getMergedCalleeTypeMetadata(const MDNode *A, const MDNode *B); + + /// Convert !captures metadata to CaptureComponents. MD may be nullptr. + LLVM_ABI static CaptureComponents toCaptureComponents(const MDNode *MD); + /// Convert CaptureComponents to !captures metadata. The return value may be + /// nullptr. + LLVM_ABI static MDNode *fromCaptureComponents(LLVMContext &Ctx, + CaptureComponents CC); }; /// Tuple of metadata. diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp index 0d4ed2dd4f2a7..22229d9c26b3b 100644 --- a/llvm/lib/Analysis/CaptureTracking.cpp +++ b/llvm/lib/Analysis/CaptureTracking.cpp @@ -321,7 +321,8 @@ UseCaptureInfo llvm::DetermineUseCaptureKind(const Use &U, const Value *Base) { case Instruction::Store: // Stored the pointer - conservatively assume it may be captured. if (U.getOperandNo() == 0) - return cast(I)->getCaptureComponents(); + return MDNode::toCaptureComponents( + I->getMetadata(LLVMContext::MD_captures)); // Volatile stores make the address observable. if (cast(I)->isVolatile()) diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index 4e1b01bf6826c..dd83168ab3c6e 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -1390,24 +1390,6 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Align Align, AssertOK(); } -CaptureComponents StoreInst::getCaptureComponents() const { - const MDNode *MD = getMetadata(LLVMContext::MD_captures); - if (!MD) - return CaptureComponents::All; - - CaptureComponents CC = CaptureComponents::None; - for (Metadata *Op : MD->operands()) { - CaptureComponents Component = - StringSwitch(cast(Op)->getString()) - .Case("address", CaptureComponents::Address) - .Case("address_is_null", CaptureComponents::AddressIsNull) - .Case("provenance", CaptureComponents::Provenance) - .Case("read_provenance", CaptureComponents::ReadProvenance); - CC |= Component; - } - return CC; -} - //===----------------------------------------------------------------------===// // AtomicCmpXchgInst Implementation //===----------------------------------------------------------------------===// diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp index 9cfb0ff4d689a..1add0c7930bc9 100644 --- a/llvm/lib/IR/Metadata.cpp +++ b/llvm/lib/IR/Metadata.cpp @@ -48,6 +48,7 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/ModRef.h" #include #include #include @@ -1435,6 +1436,40 @@ MDNode *MDNode::getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B) { return B; } +CaptureComponents MDNode::toCaptureComponents(const MDNode *MD) { + if (!MD) + return CaptureComponents::All; + + CaptureComponents CC = CaptureComponents::None; + for (Metadata *Op : MD->operands()) { + CaptureComponents Component = + StringSwitch(cast(Op)->getString()) + .Case("address", CaptureComponents::Address) + .Case("address_is_null", CaptureComponents::AddressIsNull) + .Case("provenance", CaptureComponents::Provenance) + .Case("read_provenance", CaptureComponents::ReadProvenance); + CC |= Component; + } + return CC; +} + +MDNode *MDNode::fromCaptureComponents(LLVMContext &Ctx, CaptureComponents CC) { + assert(!capturesNothing(CC) && "Can't encode captures(none)"); + if (capturesAll(CC)) + return nullptr; + + SmallVector Components; + if (capturesAddressIsNullOnly(CC)) + Components.push_back(MDString::get(Ctx, "address_is_null")); + else if (capturesAddress(CC)) + Components.push_back(MDString::get(Ctx, "address")); + if (capturesReadProvenanceOnly(CC)) + Components.push_back(MDString::get(Ctx, "read_provenance")); + else if (capturesFullProvenance(CC)) + Components.push_back(MDString::get(Ctx, "provenance")); + return MDNode::get(Ctx, Components); +} + //===----------------------------------------------------------------------===// // NamedMDNode implementation. // diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index c1ae0e4a19b41..21b2652d04120 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -3026,7 +3026,10 @@ static void combineMetadata(Instruction *K, const Instruction *J, K->setMetadata(Kind, JMD); break; case LLVMContext::MD_captures: - K->setMetadata(Kind, JMD ? MDNode::concatenate(JMD, KMD) : nullptr); + K->setMetadata( + Kind, MDNode::fromCaptureComponents( + K->getContext(), MDNode::toCaptureComponents(JMD) | + MDNode::toCaptureComponents(KMD))); break; } } diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-with-metadata.ll b/llvm/test/Transforms/SimplifyCFG/hoist-with-metadata.ll index cba0251d1e18d..85c8ed20210b8 100644 --- a/llvm/test/Transforms/SimplifyCFG/hoist-with-metadata.ll +++ b/llvm/test/Transforms/SimplifyCFG/hoist-with-metadata.ll @@ -469,7 +469,7 @@ out: define void @hoist_captures_overlap(i1 %c, ptr %x, ptr %y) { ; CHECK-LABEL: @hoist_captures_overlap( ; CHECK-NEXT: if: -; CHECK-NEXT: store ptr [[X:%.*]], ptr [[Y:%.*]], align 8, !captures [[META11:![0-9]+]] +; CHECK-NEXT: store ptr [[X:%.*]], ptr [[Y:%.*]], align 8, !captures [[META10]] ; CHECK-NEXT: ret void ; if: @@ -487,12 +487,52 @@ out: ret void } -; We could also omit the attribute in this case, as it provides no additional -; information. +define void @hoist_captures_subsume1(i1 %c, ptr %x, ptr %y) { +; CHECK-LABEL: @hoist_captures_subsume1( +; CHECK-NEXT: if: +; CHECK-NEXT: store ptr [[X:%.*]], ptr [[Y:%.*]], align 8, !captures [[META9]] +; CHECK-NEXT: ret void +; +if: + br i1 %c, label %then, label %else + +then: + store ptr %x, ptr %y, !captures !{!"address_is_null"} + br label %out + +else: + store ptr %x, ptr %y, !captures !{!"address"} + br label %out + +out: + ret void +} + +define void @hoist_captures_subsume2(i1 %c, ptr %x, ptr %y) { +; CHECK-LABEL: @hoist_captures_subsume2( +; CHECK-NEXT: if: +; CHECK-NEXT: store ptr [[X:%.*]], ptr [[Y:%.*]], align 8, !captures [[META11:![0-9]+]] +; CHECK-NEXT: ret void +; +if: + br i1 %c, label %then, label %else + +then: + store ptr %x, ptr %y, !captures !{!"provenance"} + br label %out + +else: + store ptr %x, ptr %y, !captures !{!"read_provenance"} + br label %out + +out: + ret void +} + define void @hoist_captures_full_set(i1 %c, ptr %x, ptr %y) { ; CHECK-LABEL: @hoist_captures_full_set( ; CHECK-NEXT: if: -; CHECK-NEXT: store ptr [[X:%.*]], ptr [[Y:%.*]], align 8, !captures [[META12:![0-9]+]] +; CHECK-NEXT: store ptr [[X:%.*]], ptr [[Y:%.*]], align 8 ; CHECK-NEXT: ret void ; if: @@ -574,7 +614,6 @@ out: ; CHECK: [[META7]] = !{i32 5, i32 6} ; CHECK: [[META8]] = !{i32 4, i32 5} ; CHECK: [[META9]] = !{!"address"} -; CHECK: [[META10]] = !{!"read_provenance", !"address"} -; CHECK: [[META11]] = !{!"address", !"read_provenance"} -; CHECK: [[META12]] = !{!"provenance", !"address"} +; CHECK: [[META10]] = !{!"address", !"read_provenance"} +; CHECK: [[META11]] = !{!"provenance"} ;.