[SYCL] break out offload-wrapper step to allow usage for all sycl devices.

Address some review issues from the previous commit, and fix problem seen
after latest pulldown with -fsycl

Signed-off-by: Vladimir Lazarev <[email protected]>

Author: vladimirlaz

clang/include/clang/Driver/Action.h

@@ -72,9 +72,10 @@ class Action {
     VerifyPCHJobClass,
     OffloadBundlingJobClass,
     OffloadUnbundlingJobClass,
+    OffloadWrappingJobClass,
 
     JobClassFirst = PreprocessJobClass,
-    JobClassLast = OffloadUnbundlingJobClass
+    JobClassLast = OffloadWrappingJobClass
   };
 
   // The offloading kind determines if this action is binded to a particular
@@ -615,6 +616,17 @@ class OffloadUnbundlingJobAction final : public JobAction {
   }
 };
 
+class OffloadWrappingJobAction : public JobAction {
+  void anchor() override;
+
+public:
+  OffloadWrappingJobAction(Action *Input, types::ID OutputType);
+
+  static bool classof(const Action *A) {
+    return A->getKind() == OffloadWrappingJobClass;
+  }
+};
+
 } // namespace driver
 } // namespace clang
 

clang/include/clang/Driver/ToolChain.h

@@ -130,12 +130,14 @@ class ToolChain {
   mutable std::unique_ptr<Tool> Assemble;
   mutable std::unique_ptr<Tool> Link;
   mutable std::unique_ptr<Tool> OffloadBundler;
+  mutable std::unique_ptr<Tool> OffloadWrapper;
 
   Tool *getClang() const;
   Tool *getAssemble() const;
   Tool *getLink() const;
   Tool *getClangAs() const;
   Tool *getOffloadBundler() const;
+  Tool *getOffloadWrapper() const;
 
   mutable std::unique_ptr<SanitizerArgs> SanitizerArguments;
   mutable std::unique_ptr<XRayArgs> XRayArguments;

clang/lib/Driver/Action.cpp

@@ -41,6 +41,8 @@ const char *Action::getClassName(ActionClass AC) {
     return "clang-offload-bundler";
   case OffloadUnbundlingJobClass:
     return "clang-offload-unbundler";
+  case OffloadWrappingJobClass:
+    return "clang-offload-wrapper";
   }
 
   llvm_unreachable("invalid class");
@@ -408,3 +410,9 @@ void OffloadUnbundlingJobAction::anchor() {}
 
 OffloadUnbundlingJobAction::OffloadUnbundlingJobAction(Action *Input)
     : JobAction(OffloadUnbundlingJobClass, Input, Input->getType()) {}
+
+void OffloadWrappingJobAction::anchor() {}
+
+OffloadWrappingJobAction::OffloadWrappingJobAction(Action *Input,
+                                                   types::ID Type)
+    : JobAction(OffloadWrappingJobClass, Input, Type) {}

clang/lib/Driver/Compilation.cpp

@@ -68,27 +68,27 @@ Compilation::getArgsForToolChain(const ToolChain *TC, StringRef BoundArch,
   DerivedArgList *&Entry = TCArgs[{TC, BoundArch, DeviceOffloadKind}];
   if (!Entry) {
     SmallVector<Arg *, 4> AllocatedArgs;
-    DerivedArgList *OpenMPArgs = nullptr;
-    // Translate OpenMP toolchain arguments provided via the -Xopenmp-target
+    DerivedArgList *OffloadArgs = nullptr;
+    // Translate offload toolchain arguments provided via the -Xopenmp-target
     // or -Xsycl-target flags.
     if (DeviceOffloadKind == Action::OFK_OpenMP ||
         DeviceOffloadKind == Action::OFK_SYCL) {
       const ToolChain *HostTC = getSingleOffloadToolChain<Action::OFK_Host>();
       bool SameTripleAsHost = (TC->getTriple() == HostTC->getTriple());
-      OpenMPArgs = TC->TranslateOffloadTargetArgs(
+      OffloadArgs = TC->TranslateOffloadTargetArgs(
           *TranslatedArgs, SameTripleAsHost, AllocatedArgs, DeviceOffloadKind);
     }
 
-    if (!OpenMPArgs) {
+    if (!OffloadArgs) {
       Entry = TC->TranslateArgs(*TranslatedArgs, BoundArch, DeviceOffloadKind);
       if (!Entry)
         Entry = TranslatedArgs;
     } else {
-      Entry = TC->TranslateArgs(*OpenMPArgs, BoundArch, DeviceOffloadKind);
+      Entry = TC->TranslateArgs(*OffloadArgs, BoundArch, DeviceOffloadKind);
       if (!Entry)
-        Entry = OpenMPArgs;
+        Entry = OffloadArgs;
       else
-        delete OpenMPArgs;
+        delete OffloadArgs;
     }
 
     // Add allocated arguments to the final DAL.

clang/lib/Driver/Driver.cpp

@@ -698,15 +698,15 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
   // SYCL
   //
   // We need to generate a SYCL toolchain if the user specified targets with
-  // the -fsycl-targets option.
+  // the -fsycl-targets option.  If -fsycl is supplied without -fsycl-targets
+  // we will assume SPIR-V
+  bool HasValidSYCLRuntime = C.getInputArgs().hasFlag(options::OPT_fsycl,
+                                              options::OPT_fno_sycl, false);
   if (Arg *SYCLTargets =
           C.getInputArgs().getLastArg(options::OPT_fsycl_targets_EQ)) {
     if (SYCLTargets->getNumValues()) {
       // We expect that -fsycl-targets is always used in conjunction with the
       // -fsycl option
-      bool HasValidSYCLRuntime = C.getInputArgs().hasFlag(
-          options::OPT_fsycl, options::OPT_fno_sycl, false);
-
       if (HasValidSYCLRuntime) {
         llvm::StringMap<const char *> FoundNormalizedTriples;
         for (const char *Val : SYCLTargets->getValues()) {
@@ -747,6 +747,27 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     } else
       Diag(clang::diag::warn_drv_empty_joined_argument)
           << SYCLTargets->getAsString(C.getInputArgs());
+  } else {
+    // If -fsycl is supplied without -fsycl-targets we will assume SPIR-V
+    if (HasValidSYCLRuntime) {
+      const ToolChain *HostTC =
+          C.getSingleOffloadToolChain<Action::OFK_Host>();
+      const llvm::Triple &HostTriple = HostTC->getTriple();
+      llvm::Triple TT(TargetTriple);
+      TT.setArch(llvm::Triple::spir64);
+      TT.setVendor(llvm::Triple::UnknownVendor);
+      TT.setOS(llvm::Triple(llvm::sys::getProcessTriple()).getOS());
+      TT.setEnvironment(llvm::Triple::SYCLDevice);
+      // Use the SYCL and host triples as the key into the ToolChains map,
+      // because the device toolchain we create depends on both.
+      auto &SYCLTC = ToolChains[(TT.normalize() + Twine("/") +
+                                 HostTriple.normalize()).str()];
+      if (!SYCLTC) {
+        SYCLTC = llvm::make_unique<toolchains::SYCLToolChain>(
+            *this, TT, *HostTC, C.getInputArgs());
+      }
+      C.addOffloadDeviceToolChain(SYCLTC.get(), Action::OFK_SYCL);
+    }
   }
 
   //
@@ -2942,7 +2963,7 @@ class OffloadingActionBuilder final {
       // The host depends on the generated integrated header from the device
       // compilation.
       if (CurPhase == phases::Compile) {
-         for (Action *&A : SYCLDeviceActions) {
+        for (Action *&A : SYCLDeviceActions) {
           DeviceCompilerInput =
               C.MakeAction<CompileJobAction>(A, types::TY_SYCL_Header);
         }
@@ -3032,7 +3053,12 @@ class OffloadingActionBuilder final {
       for (auto &LI : DeviceLinkerInputs) {
         auto *DeviceLinkAction =
             C.MakeAction<LinkJobAction>(LI, types::TY_Image);
-        DA.add(*DeviceLinkAction, **TC, /*BoundArch=*/nullptr,
+
+        // After the Link, wrap the files before the final host link
+        auto *DeviceWrappingAction =
+            C.MakeAction<OffloadWrappingJobAction>(DeviceLinkAction,
+                                                   types::TY_Object);
+        DA.add(*DeviceWrappingAction, **TC, /*BoundArch=*/nullptr,
                Action::OFK_SYCL);
         ++TC;
       }

clang/lib/Driver/ToolChain.cpp

@@ -286,6 +286,12 @@ Tool *ToolChain::getOffloadBundler() const {
   return OffloadBundler.get();
 }
 
+Tool *ToolChain::getOffloadWrapper() const {
+  if (!OffloadWrapper)
+    OffloadWrapper.reset(new tools::OffloadWrapper(*this));
+  return OffloadWrapper.get();
+}
+
 Tool *ToolChain::getTool(Action::ActionClass AC) const {
   switch (AC) {
   case Action::AssembleJobClass:
@@ -315,6 +321,9 @@ Tool *ToolChain::getTool(Action::ActionClass AC) const {
   case Action::OffloadBundlingJobClass:
   case Action::OffloadUnbundlingJobClass:
     return getOffloadBundler();
+
+  case Action::OffloadWrappingJobClass:
+    return getOffloadWrapper();
   }
 
   llvm_unreachable("Invalid tool kind.");
@@ -931,7 +940,9 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOffloadTargetArgs(
     unsigned Index;
     unsigned Prev;
     bool XOffloadTargetNoTriple;
-   
+
+    // TODO: functionality between OpenMP offloading and SYCL offloading
+    // is similar, can be improved
     if (DeviceOffloadKind == Action::OFK_OpenMP) {
       XOffloadTargetNoTriple =
         A->getOption().matches(options::OPT_Xopenmp_target);
@@ -966,7 +977,6 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOffloadTargetArgs(
       }
     }
 
-
     // Parse the argument to -Xopenmp-target.
     Prev = Index;
     std::unique_ptr<Arg> XOffloadTargetArg(Opts.ParseOneArg(Args, Index));
@@ -981,6 +991,8 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOffloadTargetArgs(
       continue;
     }
     if (XOffloadTargetNoTriple && XOffloadTargetArg) {
+      // TODO: similar behaviors with OpenMP and SYCL offloading, can be
+      // improved upon
       if (DeviceOffloadKind == Action::OFK_OpenMP &&
           Args.getAllArgValues(options::OPT_fopenmp_targets_EQ).size() != 1) {
         getDriver().Diag(diag::err_drv_Xopenmp_target_missing_triple);

clang/lib/Driver/ToolChains/Clang.cpp

@@ -3412,6 +3412,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   InputInfoList ModuleHeaderInputs;
   const InputInfo *CudaDeviceInput = nullptr;
   const InputInfo *OpenMPDeviceInput = nullptr;
+  const InputInfo *SYCLDeviceInput = nullptr;
   for (const InputInfo &I : Inputs) {
     if (&I == &Input) {
       // This is the primary input.
@@ -3428,6 +3429,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       CudaDeviceInput = &I;
     } else if (IsOpenMPDevice && !OpenMPDeviceInput) {
       OpenMPDeviceInput = &I;
+    } else if (IsSYCL && !SYCLDeviceInput) {
+      SYCLDeviceInput = &I;
     } else {
       llvm_unreachable("unexpectedly given multiple inputs");
     }
@@ -3544,8 +3547,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   } else if (isa<AssembleJobAction>(JA)) {
     if (IsSYCLOffloadDevice && IsSYCLDevice) {
       CmdArgs.push_back("-emit-spirv");
-    }
-    else {
+    } else {
       CmdArgs.push_back("-emit-obj");
       CollectArgsForIntegratedAssembler(C, Args, CmdArgs, D);
     }
@@ -5221,11 +5223,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   if (IsSYCL) {
-    // Host-side SYCL compilation receives the integrated header file as
+    // Host-side SYCL compilation receives the integration header file as
     // Inputs[1].  Include the header with -include
-    if (!IsSYCLOffloadDevice && Inputs.size() > 1) {
+    if (!IsSYCLOffloadDevice && SYCLDeviceInput) {
       CmdArgs.push_back("-include");
-      CmdArgs.push_back(Inputs[1].getFilename());
+      CmdArgs.push_back(SYCLDeviceInput->getFilename());
     }
     if (IsSYCLOffloadDevice && JA.getType() == types::TY_SYCL_Header) {
       // Generating a SYCL Header
@@ -6254,3 +6256,66 @@ void OffloadBundler::ConstructJobMultipleOutputs(
       TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())),
       CmdArgs, None));
 }
+
+// Begin OffloadWrapper
+
+void OffloadWrapper::ConstructJob(Compilation &C, const JobAction &JA,
+                                  const InputInfo &Output,
+                                  const InputInfoList &Inputs,
+                                  const llvm::opt::ArgList &TCArgs,
+                                  const char *LinkingOutput) const {
+  // Construct offload-wrapper command.  Also calls llc to generate the
+  // object that is fed to the linker from the wrapper generated bc file
+  assert(isa<OffloadWrappingJobAction>(JA) && "Expecting wrapping job!");
+
+  // The wrapper command looks like this:
+  // clang-offload-wrapper
+  //   -o=<outputfile>.bc
+  //   -target=sycl-x86_64-pc-linux-gnu <inputfile(s)>.spv
+  ArgStringList WrapperArgs;
+
+  std::string OutTmpName = C.getDriver().GetTemporaryPath("wrapper", "bc");
+  const char * WrapperFileName =
+      C.addTempFile(C.getArgs().MakeArgString(OutTmpName));
+  SmallString<128> OutOpt("-o=");
+  OutOpt += WrapperFileName;
+  WrapperArgs.push_back(C.getArgs().MakeArgString(OutOpt));
+  for (auto I : Inputs) {
+    WrapperArgs.push_back(I.getFilename());
+  }
+
+  SmallString<128> TargetOpt("-target=");
+  TargetOpt += Action::GetOffloadKindName(JA.getOffloadingDeviceKind());
+  TargetOpt += '-';
+  TargetOpt += getToolChain().getAuxTriple()->str();
+  WrapperArgs.push_back(C.getArgs().MakeArgString(TargetOpt));
+
+  // For SYCL, do not emit entry tables
+  if (JA.isOffloading(Action::OFK_SYCL))
+    WrapperArgs.push_back("-emit-entry-table=0");
+
+  C.addCommand(llvm::make_unique<Command>(JA, *this,
+      TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())),
+      WrapperArgs, None));
+
+  // Construct llc command.
+  // The output is an object file
+  ArgStringList LlcArgs{"-filetype=obj", "-o",  Output.getFilename(),
+                        WrapperFileName};
+  llvm::Reloc::Model RelocationModel;
+  unsigned PICLevel;
+  bool IsPIE;
+  std::tie(RelocationModel, PICLevel, IsPIE) =
+      ParsePICArgs(getToolChain(), TCArgs);
+  if (PICLevel > 0) {
+      LlcArgs.push_back("-relocation-model=pic");
+  }
+  if (IsPIE) {
+      LlcArgs.push_back("-enable-pie");
+  }
+  SmallString<128> LlcPath(C.getDriver().Dir);
+  llvm::sys::path::append(LlcPath, "llc");
+  const char *Llc = C.getArgs().MakeArgString(LlcPath);
+  C.addCommand(llvm::make_unique<Command>(JA, *this, Llc, LlcArgs, None));
+}
+

clang/lib/Driver/ToolChains/Clang.h

@@ -147,6 +147,19 @@ class LLVM_LIBRARY_VISIBILITY OffloadBundler final : public Tool {
                                    const llvm::opt::ArgList &TCArgs,
                                    const char *LinkingOutput) const override;
 };
+
+/// Offload wrapper tool.
+class LLVM_LIBRARY_VISIBILITY OffloadWrapper final : public Tool {
+public:
+  OffloadWrapper(const ToolChain &TC)
+      : Tool("offload wrapper", "clang-offload-wrapper", TC) {}
+
+  bool hasIntegratedCPP() const override { return false; }
+  void ConstructJob(Compilation &C, const JobAction &JA,
+                    const InputInfo &Output, const InputInfoList &Inputs,
+                    const llvm::opt::ArgList &TCArgs,
+                    const char *LinkingOutput) const override;
+};
 } // end namespace tools
 
 } // end namespace driver