Unify plugin and built-in generators to use request/response interface

PiperOrigin-RevId: 801065870

Author: mkruskal-google

src/google/protobuf/compiler/command_line_interface.cc

@@ -28,6 +28,8 @@
 #include <string>
 #include <utility>
 #include <vector>
+
+#include "google/protobuf/compiler/plugin.h"
 #ifdef major
 #undef major
 #endif
@@ -320,7 +322,7 @@ void CommandLineInterface::GetTransitiveDependencies(
     const FileDescriptor* file,
     absl::flat_hash_set<const FileDescriptor*>* already_seen,
     RepeatedPtrField<FileDescriptorProto>* output,
-    const TransitiveDependencyOptions& options) {
+    const TransitiveDependencyOptions& options) const {
   if (!already_seen->insert(file).second) {
     // Already saw this file.  Skip.
     return;
@@ -2733,30 +2735,8 @@ bool CommandLineInterface::GenerateOutput(
     }
   } else {
     // Regular generator.
-    std::string parameters = output_directive.parameter;
-    if (!generator_parameters_[output_directive.name].empty()) {
-      if (!parameters.empty()) {
-        parameters.append(",");
-      }
-      parameters.append(generator_parameters_[output_directive.name]);
-    }
-    if (!EnforceProto3OptionalSupport(
-            output_directive.name,
-            output_directive.generator->GetSupportedFeatures(), parsed_files)) {
-      return false;
-    }
-
-    if (!EnforceEditionsSupport(
-            output_directive.name,
-            output_directive.generator->GetSupportedFeatures(),
-            output_directive.generator->GetMinimumEdition(),
-            output_directive.generator->GetMaximumEdition(), parsed_files)) {
-      return false;
-    }
-
-    if (!output_directive.generator->GenerateAll(parsed_files, parameters,
-                                                 generator_context, &error)) {
-      // Generator returned an error.
+    if (!GenerateBuiltInOutput(parsed_files, output_directive,
+                               generator_context, &error)) {
       std::cerr << output_directive.name << ": " << error << std::endl;
       return false;
     }
@@ -2846,22 +2826,16 @@ bool CommandLineInterface::GenerateDependencyManifestFile(
   return true;
 }
 
-bool CommandLineInterface::GeneratePluginOutput(
-    const std::vector<const FileDescriptor*>& parsed_files,
-    const std::string& plugin_name, const std::string& parameter,
-    GeneratorContext* generator_context, std::string* error) {
+CodeGeneratorRequest CommandLineInterface::CreateCodeGeneratorRequest(
+    std::vector<const FileDescriptor*> parsed_files, std::string parameter,
+    bool copy_json_name, bool bootstrap) const {
   CodeGeneratorRequest request;
-  CodeGeneratorResponse response;
-  std::string processed_parameter = parameter;
-
-  bool bootstrap = GetBootstrapParam(processed_parameter);
 
   // Build the request.
-  if (!processed_parameter.empty()) {
-    request.set_parameter(processed_parameter);
+  if (!parameter.empty()) {
+    request.set_parameter(parameter);
   }
 
-
   absl::flat_hash_set<const FileDescriptor*> already_seen;
   for (const FileDescriptor* file : parsed_files) {
     request.add_file_to_generate(file->name());
@@ -2877,9 +2851,6 @@ bool CommandLineInterface::GeneratePluginOutput(
   const DescriptorPool* pool = parsed_files[0]->pool();
   absl::flat_hash_set<std::string> files_to_generate(input_files_.begin(),
                                                      input_files_.end());
-  static const auto builtin_plugins = new absl::flat_hash_set<std::string>(
-      {"protoc-gen-cpp", "protoc-gen-java", "protoc-gen-mutable_java",
-       "protoc-gen-python"});
   for (FileDescriptorProto& file_proto : *request.mutable_proto_file()) {
     if (files_to_generate.contains(file_proto.name())) {
       const FileDescriptor* file = pool->FindFileByName(file_proto.name());
@@ -2889,8 +2860,7 @@ bool CommandLineInterface::GeneratePluginOutput(
       if (!bootstrap) {
         file->CopySourceCodeInfoTo(&file_proto);
 
-        // The built-in code generators didn't use the json names.
-        if (!builtin_plugins->contains(plugin_name)) {
+        if (copy_json_name) {
           file->CopyJsonNameTo(&file_proto);
         }
       }
@@ -2905,21 +2875,12 @@ bool CommandLineInterface::GeneratePluginOutput(
   version->set_patch(PROTOBUF_VERSION % 1000);
   version->set_suffix(PROTOBUF_VERSION_SUFFIX);
 
-  // Invoke the plugin.
-  Subprocess subprocess;
-
-  if (plugins_.count(plugin_name) > 0) {
-    subprocess.Start(plugins_[plugin_name], Subprocess::EXACT_NAME);
-  } else {
-    subprocess.Start(plugin_name, Subprocess::SEARCH_PATH);
-  }
-
-  std::string communicate_error;
-  if (!subprocess.Communicate(request, &response, &communicate_error)) {
-    *error = absl::Substitute("$0: $1", plugin_name, communicate_error);
-    return false;
-  }
+  return request;
+}
 
+bool CommandLineInterface::GenerateCodeFromResponse(
+    const CodeGeneratorResponse& response, GeneratorContext* generator_context,
+    bool bootstrap, std::string plugin_name, std::string* error) {
   // Write the files.  We do this even if there was a generator error in order
   // to match the behavior of a compiled-in generator.
   std::unique_ptr<io::ZeroCopyOutputStream> current_output;
@@ -2956,6 +2917,46 @@ bool CommandLineInterface::GeneratePluginOutput(
     writer.WriteString(output_file.content());
   }
 
+  return true;
+}
+
+bool CommandLineInterface::GeneratePluginOutput(
+    const std::vector<const FileDescriptor*>& parsed_files,
+    const std::string& plugin_name, const std::string& parameter,
+    GeneratorContext* generator_context, std::string* error) {
+  // TODO Remove these special-cases and send json names to all
+  // plugins.
+  static const auto builtin_plugins = new absl::flat_hash_set<std::string>(
+      {"protoc-gen-cpp", "protoc-gen-java", "protoc-gen-mutable_java",
+       "protoc-gen-python"});
+
+  bool bootstrap = GetBootstrapParam(parameter);
+  CodeGeneratorRequest request = CreateCodeGeneratorRequest(
+      parsed_files, parameter,
+      // The built-in code generators didn't use the json names.
+      /*copy_json_name=*/!builtin_plugins->contains(plugin_name), bootstrap);
+  CodeGeneratorResponse response;
+
+  // Invoke the plugin.
+  Subprocess subprocess;
+
+  if (plugins_.count(plugin_name) > 0) {
+    subprocess.Start(plugins_[plugin_name], Subprocess::EXACT_NAME);
+  } else {
+    subprocess.Start(plugin_name, Subprocess::SEARCH_PATH);
+  }
+
+  std::string communicate_error;
+  if (!subprocess.Communicate(request, &response, &communicate_error)) {
+    *error = absl::Substitute("$0: $1", plugin_name, communicate_error);
+    return false;
+  }
+
+  if (!GenerateCodeFromResponse(response, generator_context, bootstrap,
+                                plugin_name, error)) {
+    return false;
+  }
+
   // Check for errors.
   bool success = true;
   if (!EnforceProto3OptionalSupport(plugin_name, response.supported_features(),
@@ -2977,6 +2978,51 @@ bool CommandLineInterface::GeneratePluginOutput(
   return success;
 }
 
+bool CommandLineInterface::GenerateBuiltInOutput(
+    const std::vector<const FileDescriptor*>& parsed_files,
+    const OutputDirective& output_directive,
+    GeneratorContext* generator_context, std::string* error) {
+  std::string parameters = output_directive.parameter;
+  if (!generator_parameters_[output_directive.name].empty()) {
+    if (!parameters.empty()) {
+      parameters.append(",");
+    }
+    parameters.append(generator_parameters_[output_directive.name]);
+  }
+  if (!EnforceProto3OptionalSupport(
+          output_directive.name,
+          output_directive.generator->GetSupportedFeatures(), parsed_files)) {
+    return false;
+  }
+
+  if (!EnforceEditionsSupport(
+          output_directive.name,
+          output_directive.generator->GetSupportedFeatures(),
+          output_directive.generator->GetMinimumEdition(),
+          output_directive.generator->GetMaximumEdition(), parsed_files)) {
+    return false;
+  }
+
+  CodeGeneratorRequest request =
+      CreateCodeGeneratorRequest(parsed_files, parameters);
+  CodeGeneratorResponse response;
+  if (!GenerateCode(request, *output_directive.generator, &response, error)) {
+    return false;
+  }
+  if (response.has_error()) {
+    *error = response.error();
+    return false;
+  }
+
+  if (!GenerateCodeFromResponse(response, generator_context,
+                                /*bootstrap=*/false, output_directive.name,
+                                error)) {
+    return false;
+  }
+
+  return true;
+}
+
 bool CommandLineInterface::EncodeOrDecode(const DescriptorPool* pool) {
   // Look up the type.
   const Descriptor* type = pool->FindMessageTypeByName(codec_type_);

src/google/protobuf/compiler/command_line_interface.h

@@ -26,6 +26,7 @@
 #include "absl/container/flat_hash_map.h"
 #include "absl/container/flat_hash_set.h"
 #include "absl/strings/string_view.h"
+#include "google/protobuf/compiler/plugin.pb.h"
 #include "google/protobuf/descriptor.pb.h"
 #include "google/protobuf/descriptor_database.h"
 #include "google/protobuf/port.h"
@@ -222,17 +223,6 @@ class PROTOC_EXPORT CommandLineInterface {
   bool MakeInputsBeProtoPathRelative(DiskSourceTree* source_tree,
                                      DescriptorDatabase* fallback_database);
 
-  // Fails if these files use proto3 optional and the code generator doesn't
-  // support it. This is a permanent check.
-  bool EnforceProto3OptionalSupport(
-      const std::string& codegen_name, uint64_t supported_features,
-      const std::vector<const FileDescriptor*>& parsed_files) const;
-
-  bool EnforceEditionsSupport(
-      const std::string& codegen_name, uint64_t supported_features,
-      Edition minimum_edition, Edition maximum_edition,
-      const std::vector<const FileDescriptor*>& parsed_files) const;
-
   bool EnforceProtocEditionsSupport(
       const std::vector<const FileDescriptor*>& parsed_files) const;
 
@@ -295,6 +285,30 @@ class PROTOC_EXPORT CommandLineInterface {
       const std::vector<const FileDescriptor*>& parsed_files,
       const std::string& plugin_name, const std::string& parameter,
       GeneratorContext* generator_context, std::string* error);
+  bool GenerateBuiltInOutput(
+      const std::vector<const FileDescriptor*>& parsed_files,
+      const OutputDirective& output_directive,
+      GeneratorContext* generator_context, std::string* error);
+
+  // Common code for both plugins and built-in generators.
+  CodeGeneratorRequest CreateCodeGeneratorRequest(
+      std::vector<const FileDescriptor*> parsed_files, std::string parameter,
+      bool copy_json_name = false, bool bootstrap = false) const;
+  bool GenerateCodeFromResponse(const CodeGeneratorResponse& response,
+                                GeneratorContext* generator_context,
+                                bool bootstrap, std::string plugin_name,
+                                std::string* error);
+
+  // Fails if these files use proto3 optional and the code generator doesn't
+  // support it. This is a permanent check.
+  bool EnforceProto3OptionalSupport(
+      const std::string& codegen_name, uint64_t supported_features,
+      const std::vector<const FileDescriptor*>& parsed_files) const;
+
+  bool EnforceEditionsSupport(
+      const std::string& codegen_name, uint64_t supported_features,
+      Edition minimum_edition, Edition maximum_edition,
+      const std::vector<const FileDescriptor*>& parsed_files) const;
 
   // Implements --encode and --decode.
   bool EncodeOrDecode(const DescriptorPool* pool);
@@ -342,7 +356,7 @@ class PROTOC_EXPORT CommandLineInterface {
       absl::flat_hash_set<const FileDescriptor*>* already_seen,
       RepeatedPtrField<FileDescriptorProto>* output,
       const TransitiveDependencyOptions& options =
-          TransitiveDependencyOptions());
+          TransitiveDependencyOptions()) const;
 
 
   // -----------------------------------------------------------------

src/google/protobuf/compiler/java/file.cc

@@ -105,15 +105,6 @@ bool CollectExtensions(const Message& message, FieldDescriptorSet* extensions) {
   return true;
 }
 
-void CollectPublicDependencies(
-    const FileDescriptor* file,
-    absl::flat_hash_set<const FileDescriptor*>* dependencies) {
-  if (file == nullptr || !dependencies->insert(file).second) return;
-  for (int i = 0; file != nullptr && i < file->public_dependency_count(); i++) {
-    CollectPublicDependencies(file->public_dependency(i), dependencies);
-  }
-}
-
 // Finds all extensions for custom options in the given file descriptor with the
 // builder pool which resolves Java features instead of the generated pool.
 void CollectExtensions(const FileDescriptor& file,
@@ -138,20 +129,6 @@ void CollectExtensions(const FileDescriptor& file,
   extensions->clear();
   // Unknown extensions are ok and expected in the case of option imports.
   CollectExtensions(*dynamic_file_proto, extensions);
-
-  // TODO: Remove descriptor pool pollution from protoc full.
-  // Check against dependencies to handle option dependencies polluting pool
-  // from using protoc_full with built-in generators instead of plugins.
-  // Option dependencies and transitive dependencies are not allowed, except in
-  // the case of import public.
-  absl::flat_hash_set<const FileDescriptor*> dependencies;
-  dependencies.insert(&file);
-  for (int i = 0; i < file.dependency_count(); i++) {
-    CollectPublicDependencies(file.dependency(i), &dependencies);
-  }
-  absl::erase_if(*extensions, [&](const FieldDescriptor* fieldDescriptor) {
-    return !dependencies.contains(fieldDescriptor->file());
-  });
 }
 
 // Our static initialization methods can become very, very large.