bazel: Migrate py_proto_library to @com_github_grpc_grpc

sergiitk · sergiitk · commit ea7d1c299090 · 2023-11-13T16:18:26.000-08:00
Signed-off-by: Sergii Tkachenko &lt;sergiitk@google.com&gt;
diff --git a/api/bazel/BUILD b/api/bazel/BUILD
@@ -7,7 +7,6 @@ load(
     "EXTERNAL_PROTO_CC_BAZEL_DEP_MAP",
     "EXTERNAL_PROTO_GO_BAZEL_DEP_MAP",
     "EXTERNAL_PROTO_IMPORT_BAZEL_DEP_MAP",
-    "EXTERNAL_PROTO_PY_BAZEL_DEP_MAP",
 )
 
 licenses(["notice"])  # Apache 2
@@ -38,6 +37,5 @@ json_data(
         cc = EXTERNAL_PROTO_CC_BAZEL_DEP_MAP,
         go = EXTERNAL_PROTO_GO_BAZEL_DEP_MAP,
         imports = EXTERNAL_PROTO_IMPORT_BAZEL_DEP_MAP,
-        py = EXTERNAL_PROTO_PY_BAZEL_DEP_MAP,
     ),
 )
diff --git a/api/bazel/api_build_system.bzl b/api/bazel/api_build_system.bzl
@@ -1,14 +1,13 @@
 load("@com_envoyproxy_protoc_gen_validate//bazel:pgv_proto_library.bzl", "pgv_cc_proto_library")
 load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library")
-load("@com_google_protobuf//:protobuf.bzl", _py_proto_library = "py_proto_library")
+load("@com_github_grpc_grpc//bazel:python_rules.bzl", _py_proto_library = "py_proto_library")
 load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
 load("@io_bazel_rules_go//go:def.bzl", "go_test")
 load("@rules_proto//proto:defs.bzl", "proto_library")
 load(
     "//bazel:external_proto_deps.bzl",
     "EXTERNAL_PROTO_CC_BAZEL_DEP_MAP",
     "EXTERNAL_PROTO_GO_BAZEL_DEP_MAP",
-    "EXTERNAL_PROTO_PY_BAZEL_DEP_MAP",
 )
 load(
     "//bazel/cc_proto_descriptor_library:builddefs.bzl",
@@ -52,63 +51,6 @@ def _go_proto_mapping(dep):
 def _cc_proto_mapping(dep):
     return _proto_mapping(dep, EXTERNAL_PROTO_CC_BAZEL_DEP_MAP, _CC_PROTO_SUFFIX)
 
-def _py_proto_mapping(dep):
-    return _proto_mapping(dep, EXTERNAL_PROTO_PY_BAZEL_DEP_MAP, _PY_PROTO_SUFFIX)
-
-# TODO(htuch): Convert this to native py_proto_library once
-# https://github.com/bazelbuild/bazel/issues/3935 and/or
-# https://github.com/bazelbuild/bazel/issues/2626 are resolved.
-def _api_py_proto_library(name, srcs = [], deps = []):
-    mapped_deps = [_py_proto_mapping(dep) for dep in deps]
-    mapped_unique_deps = {k: True for k in mapped_deps}.keys()
-    _py_proto_library(
-        name = name + _PY_PROTO_SUFFIX,
-        srcs = srcs,
-        default_runtime = "@com_google_protobuf//:protobuf_python",
-        protoc = "@com_google_protobuf//:protoc",
-        deps = mapped_unique_deps + [
-            "@com_envoyproxy_protoc_gen_validate//validate:validate_py",
-            "@com_google_googleapis//google/rpc:status_py_proto",
-            "@com_google_googleapis//google/api:annotations_py_proto",
-            "@com_google_googleapis//google/api:http_py_proto",
-            "@com_google_googleapis//google/api:httpbody_py_proto",
-        ],
-        visibility = ["//visibility:public"],
-    )
-
-# This defines googleapis py_proto_library. The repository does not provide its definition and requires
-# overriding it in the consuming project (see https://github.com/grpc/grpc/issues/19255 for more details).
-def py_proto_library(name, deps = [], plugin = None):
-    srcs = [dep[:-6] + ".proto" if dep.endswith("_proto") else dep for dep in deps]
-    proto_deps = []
-
-    # py_proto_library in googleapis specifies *_proto rules in dependencies.
-    # By rewriting *_proto to *.proto above, the dependencies in *_proto rules are not preserved.
-    # As a workaround, manually specify the proto dependencies for the imported python rules.
-    if name == "annotations_py_proto":
-        proto_deps = proto_deps + [":http_py_proto"]
-
-    # checked.proto depends on syntax.proto, we have to add this dependency manually as well.
-    if name == "checked_py_proto":
-        proto_deps = proto_deps + [":syntax_py_proto"]
-
-    # Special handling for expr_proto target
-    if srcs[0] == ":expr_moved.proto":
-        srcs = ["checked.proto", "eval.proto", "explain.proto", "syntax.proto", "value.proto"]
-        proto_deps = proto_deps + ["@com_google_googleapis//google/rpc:status_py_proto"]
-
-    # py_proto_library does not support plugin as an argument yet at gRPC v1.25.0:
-    # https://github.com/grpc/grpc/blob/v1.25.0/bazel/python_rules.bzl#L72.
-    # plugin should also be passed in here when gRPC version is greater than v1.25.x.
-    _py_proto_library(
-        name = name,
-        srcs = srcs,
-        default_runtime = "@com_google_protobuf//:protobuf_python",
-        protoc = "@com_google_protobuf//:protoc",
-        deps = proto_deps + ["@com_google_protobuf//:protobuf_python"],
-        visibility = ["//visibility:public"],
-    )
-
 def _api_cc_grpc_library(name, proto, deps = []):
     cc_grpc_library(
         name = name,
@@ -157,7 +99,15 @@ def api_cc_py_proto_library(
         deps = [relative_name],
         visibility = ["//visibility:public"],
     )
-    _api_py_proto_library(name, srcs, deps)
+
+    # Uses gRPC implementation of py_proto_library.
+    # https://github.com/grpc/grpc/blob/v1.59.1/bazel/python_rules.bzl#L160
+    _py_proto_library(
+        name = name + _PY_PROTO_SUFFIX,
+        # Actual dependencies are resolved automatically from the proto_library dep tree.
+        deps = [relative_name],
+        visibility = ["//visibility:public"],
+    )
 
     # Optionally define gRPC services
     if has_services:
diff --git a/api/bazel/external_proto_deps.bzl b/api/bazel/external_proto_deps.bzl
@@ -49,15 +49,3 @@ EXTERNAL_PROTO_CC_BAZEL_DEP_MAP = {
     "@opentelemetry_proto//:metrics": "@opentelemetry_proto//:metrics_cc_proto",
     "@opentelemetry_proto//:common": "@opentelemetry_proto//:common_cc_proto",
 }
-
-# This maps from the Bazel proto_library target to the Python language binding target for external dependencies.
-EXTERNAL_PROTO_PY_BAZEL_DEP_MAP = {
-    "@com_google_googleapis//google/api/expr/v1alpha1:checked_proto": "@com_google_googleapis//google/api/expr/v1alpha1:expr_py_proto",
-    "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto": "@com_google_googleapis//google/api/expr/v1alpha1:expr_py_proto",
-    "@opencensus_proto//opencensus/proto/trace/v1:trace_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_proto_py",
-    "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto_py",
-    "@opentelemetry_proto//:trace": "@opentelemetry_proto//:trace_py_proto",
-    "@opentelemetry_proto//:logs": "@opentelemetry_proto//:logs_py_proto",
-    "@opentelemetry_proto//:metrics": "@opentelemetry_proto//:metrics_py_proto",
-    "@opentelemetry_proto//:common": "@opentelemetry_proto//:common_py_proto",
-}
diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl
@@ -39,8 +39,8 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_desc = "xDS API Working Group (xDS-WG)",
         project_url = "https://github.com/cncf/xds",
         # During the UDPA -> xDS migration, we aren't working with releases.
-        version = "523115ebc1014a83e9cf1e85194ef8f8739d87c7",
-        sha256 = "3bd28b29380372d54848111b12e24ec684f890032b42b2719ee6971658016b72",
+        version = "83c031ea693357fdf7a7fea9a68a785d97864a38",
+        sha256 = "35bdcdbcd2e437058ad1f74a91b49525339b028a6b52760ab019fd9efa5a6d44",
         release_date = "2023-06-07",
         strip_prefix = "xds-{version}",
         urls = ["https://github.com/cncf/xds/archive/{version}.tar.gz"],
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -368,10 +368,8 @@ def envoy_dependencies(skip_targets = []):
         name = "com_google_googleapis_imports",
         cc = True,
         go = True,
+        python = True,
         grpc = True,
-        rules_override = {
-            "py_proto_library": ["@envoy_api//bazel:api_build_system.bzl", ""],
-        },
     )
     native.bind(
         name = "bazel_runfiles",
diff --git a/tools/protodoc/BUILD b/tools/protodoc/BUILD
@@ -1,5 +1,6 @@
 load("@base_pip3//:requirements.bzl", "requirement")
-load("@com_google_protobuf//:protobuf.bzl", "py_proto_library")
+load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library")
+load("@rules_proto//proto:defs.bzl", "proto_library")
 load("//bazel:envoy_build_system.bzl", "envoy_package")
 load("//tools/base:envoy_python.bzl", "envoy_genjson", "envoy_jinja_env", "envoy_py_data", "envoy_pytool_binary", "envoy_pytool_library")
 load("//tools/protodoc:protodoc.bzl", "protodoc_rule")
@@ -21,10 +22,22 @@ envoy_pytool_binary(
     ],
 )
 
-py_proto_library(
+proto_library(
     name = "manifest_proto",
     srcs = ["manifest.proto"],
-    deps = ["@com_google_protobuf//:protobuf_python"],
+    deps = [
+        "@com_google_protobuf//:struct_proto",
+    ],
+)
+
+py_proto_library(
+    name = "manifest_py_pb2",
+    deps = [":manifest_proto"],
+)
+
+py_proto_library(
+    name = "validate_py_pb2",
+    deps = ["@com_envoyproxy_protoc_gen_validate//validate:validate_proto"],
 )
 
 envoy_py_data(
@@ -39,7 +52,7 @@ envoy_pytool_binary(
     data = ["@envoy_api//:v3_proto_set"],
     deps = [
         requirement("envoy.base.utils"),
-        ":manifest_proto",
+        ":manifest_py_pb2",
         ":protodoc_manifest_untyped",
         "@com_google_protobuf//:protobuf_python",
     ],
@@ -110,8 +123,8 @@ envoy_pytool_binary(
     deps = [
         ":data",
         ":jinja",
+        ":validate_py_pb2",
         "//tools/api_proto_plugin",
-        "@com_envoyproxy_protoc_gen_validate//validate:validate_py",
         "@com_github_cncf_xds//udpa/annotations:pkg_py_proto",
         "@com_github_cncf_xds//xds/annotations/v3:pkg_py_proto",
         requirement("envoy.code.check"),
diff --git a/tools/protoprint/BUILD b/tools/protoprint/BUILD
@@ -1,4 +1,5 @@
 load("@base_pip3//:requirements.bzl", "requirement")
+load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library")
 load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix")
 load("@rules_pkg//pkg:pkg.bzl", "pkg_tar")
 load("//bazel:envoy_build_system.bzl", "envoy_package")
@@ -22,14 +23,14 @@ envoy_pytool_binary(
     ],
     visibility = ["//visibility:public"],
     deps = [
+        ":validate_py_pb2",
         "//tools/api_versioning:utils",
         "//tools/protoxform:options",
         "//tools/protoxform:utils",
         "@envoy_repo",
         requirement("packaging"),
         "//tools/type_whisperer",
         "//tools/type_whisperer:api_type_db_proto_py_proto",
-        "@com_envoyproxy_protoc_gen_validate//validate:validate_py",
         "@com_github_cncf_xds//udpa/annotations:pkg_py_proto",
         "@com_github_cncf_xds//xds/annotations/v3:pkg_py_proto",
         "@com_google_googleapis//google/api:annotations_py_proto",
@@ -38,6 +39,11 @@ envoy_pytool_binary(
     ],
 )
 
+py_proto_library(
+    name = "validate_py_pb2",
+    deps = ["@com_envoyproxy_protoc_gen_validate//validate:validate_proto"],
+)
+
 protoprint_rule(
     name = "protoprinted_srcs",
     deps = [
diff --git a/tools/protoxform/BUILD b/tools/protoxform/BUILD
@@ -1,4 +1,5 @@
 load("@aspect_bazel_lib//lib:jq.bzl", "jq")
+load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library")
 load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix")
 load("@rules_pkg//pkg:pkg.bzl", "pkg_tar")
 load("@rules_python//python:defs.bzl", "py_binary", "py_library")
@@ -15,7 +16,6 @@ py_binary(
         ":utils",
         "//tools/api_proto_plugin",
         "//tools/type_whisperer:api_type_db_proto_py_proto",
-        "@com_envoyproxy_protoc_gen_validate//validate:validate_py",
         "@com_github_cncf_xds//udpa/annotations:pkg_py_proto",
         "@com_github_cncf_xds//xds/annotations/v3:pkg_py_proto",
         "@com_google_googleapis//google/api:annotations_py_proto",
@@ -33,6 +33,14 @@ py_library(
     name = "utils",
     srcs = ["utils.py"],
     visibility = ["//visibility:public"],
+    deps = [
+        ":validate_py_pb2",
+    ],
+)
+
+py_proto_library(
+    name = "validate_py_pb2",
+    deps = ["@com_envoyproxy_protoc_gen_validate//validate:validate_proto"],
 )
 
 protoxform_rule(

Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,6 @@ load(`
`7`	`7`	`"EXTERNAL_PROTO_CC_BAZEL_DEP_MAP",`
`8`	`8`	`"EXTERNAL_PROTO_GO_BAZEL_DEP_MAP",`
`9`	`9`	`"EXTERNAL_PROTO_IMPORT_BAZEL_DEP_MAP",`
`10`		`- "EXTERNAL_PROTO_PY_BAZEL_DEP_MAP",`
`11`	`10`	`)`
`12`	`11`
`13`	`12`	`licenses(["notice"]) # Apache 2`
`@@ -38,6 +37,5 @@ json_data(`
`38`	`37`	`cc = EXTERNAL_PROTO_CC_BAZEL_DEP_MAP,`
`39`	`38`	`go = EXTERNAL_PROTO_GO_BAZEL_DEP_MAP,`
`40`	`39`	`imports = EXTERNAL_PROTO_IMPORT_BAZEL_DEP_MAP,`
`41`		`- py = EXTERNAL_PROTO_PY_BAZEL_DEP_MAP,`
`42`	`40`	`),`
`43`	`41`	`)`