chore(python): autogenerate docs/index.rst (#1114)

* chore(python): autogenerate docs/index.rst

* set default path to owl-bot-staging

* optimize code using glob

* move the default_version to the front of the list

* Drop redundant default_version argument

* remove unused imports

* read default_version from .repo-metadata.json

* exclude docs/index.rst if default_version is not specified

* check for versions kwarg

* only generate docs/index.rst if default_version is specified

* lint

* fix build

* fix build

* consolidate the duplicated detect_versions function

* remove unused import

* remove unused imports

* update comments

* remove unused import

* run black

Author: parthea

synthtool/gcp/common.py

@@ -209,7 +209,15 @@ def py_library(self, **kwargs) -> Path:
         # kwargs["metadata"] is required to load values from .repo-metadata.json
         if "metadata" not in kwargs:
             kwargs["metadata"] = {}
-        # rename variable to accomodate existing synth.py files
+
+        # load common repo meta information (metadata that's not language specific).
+        self._load_generic_metadata(kwargs["metadata"])
+
+        # initialize default_version if it doesn't exist in kwargs["metadata"]['repo']
+        if "default_version" not in kwargs["metadata"]["repo"]:
+            kwargs["metadata"]["repo"]["default_version"] = ""
+
+        # rename variable to accommodate existing owlbot.py files
         if "system_test_dependencies" in kwargs:
             kwargs["system_test_local_dependencies"] = kwargs[
                 "system_test_dependencies"
@@ -237,6 +245,13 @@ def py_library(self, **kwargs) -> Path:
         if "samples" not in kwargs:
             self.excludes += ["samples/AUTHORING_GUIDE.md", "samples/CONTRIBUTING.md"]
 
+        # Don't add `docs/index.rst` if `versions` is not provided or `default_version` is empty
+        if (
+            "versions" not in kwargs
+            or not kwargs["metadata"]["repo"]["default_version"]
+        ):
+            self.excludes += ["docs/index.rst"]
+
         # Assume the python-docs-samples Dockerfile is used for samples by default
         if "custom_samples_dockerfile" not in kwargs:
             kwargs["custom_samples_dockerfile"] = False
@@ -323,6 +338,69 @@ def _load_generic_metadata(self, metadata: Dict):
             metadata["repo"] = _load_repo_metadata()
 
 
+def detect_versions(
+    path: str = "./src",
+    default_version: Optional[str] = None,
+    default_first: Optional[bool] = None,
+) -> List[str]:
+    """
+    Detects the versions a library has, based on distinct folders
+    within path. This is based on the fact that our GAPIC libraries are
+    structured as follows:
+
+    src/v1
+    src/v1beta
+    src/v1alpha
+
+    With folder names mapping directly to versions.
+
+    Returns: a list of the sorted subdirectories; for the example above:
+      ['v1', 'v1alpha', 'v1beta']
+      If the `default_version` argument is not provided, the `default_version`
+      will be read from `.repo-metadata.json`, if it exists.
+      If `default_version` is available, the `default_version` is moved to
+      at the front or the end of the sorted list depending on the value of `default_first`.
+      The `default_version` will be first in the list when `default_first` is `True`.
+    """
+
+    versions = []
+
+    if not default_version:
+        try:
+            # Get the `default_version` from ``.repo-metadata.json`.
+            default_version = json.load(open(".repo-metadata.json", "rt")).get(
+                "default_version"
+            )
+        except FileNotFoundError:
+            pass
+
+    # Sort the sub directories alphabetically.
+    sub_dirs = sorted([p.name for p in Path(path).glob("*v[1-9]*")])
+
+    if sub_dirs:
+        # if `default_version` is not specified, return the sorted directories.
+        if not default_version:
+            versions = sub_dirs
+        else:
+            # The subdirectory with the same suffix as the default_version
+            # will be the default client.
+            default_client = next(
+                iter([d for d in sub_dirs if d.endswith(default_version)]), None
+            )
+
+            # start with all the versions except for the default client
+            versions = [d for d in sub_dirs if not d.endswith(default_version)]
+
+            if default_client:
+                # If `default_first` is true, the default_client will be first
+                # in the list.
+                if default_first:
+                    versions = [default_client] + versions
+                else:
+                    versions += [default_client]
+    return versions
+
+
 def decamelize(value: str):
     """ parser to convert fooBar.js to Foo Bar. """
     if not value:

synthtool/gcp/templates/python_library/docs/index.rst

@@ -0,0 +1,36 @@
+.. include:: README.rst
+
+.. include:: multiprocessing.rst
+{% if versions|length > 1 %}
+This package includes clients for multiple versions of {{ metadata['repo']['name_pretty'] }}.
+By default, you will get version ``{{ versions | first }}``.
+{% endif %}
+{% for version in versions %}
+API Reference
+-------------
+.. toctree::
+    :maxdepth: 2
+
+    {{ version }}/services
+    {{ version }}/types
+{% endfor %}
+{%- if migration_guide_version %}
+Migration Guide
+---------------
+
+See the guide below for instructions on migrating to the {{ migration_guide_version }} release of this library.
+
+.. toctree::
+    :maxdepth: 2
+
+    UPGRADING
+{% endif %}
+Changelog
+---------
+
+For a list of all ``{{ metadata['repo']['distribution_name'] }}`` releases:
+
+.. toctree::
+   :maxdepth: 2
+
+   changelog

synthtool/languages/node.py

@@ -15,7 +15,6 @@
 import json
 from jinja2 import FileSystemLoader, Environment
 from pathlib import Path
-import os
 import re
 from synthtool import _tracked_paths, gcp, shell, transforms
 from synthtool.gcp import samples, snippets
@@ -214,36 +213,6 @@ def compile_protos(hide_output=False):
     shell.run(["npx", "compileProtos", "src"], hide_output=hide_output)
 
 
-def detect_versions(
-    path: str = "./src", default_version: Optional[str] = None
-) -> List[str]:
-    """
-    Detects the versions a library has, based on distinct folders
-    within path. This is based on the fact that our GAPIC libraries are
-    structured as follows:
-
-    src/v1
-    src/v1beta
-    src/v1alpha
-
-    With folder names mapping directly to versions.
-
-    Returns: a list of the subdirectories; for the example above:
-      ['v1', 'v1alpha', 'v1beta']
-      If specified, the default_version is guaranteed to be listed last.
-      Otherwise, the list is sorted alphabetically.
-    """
-    versions = []
-    if os.path.isdir(path):
-        for directory in os.listdir(path):
-            if os.path.isdir(os.path.join(path, directory)):
-                versions.append(directory)
-    versions.sort()
-    if default_version is not None:
-        versions = [v for v in versions if v != default_version] + [default_version]
-    return versions
-
-
 def compile_protos_hermetic(hide_output=False):
     """
     Compiles protos into .json, .js, and .d.ts files using

tests/test_common.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import tempfile
 from pathlib import Path
@@ -20,7 +21,7 @@
 from pytest import raises
 
 import synthtool as s
-from synthtool.gcp.common import _get_default_branch_name, decamelize
+from synthtool.gcp.common import _get_default_branch_name, decamelize, detect_versions
 
 from . import util
 
@@ -159,3 +160,76 @@ def test_py_samples_multiple_override_content():
         with open("README.md") as f:
             result = f.read()
             assert "Last Example" in result
+
+
+def test_detect_versions_src():
+    temp_dir = Path(tempfile.mkdtemp())
+    src_dir = temp_dir / "src"
+    for v in ("v1", "v2", "v3"):
+        os.makedirs(src_dir / v)
+
+    with util.chdir(temp_dir):
+        versions = detect_versions()
+        assert ["v1", "v2", "v3"] == versions
+
+
+def test_detect_versions_staging():
+    temp_dir = Path(tempfile.mkdtemp())
+    staging_dir = temp_dir / "owl-bot-staging"
+    for v in ("v1", "v2", "v3"):
+        os.makedirs(staging_dir / v)
+
+    versions = detect_versions(staging_dir)
+    assert ["v1", "v2", "v3"] == versions
+
+
+def test_detect_versions_dir_not_found():
+    temp_dir = Path(tempfile.mkdtemp())
+
+    versions = detect_versions(temp_dir / "does-not-exist")
+    assert [] == versions
+
+
+def test_detect_versions_with_default_version():
+    temp_dir = Path(tempfile.mkdtemp())
+    src_dir = temp_dir / "src"
+    vs = ("v1", "v2", "v3")
+    for v in vs:
+        os.makedirs(src_dir / v)
+
+    with util.chdir(temp_dir):
+        versions = detect_versions(default_version="v1")
+        assert ["v2", "v3", "v1"] == versions
+        versions = detect_versions(default_version="v2")
+        assert ["v1", "v3", "v2"] == versions
+        versions = detect_versions(default_version="v3")
+        assert ["v1", "v2", "v3"] == versions
+
+
+def test_detect_versions_with_default_version_from_metadata():
+    temp_dir = Path(tempfile.mkdtemp())
+    default_dir = temp_dir / "src"
+    for v in ("api_v1", "api_v2", "api_v3"):
+        os.makedirs(default_dir / v)
+
+    with util.chdir(temp_dir):
+        # Set default_version to "api_v1"
+        test_json = {"default_version": "api_v1"}
+        with open(".repo-metadata.json", "w") as metadata:
+            json.dump(test_json, metadata)
+        versions = detect_versions(default_first=True)
+        assert ["api_v1", "api_v2", "api_v3"] == versions
+
+        # Set default_version to "api_v2"
+        test_json = {"default_version": "api_v2"}
+        with open(".repo-metadata.json", "w") as metadata:
+            json.dump(test_json, metadata)
+        versions = detect_versions(default_first=True)
+        assert ["api_v2", "api_v1", "api_v3"] == versions
+
+        # Set default_version to "api_v3"
+        test_json = {"default_version": "api_v3"}
+        with open(".repo-metadata.json", "w") as metadata:
+            json.dump(test_json, metadata)
+        versions = detect_versions(default_first=True)
+        assert ["api_v3", "api_v1", "api_v2"] == versions

tests/test_node.py

@@ -13,9 +13,7 @@
 # limitations under the License.
 
 import filecmp
-import os
 import pathlib
-import tempfile
 from pathlib import Path
 from unittest import TestCase
 from unittest.mock import patch
@@ -272,47 +270,3 @@ def patch(library: Path):
     assert "import * as v2" in staging_text
     assert "export * as v2" not in staging_text
     assert "export * as v2" in text
-
-
-def test_detect_versions_src():
-    temp_dir = Path(tempfile.mkdtemp())
-    src_dir = temp_dir / "src"
-    for v in ("v1", "v2", "v3"):
-        os.makedirs(src_dir / v)
-
-    with util.chdir(temp_dir):
-        versions = node.detect_versions()
-        assert ["v1", "v2", "v3"] == versions
-
-
-def test_detect_versions_staging():
-    temp_dir = Path(tempfile.mkdtemp())
-    staging_dir = temp_dir / "owl-bot-staging"
-    for v in ("v1", "v2", "v3"):
-        os.makedirs(staging_dir / v)
-
-    versions = node.detect_versions(staging_dir)
-    assert ["v1", "v2", "v3"] == versions
-
-
-def test_detect_versions_dir_not_found():
-    temp_dir = Path(tempfile.mkdtemp())
-
-    versions = node.detect_versions(temp_dir / "does-not-exist")
-    assert [] == versions
-
-
-def test_detect_versions_with_default():
-    temp_dir = Path(tempfile.mkdtemp())
-    src_dir = temp_dir / "src"
-    vs = ("v1", "v2", "v3")
-    for v in vs:
-        os.makedirs(src_dir / v)
-
-    with util.chdir(temp_dir):
-        versions = node.detect_versions(default_version="v1")
-        assert ["v2", "v3", "v1"] == versions
-        versions = node.detect_versions(default_version="v2")
-        assert ["v1", "v3", "v2"] == versions
-        versions = node.detect_versions(default_version="v3")
-        assert ["v1", "v2", "v3"] == versions