Add --projects cli flag to black-primer

Makes it possible to run a subset of projects on black primer

Author: nipunn1313

CHANGES.md

@@ -8,6 +8,7 @@
 - Add new `--workers` parameter (#2514)
 - Fixed feature detection for positional-only arguments in lambdas (#2532)
 - Bumped typed-ast version minimum to 1.4.3 for 3.10 compatiblity (#2519)
+- Add primer support for --projects (#2555)
 
 ### _Blackd_
 

mypy.ini

@@ -35,3 +35,7 @@ cache_dir=/dev/null
 [mypy-black_primer.*]
 # Until we're not supporting 3.6 primer needs this
 disallow_any_generics=False
+
+[mypy-tests.test_primer]
+# Until we're not supporting 3.6 primer needs this
+disallow_any_generics=False

src/black_primer/cli.py

@@ -48,11 +48,13 @@ async def async_main(
     keep: bool,
     long_checkouts: bool,
     no_diff: bool,
+    projects: str,
     rebase: bool,
     workdir: str,
     workers: int,
 ) -> int:
     work_path = Path(workdir)
+    projects_to_run = set(p for p in projects.split(",")) if projects else None
     if not work_path.exists():
         LOG.debug(f"Creating {work_path}")
         work_path.mkdir()
@@ -70,6 +72,7 @@ async def async_main(
             long_checkouts,
             rebase,
             no_diff,
+            projects_to_run,
         )
         return int(ret_val)
     finally:
@@ -116,6 +119,10 @@ async def async_main(
     show_default=True,
     help="Disable showing source file changes in black output",
 )
+@click.option(
+    "--projects",
+    help="Comma separated list of projects to run (Default: run all)",
+)
 @click.option(
     "-R",
     "--rebase",

src/black_primer/lib.py

@@ -20,6 +20,7 @@
     NamedTuple,
     Optional,
     Sequence,
+    Set,
     Tuple,
     Union,
 )
@@ -283,17 +284,26 @@ def handle_PermissionError(
 
 async def load_projects_queue(
     config_path: Path,
+    projects_to_run: Optional[Set[str]],
 ) -> Tuple[Dict[str, Any], asyncio.Queue]:
     """Load project config and fill queue with all the project names"""
     with config_path.open("r") as cfp:
         config = json.load(cfp)
 
     # TODO: Offer more options here
-    # e.g. Run on X random packages or specific sub list etc.
+    # e.g. Run on X random packages etc.
     project_names = sorted(config["projects"].keys())
+    projects_to_run = projects_to_run or set(project_names)
     queue: asyncio.Queue = asyncio.Queue(maxsize=len(project_names))
     for project in project_names:
-        await queue.put(project)
+        if project in projects_to_run:
+            await queue.put(project)
+            projects_to_run.remove(project)
+
+    if projects_to_run:
+        LOG.error(
+            f"Project not found: {projects_to_run}. Available projects: {project_names}"
+        )
 
     return config, queue
 
@@ -369,6 +379,7 @@ async def process_queue(
     long_checkouts: bool = False,
     rebase: bool = False,
     no_diff: bool = False,
+    projects_to_run: Optional[Set[str]] = None,
 ) -> int:
     """
     Process the queue with X workers and evaluate results
@@ -383,7 +394,7 @@ async def process_queue(
     results.stats["success"] = 0
     results.stats["wrong_py_ver"] = 0
 
-    config, queue = await load_projects_queue(Path(config_file))
+    config, queue = await load_projects_queue(Path(config_file), projects_to_run)
     project_count = queue.qsize()
     s = "" if project_count == 1 else "s"
     LOG.info(f"{project_count} project{s} to run Black over")

tests/test_format.py

@@ -93,6 +93,8 @@
     "src/black/strings.py",
     "src/black/trans.py",
     "src/blackd/__init__.py",
+    "src/black_primer/cli.py",
+    "src/black_primer/lib.py",
     "src/blib2to3/pygram.py",
     "src/blib2to3/pytree.py",
     "src/blib2to3/pgen2/conv.py",

tests/test_primer.py

@@ -11,7 +11,7 @@
 from platform import system
 from subprocess import CalledProcessError
 from tempfile import TemporaryDirectory, gettempdir
-from typing import Any, Callable, Iterator, Tuple
+from typing import Any, Callable, Iterator, List, Tuple, TypeVar
 from unittest.mock import Mock, patch
 
 from click.testing import CliRunner
@@ -89,6 +89,24 @@ async def return_zero(*args: Any, **kwargs: Any) -> int:
     return 0
 
 
+if sys.version_info >= (3, 9):
+    T = TypeVar("T")
+    Q = asyncio.Queue[T]
+else:
+    T = Any
+    Q = asyncio.Queue
+
+
+def collect(queue: Q) -> List[T]:
+    ret = []
+    while True:
+        try:
+            item = queue.get_nowait()
+            ret.append(item)
+        except asyncio.QueueEmpty:
+            return ret
+
+
 class PrimerLibTests(unittest.TestCase):
     def test_analyze_results(self) -> None:
         fake_results = lib.Results(
@@ -202,6 +220,26 @@ def test_process_queue(self, mock_stdout: Mock) -> None:
                 )
                 self.assertEqual(0, return_val)
 
+    @event_loop()
+    def test_load_projects_queue(self) -> None:
+        """Test the process queue on primer itself
+        - If you have non black conforming formatting in primer itself this can fail"""
+        loop = asyncio.get_event_loop()
+        config_path = Path(lib.__file__).parent / "primer.json"
+
+        config, projects_queue = loop.run_until_complete(
+            lib.load_projects_queue(config_path, None)
+        )
+        projects = collect(projects_queue)
+        self.assertEqual(len(config["projects"].keys()), 22)
+        self.assertEqual(set(projects), set(config["projects"].keys()))
+
+        config, projects_queue = loop.run_until_complete(
+            lib.load_projects_queue(config_path, set(["django", "pyramid", "nonsense"]))
+        )
+        projects = collect(projects_queue)
+        self.assertEqual(projects, ["django", "pyramid"])
+
 
 class PrimerCLITests(unittest.TestCase):
     @event_loop()
@@ -217,6 +255,7 @@ def test_async_main(self) -> None:
             "workdir": str(work_dir),
             "workers": 69,
             "no_diff": False,
+            "projects": "",
         }
         with patch("black_primer.cli.lib.process_queue", return_zero):
             return_val = loop.run_until_complete(cli.async_main(**args))  # type: ignore