Azdev enhancement for working with CI. (#115)

* Azdev enhancement for working with CI.

* Enhance diffing mechanism.

* Remove unused test files.

* Fix issues when installed from edge build.

Author: tjprescott

HISTORY.rst

@@ -3,6 +3,21 @@
 Release History
 ===============
 
+0.1.5
+++++++
+* `azdev extension add/remove`: Added ability to supply wildcard (*) to install all available dev extensions
+                                and remove all installed dev extensions.
+* `azdev setup`: Added ability to install all extensions using `--ext/-e *`. Added ability to install CLI edge build
+                 with `--cli/-c EDGE`.
+* `azdev style/test/linter`: Add special names CLI and EXT to allow running on just CLI modules or just extensions.
+                             Add new argument group `--tgt`, `--src`, `--repo` to allow checking only modules or
+                             extensions which have changed based on a git diff.
+* `azdev linter`: Added `--include-whl-extensions` flag to permit running the linter on extensions installed using
+                  the `az extension add` command.
+* `azdev verify license`: Command will not check any dev-installed CLI and extension repos. Previously, it only
+                          checked the CLI repo.
+* Added new commands `azdev cli/extension generate-docs` to generate sphinx documentation.
+
 0.1.4
 ++++++
 * `azdev linter`: Fix issue with help example rule.

azdev/__init__.py

@@ -4,4 +4,4 @@
 # license information.
 # -----------------------------------------------------------------------------
 
-__VERSION__ = '0.1.4'
+__VERSION__ = '0.1.5'

azdev/commands.py

@@ -62,13 +62,3 @@ def operation_group(name):
 
     with CommandGroup(self, 'extension', operation_group('help')) as g:
         g.command('generate-docs', 'generate_extension_ref_docs')
-
-    # TODO: implement
-    # with CommandGroup(self, 'coverage', command_path) as g:
-    #    g.command('command', 'command_coverage')
-    #    g.command('code', 'code_coverage')
-
-    # TODO: implement
-    # with CommandGroup(self, 'verify', command_path) as g:
-    #    g.command('package', 'verify_packages')
-    #    g.command('dependencies', 'verify_dependencies')

azdev/help.py

@@ -65,22 +65,6 @@
 """
 
 
-helps['coverage'] = """
-    short-summary: Test coverage statistics and reports.
-"""
-
-
-helps['coverage code'] = """
-    short-summary: Run CLI tests with code coverage.
-"""
-
-
-helps['coverage command'] = """
-    short-summary: Analyze CLI test run data for command and argument coverage.
-    long-summary: This does not run any tests!
-"""
-
-
 helps['verify'] = """
     short-summary: Verify CLI product features.
 """
@@ -113,6 +97,9 @@
 
 helps['style'] = """
     short-summary: Check code style (pylint and PEP8).
+    examples:
+        - name: Check style for only those modules which have changed based on a git diff.
+          text: azdev style --repo azure-cli --tgt upstream/master --src upstream/dev
 """
 
 
@@ -134,11 +121,17 @@
 
         - name: Run tests for a module but run the tests that failed last time first.
           text: azdev test {mod} -a --ff
+
+        - name: Run tests for only those modules which have changed based on a git diff.
+          text: azdev test --repo azure-cli --tgt upstream/master --src upstream/dev
 """
 
 
 helps['linter'] = """
     short-summary: Static code checks of the CLI command table.
+    examples:
+        - name: Check linter rules for only those modules which have changed based on a git diff.
+          text: azdev linter --repo azure-cli --tgt upstream/master --src upstream/dev
 """
 
 
@@ -152,16 +145,6 @@
 """
 
 
-helps['sdk'] = """
-    short-summary: Perform quick Python SDK operations.
-"""
-
-
-helps['sdk draft'] = """
-    short-summary: Install draft packages from the Python SDK repo.
-"""
-
-
 helps['extension'] = """
     short-summary: Control which CLI extensions are visible in the development environment.
 """

azdev/operations/extensions/__init__.py

@@ -26,16 +26,19 @@ def add_extension(extensions):
     ext_paths = get_ext_repo_paths()
     all_extensions = find_files(ext_paths, 'setup.py')
 
-    paths_to_add = []
-    for path in all_extensions:
-        folder = os.path.dirname(path)
-        long_name = os.path.basename(folder)
-        if long_name in extensions:
-            paths_to_add.append(folder)
-            extensions.remove(long_name)
-    # raise error if any extension wasn't found
-    if extensions:
-        raise CLIError('extension(s) not found: {}'.format(' '.join(extensions)))
+    if extensions == ['*']:
+        paths_to_add = [os.path.dirname(path) for path in all_extensions if 'site-packages' not in path]
+    else:
+        paths_to_add = []
+        for path in all_extensions:
+            folder = os.path.dirname(path)
+            long_name = os.path.basename(folder)
+            if long_name in extensions:
+                paths_to_add.append(folder)
+                extensions.remove(long_name)
+        # raise error if any extension wasn't found
+        if extensions:
+            raise CLIError('extension(s) not found: {}'.format(' '.join(extensions)))
 
     for path in paths_to_add:
         result = pip_cmd('install -e {}'.format(path), "Adding extension '{}'...".format(path))
@@ -49,16 +52,20 @@ def remove_extension(extensions):
     installed_paths = find_files(ext_paths, '*.*-info')
     paths_to_remove = []
     names_to_remove = []
-    for path in installed_paths:
-        folder = os.path.dirname(path)
-        long_name = os.path.basename(folder)
-        if long_name in extensions:
-            paths_to_remove.append(folder)
-            names_to_remove.append(long_name)
-            extensions.remove(long_name)
-    # raise error if any extension not installed
-    if extensions:
-        raise CLIError('extension(s) not installed: {}'.format(' '.join(extensions)))
+    if extensions == ['*']:
+        paths_to_remove = [os.path.dirname(path) for path in installed_paths]
+        names_to_remove = [os.path.basename(os.path.dirname(path)) for path in installed_paths]
+    else:
+        for path in installed_paths:
+            folder = os.path.dirname(path)
+            long_name = os.path.basename(folder)
+            if long_name in extensions:
+                paths_to_remove.append(folder)
+                names_to_remove.append(long_name)
+                extensions.remove(long_name)
+        # raise error if any extension not installed
+        if extensions:
+            raise CLIError('extension(s) not installed: {}'.format(' '.join(extensions)))
 
     # removes any links that may have been added to site-packages.
     for ext in names_to_remove:
@@ -113,12 +120,17 @@ def list_extensions():
         except IndexError:
             continue
 
+        # ignore anything in site-packages folder
+        if 'site-packages' in ext_path:
+            continue
+
         folder = os.path.dirname(ext_path)
         long_name = os.path.basename(folder)
         if long_name not in installed_names:
             results.append({'name': long_name, 'install': '', 'path': folder})
         else:
             results.append({'name': long_name, 'install': 'Installed', 'path': folder})
+
     return results
 
 

azdev/operations/legal.py

@@ -9,7 +9,7 @@
 from knack.util import CLIError
 
 from azdev.utilities import (
-    display, heading, subheading, get_cli_repo_path)
+    display, heading, subheading, get_cli_repo_path, get_ext_repo_paths)
 
 
 LICENSE_HEADER = """# --------------------------------------------------------------------------------------------
@@ -18,25 +18,32 @@
 # --------------------------------------------------------------------------------------------
 """
 
+_IGNORE_SUBDIRS = ['__pycache__', 'vendored_sdks', 'site-packages', 'env']
+
 
 def check_license_headers():
 
     heading('Verify License Headers')
 
     cli_path = get_cli_repo_path()
-    env_path = os.path.join(cli_path, 'env')
+    all_paths = [cli_path]
+    for path in get_ext_repo_paths():
+        all_paths.append(path)
 
     files_without_header = []
-    for current_dir, _, files in os.walk(cli_path):
-        if current_dir.startswith(env_path):
-            continue
-
-        file_itr = (os.path.join(current_dir, p) for p in files if p.endswith('.py') and p != 'azure_bdist_wheel.py')
-        for python_file in file_itr:
-            with open(python_file, 'r', encoding='utf-8') as f:
-                file_text = f.read()
-                if file_text and LICENSE_HEADER not in file_text:
-                    files_without_header.append(os.path.join(current_dir, python_file))
+    for path in all_paths:
+        for current_dir, subdirs, files in os.walk(path):
+            for i, x in enumerate(subdirs):
+                if x in _IGNORE_SUBDIRS or x.startswith('.'):
+                    del subdirs[i]
+
+            # pylint: disable=line-too-long
+            file_itr = (os.path.join(current_dir, p) for p in files if p.endswith('.py') and p != 'azure_bdist_wheel.py')
+            for python_file in file_itr:
+                with open(python_file, 'r', encoding='utf-8') as f:
+                    file_text = f.read()
+                    if file_text and LICENSE_HEADER not in file_text:
+                        files_without_header.append(os.path.join(current_dir, python_file))
 
     subheading('Results')
     if files_without_header:

azdev/operations/linter/__init__.py

@@ -16,7 +16,7 @@
 from knack.util import CLIError
 
 from azdev.utilities import (
-    heading, subheading, display, get_path_table, require_azure_cli)
+    heading, subheading, display, get_path_table, require_azure_cli, filter_by_git_diff)
 
 from .linter import LinterManager, LinterScope, RuleError
 from .util import filter_modules
@@ -26,7 +26,8 @@
 
 
 # pylint:disable=too-many-locals
-def run_linter(modules=None, rule_types=None, rules=None, ci_exclusions=None):
+def run_linter(modules=None, rule_types=None, rules=None, ci_exclusions=None,
+               git_source=None, git_target=None, git_repo=None, include_whl_extensions=False):
 
     require_azure_cli()
 
@@ -36,12 +37,27 @@ def run_linter(modules=None, rule_types=None, rules=None, ci_exclusions=None):
 
     heading('CLI Linter')
 
+    # allow user to run only on CLI or extensions
+    cli_only = modules == ['CLI']
+    ext_only = modules == ['EXT']
+    if cli_only or ext_only:
+        modules = None
+
     # needed to remove helps from azdev
     azdev_helps = helps.copy()
     exclusions = {}
-    selected_modules = get_path_table(include_only=modules)
+    selected_modules = get_path_table(include_only=modules, include_whl_extensions=include_whl_extensions)
+
+    if cli_only:
+        selected_modules['ext'] = {}
+    if ext_only:
+        selected_modules['mod'] = {}
+        selected_modules['core'] = {}
+
+    # filter down to only modules that have changed based on git diff
+    selected_modules = filter_by_git_diff(selected_modules, git_source, git_target, git_repo)
 
-    if not selected_modules:
+    if not any((selected_modules[x] for x in selected_modules)):
         raise CLIError('No modules selected.')
 
     selected_mod_names = list(selected_modules['mod'].keys()) + list(selected_modules['core'].keys()) + \
@@ -86,7 +102,7 @@ def run_linter(modules=None, rule_types=None, rules=None, ci_exclusions=None):
 
     # trim command table and help to just selected_modules
     command_loader, help_file_entries = filter_modules(
-        command_loader, help_file_entries, modules=selected_mod_names)
+        command_loader, help_file_entries, modules=selected_mod_names, include_whl_extensions=include_whl_extensions)
 
     if not command_loader.command_table:
         raise CLIError('No commands selected to check.')

azdev/operations/linter/util.py

@@ -9,7 +9,7 @@
 
 from knack.log import get_logger
 
-from azdev.utilities import COMMAND_MODULE_PREFIX
+from azdev.utilities import get_name_index
 
 
 logger = get_logger(__name__)
@@ -18,29 +18,30 @@
 _LOADER_CLS_RE = re.compile('.*azure/cli/command_modules/(?P<module>[^/]*)/__init__.*')
 
 
-def filter_modules(command_loader, help_file_entries, modules=None):
+def filter_modules(command_loader, help_file_entries, modules=None, include_whl_extensions=False):
     """ Modify the command table and help entries to only include certain modules/extensions.
 
     : param command_loader: The CLICommandsLoader containing the command table to filter.
     : help_file_entries: The dict of HelpFile entries to filter.
     : modules: [str] list of module or extension names to retain.
     """
-    return _filter_mods(command_loader, help_file_entries, modules=modules)
+    return _filter_mods(command_loader, help_file_entries, modules=modules,
+                        include_whl_extensions=include_whl_extensions)
 
 
-def exclude_commands(command_loader, help_file_entries, module_exclusions):
+def exclude_commands(command_loader, help_file_entries, module_exclusions, include_whl_extensions=False):
     """ Modify the command table and help entries to exclude certain modules/extensions.
 
     : param command_loader: The CLICommandsLoader containing the command table to filter.
     : help_file_entries: The dict of HelpFile entries to filter.
     : modules: [str] list of module or extension names to remove.
     """
-    return _filter_mods(command_loader, help_file_entries, modules=module_exclusions, exclude=True)
+    return _filter_mods(command_loader, help_file_entries, modules=module_exclusions, exclude=True,
+                        include_whl_extensions=include_whl_extensions)
 
 
-def _filter_mods(command_loader, help_file_entries, modules=None, exclude=False):
+def _filter_mods(command_loader, help_file_entries, modules=None, exclude=False, include_whl_extensions=False):
     modules = modules or []
-    modules = [x.replace(COMMAND_MODULE_PREFIX, '') for x in modules]
 
     # command tables and help entries must be copied to allow for seperate linter scope
     command_table = command_loader.command_table.copy()
@@ -49,6 +50,7 @@ def _filter_mods(command_loader, help_file_entries, modules=None, exclude=False)
     command_loader.command_table = command_table
     command_loader.command_group_table = command_group_table
     help_file_entries = help_file_entries.copy()
+    name_index = get_name_index(include_whl_extensions=include_whl_extensions)
 
     for command_name in list(command_loader.command_table.keys()):
         try:
@@ -58,7 +60,11 @@ def _filter_mods(command_loader, help_file_entries, modules=None, exclude=False)
             logger.warning(ex)
             source_name = None
 
-        is_specified = source_name in modules
+        try:
+            long_name = name_index[source_name]
+            is_specified = source_name in modules or long_name in modules
+        except KeyError:
+            is_specified = False
         if is_specified == exclude:
             # brute force method of ignoring commands from a module or extension
             command_loader.command_table.pop(command_name, None)