Merge pull request #4152 from APN-Pucky/develop

add support for --list-software --output-format=json

Author: boegel

easybuild/tools/docs.py

@@ -38,6 +38,7 @@
 """
 import copy
 import inspect
+import json
 import os
 from easybuild.tools import LooseVersion
 
@@ -72,6 +73,7 @@
 DETAILED = 'detailed'
 SIMPLE = 'simple'
 
+FORMAT_JSON = 'json'
 FORMAT_MD = 'md'
 FORMAT_RST = 'rst'
 FORMAT_TXT = 'txt'
@@ -115,6 +117,11 @@ def avail_cfgfile_constants(go_cfg_constants, output_format=FORMAT_TXT):
     return generate_doc('avail_cfgfile_constants_%s' % output_format, [go_cfg_constants])
 
 
+def avail_cfgfile_constants_json(go_cfg_constants):
+    """Generate documentation on constants for configuration files in json format"""
+    raise NotImplementedError("JSON output format not supported for avail_cfgfile_constants_json")
+
+
 def avail_cfgfile_constants_txt(go_cfg_constants):
     """Generate documentation on constants for configuration files in txt format"""
     doc = [
@@ -184,6 +191,11 @@ def avail_easyconfig_constants(output_format=FORMAT_TXT):
     return generate_doc('avail_easyconfig_constants_%s' % output_format, [])
 
 
+def avail_easyconfig_constants_json():
+    """Generate easyconfig constant documentation in json format"""
+    raise NotImplementedError("JSON output format not supported for avail_easyconfig_constants_json")
+
+
 def avail_easyconfig_constants_txt():
     """Generate easyconfig constant documentation in txt format"""
     doc = ["Constants that can be used in easyconfigs"]
@@ -242,6 +254,11 @@ def avail_easyconfig_licenses(output_format=FORMAT_TXT):
     return generate_doc('avail_easyconfig_licenses_%s' % output_format, [])
 
 
+def avail_easyconfig_licenses_json():
+    """Generate easyconfig license documentation in json format"""
+    raise NotImplementedError("JSON output format not supported for avail_easyconfig_licenses_json")
+
+
 def avail_easyconfig_licenses_txt():
     """Generate easyconfig license documentation in txt format"""
     doc = ["License constants that can be used in easyconfigs"]
@@ -354,6 +371,13 @@ def avail_easyconfig_params_rst(title, grouped_params):
     return '\n'.join(doc)
 
 
+def avail_easyconfig_params_json():
+    """
+    Compose overview of available easyconfig parameters, in json format.
+    """
+    raise NotImplementedError("JSON output format not supported for avail_easyconfig_params_json")
+
+
 def avail_easyconfig_params_txt(title, grouped_params):
     """
     Compose overview of available easyconfig parameters, in plain text format.
@@ -426,6 +450,11 @@ def avail_easyconfig_templates(output_format=FORMAT_TXT):
     return generate_doc('avail_easyconfig_templates_%s' % output_format, [])
 
 
+def avail_easyconfig_templates_json():
+    """ Returns template documentation in json text format """
+    raise NotImplementedError("JSON output format not supported for avail_easyconfig_templates")
+
+
 def avail_easyconfig_templates_txt():
     """ Returns template documentation in plain text format """
     # This has to reflect the methods/steps used in easyconfig _generate_template_values
@@ -640,6 +669,8 @@ def avail_classes_tree(classes, class_names, locations, detailed, format_strings
 
 
 def list_easyblocks(list_easyblocks=SIMPLE, output_format=FORMAT_TXT):
+    if output_format == FORMAT_JSON:
+        raise NotImplementedError("JSON output format not supported for list_easyblocks")
     format_strings = {
         FORMAT_MD: {
             'det_root_templ': "- **%s** (%s%s)",
@@ -1024,6 +1055,38 @@ def list_software_txt(software, detailed=False):
     return '\n'.join(lines)
 
 
+def list_software_json(software, detailed=False):
+    """
+    Return overview of supported software in json
+
+    :param software: software information (strucuted like list_software does)
+    :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
+    :return: multi-line string presenting requested info
+    """
+    lines = ['[']
+    for key in sorted(software, key=lambda x: x.lower()):
+        for entry in software[key]:
+            if detailed:
+                # deep copy here to avoid modifying the original dict
+                entry = copy.deepcopy(entry)
+                entry['description'] = ' '.join(entry['description'].split('\n')).strip()
+            else:
+                entry = {}
+            entry['name'] = key
+
+            lines.append(json.dumps(entry, indent=4, sort_keys=True, separators=(',', ': ')) + ",")
+            if not detailed:
+                break
+
+    # remove trailing comma on last line
+    if len(lines) > 1:
+        lines[-1] = lines[-1].rstrip(',')
+
+    lines.append(']')
+
+    return '\n'.join(lines)
+
+
 def list_toolchains(output_format=FORMAT_TXT):
     """Show list of known toolchains."""
     _, all_tcs = search_toolchain('')
@@ -1173,6 +1236,11 @@ def list_toolchains_txt(tcs):
     return '\n'.join(doc)
 
 
+def list_toolchains_json(tcs):
+    """ Returns overview of all toolchains in json format """
+    raise NotImplementedError("JSON output not implemented yet for --list-toolchains")
+
+
 def avail_toolchain_opts(name, output_format=FORMAT_TXT):
     """Show list of known options for given toolchain."""
     tc_class, _ = search_toolchain(name)
@@ -1226,6 +1294,11 @@ def avail_toolchain_opts_rst(name, tc_dict):
     return '\n'.join(doc)
 
 
+def avail_toolchain_opts_json(name, tc_dict):
+    """ Returns overview of toolchain options in jsonformat """
+    raise NotImplementedError("JSON output not implemented yet for --avail-toolchain-opts")
+
+
 def avail_toolchain_opts_txt(name, tc_dict):
     """ Returns overview of toolchain options in txt format """
     doc = ["Available options for %s toolchain:" % name]
@@ -1252,6 +1325,13 @@ def get_easyblock_classes(package_name):
     return easyblocks
 
 
+def gen_easyblocks_overview_json(package_name, path_to_examples, common_params=None, doc_functions=None):
+    """
+    Compose overview of all easyblocks in the given package in json format
+    """
+    raise NotImplementedError("JSON output not implemented yet for gen_easyblocks_overview")
+
+
 def gen_easyblocks_overview_md(package_name, path_to_examples, common_params=None, doc_functions=None):
     """
     Compose overview of all easyblocks in the given package in MarkDown format

easybuild/tools/options.py

@@ -79,7 +79,7 @@
 from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path
 from easybuild.tools.config import BuildOptions, ConfigurationVariables
 from easybuild.tools.configobj import ConfigObj, ConfigObjError
-from easybuild.tools.docs import FORMAT_MD, FORMAT_RST, FORMAT_TXT
+from easybuild.tools.docs import FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT
 from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses
 from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates
 from easybuild.tools.docs import list_easyblocks, list_toolchains
@@ -472,7 +472,8 @@ def override_options(self):
             'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True),
             'optarch': ("Set architecture optimization, overriding native architecture optimizations",
                         None, 'store', None),
-            'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT]),
+            'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT,
+                              [FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT]),
             'output-style': ("Control output style; auto implies using Rich if available to produce rich output, "
                              "with fallback to basic colored output",
                              'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES),

test/framework/docs.py

@@ -405,6 +405,104 @@
 ``1.4``|``GCC/4.6.3``, ``system``
 ``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
 
+LIST_SOFTWARE_SIMPLE_MD = """# List of supported software
+
+EasyBuild supports 2 different software packages (incl. toolchains, bundles):
+
+[g](#g)
+
+
+## G
+
+* GCC
+* gzip"""
+
+LIST_SOFTWARE_DETAILED_MD = """# List of supported software
+
+EasyBuild supports 2 different software packages (incl. toolchains, bundles):
+
+[g](#g)
+
+
+## G
+
+
+[GCC](#gcc) - [gzip](#gzip)
+
+
+### GCC
+
+%(gcc_descr)s
+
+*homepage*: <http://gcc.gnu.org/>
+
+version  |toolchain
+---------|----------
+``4.6.3``|``system``
+
+### gzip
+
+%(gzip_descr)s
+
+*homepage*: <http://www.gzip.org/>
+
+version|toolchain
+-------|-------------------------------
+``1.4``|``GCC/4.6.3``, ``system``
+``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+
+LIST_SOFTWARE_SIMPLE_JSON = """[
+{
+    "name": "GCC"
+},
+{
+    "name": "gzip"
+}
+]"""
+
+LIST_SOFTWARE_DETAILED_JSON = """[
+{
+    "description": "%(gcc_descr)s",
+    "homepage": "http://gcc.gnu.org/",
+    "name": "GCC",
+    "toolchain": "system",
+    "version": "4.6.3",
+    "versionsuffix": ""
+},
+{
+    "description": "%(gzip_descr)s",
+    "homepage": "http://www.gzip.org/",
+    "name": "gzip",
+    "toolchain": "GCC/4.6.3",
+    "version": "1.4",
+    "versionsuffix": ""
+},
+{
+    "description": "%(gzip_descr)s",
+    "homepage": "http://www.gzip.org/",
+    "name": "gzip",
+    "toolchain": "system",
+    "version": "1.4",
+    "versionsuffix": ""
+},
+{
+    "description": "%(gzip_descr)s",
+    "homepage": "http://www.gzip.org/",
+    "name": "gzip",
+    "toolchain": "foss/2018a",
+    "version": "1.5",
+    "versionsuffix": ""
+},
+{
+    "description": "%(gzip_descr)s",
+    "homepage": "http://www.gzip.org/",
+    "name": "gzip",
+    "toolchain": "intel/2018a",
+    "version": "1.5",
+    "versionsuffix": ""
+}
+]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+
 
 class DocsTest(EnhancedTestCase):
 
@@ -541,6 +639,9 @@ def test_license_docs(self):
         regex = re.compile(r"^``GPLv3``\s*|The GNU General Public License", re.M)
         self.assertTrue(regex.search(lic_docs), "%s found in: %s" % (regex.pattern, lic_docs))
 
+        # expect NotImplementedError for JSON output
+        self.assertRaises(NotImplementedError, avail_easyconfig_licenses, output_format='json')
+
     def test_list_easyblocks(self):
         """
         Tests for list_easyblocks function
@@ -569,6 +670,9 @@ def test_list_easyblocks(self):
         txt = list_easyblocks(list_easyblocks='detailed', output_format='md')
         self.assertEqual(txt, LIST_EASYBLOCKS_DETAILED_MD % {'topdir': topdir_easyblocks})
 
+        # expect NotImplementedError for JSON output
+        self.assertRaises(NotImplementedError, list_easyblocks, output_format='json')
+
     def test_list_software(self):
         """Test list_software* functions."""
         build_options = {
@@ -587,6 +691,9 @@ def test_list_software(self):
         self.assertEqual(list_software(output_format='md'), LIST_SOFTWARE_SIMPLE_MD)
         self.assertEqual(list_software(output_format='md', detailed=True), LIST_SOFTWARE_DETAILED_MD)
 
+        self.assertEqual(list_software(output_format='json'), LIST_SOFTWARE_SIMPLE_JSON)
+        self.assertEqual(list_software(output_format='json', detailed=True), LIST_SOFTWARE_DETAILED_JSON)
+
         # GCC/4.6.3 is installed, no gzip module installed
         txt = list_software(output_format='txt', detailed=True, only_installed=True)
         self.assertTrue(re.search(r'^\* GCC', txt, re.M))
@@ -690,6 +797,10 @@ def test_list_toolchains(self):
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst))
 
+        # expect NotImplementedError for json output format
+        with self.assertRaises(NotImplementedError):
+            list_toolchains(output_format='json')
+
     def test_avail_cfgfile_constants(self):
         """
         Test avail_cfgfile_constants to generate overview of constants that can be used in a configuration file.
@@ -734,6 +845,10 @@ def test_avail_cfgfile_constants(self):
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst))
 
+        # expect NotImplementedError for json output format
+        with self.assertRaises(NotImplementedError):
+            avail_cfgfile_constants(option_parser.go_cfg_constants, output_format='json')
+
     def test_avail_easyconfig_constants(self):
         """
         Test avail_easyconfig_constants to generate overview of constants that can be used in easyconfig files.
@@ -777,6 +892,10 @@ def test_avail_easyconfig_constants(self):
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst))
 
+        # expect NotImplementedError for json output format
+        with self.assertRaises(NotImplementedError):
+            avail_easyconfig_constants(output_format='json')
+
     def test_avail_easyconfig_templates(self):
         """
         Test avail_easyconfig_templates to generate overview of templates that can be used in easyconfig files.
@@ -827,6 +946,10 @@ def test_avail_easyconfig_templates(self):
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst))
 
+        # expect NotImplementedError for json output format
+        with self.assertRaises(NotImplementedError):
+            avail_easyconfig_templates(output_format='json')
+
     def test_avail_toolchain_opts(self):
         """
         Test avail_toolchain_opts to generate overview of supported toolchain options.
@@ -911,6 +1034,12 @@ def test_avail_toolchain_opts(self):
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst))
 
+        # expect NotImplementedError for json output format
+        with self.assertRaises(NotImplementedError):
+            avail_toolchain_opts('foss', output_format='json')
+        with self.assertRaises(NotImplementedError):
+            avail_toolchain_opts('intel', output_format='json')
+
     def test_mk_table(self):
         """
         Tests for mk_*_table functions.