diff --git a/bloom/config.py b/bloom/config.py index 1117f3a4..bc3469ee 100644 --- a/bloom/config.py +++ b/bloom/config.py @@ -201,6 +201,8 @@ def __str__(self): 'git-bloom-generate -y rosdebian --prefix release/:{ros_distro}' ' :{ros_distro} -i :{release_inc}', 'git-bloom-generate -y rosrpm --prefix release/:{ros_distro}' + ' :{ros_distro} -i :{release_inc}', + 'git-bloom-generate -y rosconda --prefix release/:{ros_distro}' ' :{ros_distro} -i :{release_inc}' ], [ @@ -216,6 +218,8 @@ def __str__(self): 'git-bloom-generate -y rosdebian --prefix release/:{ros_distro}' ' :{ros_distro} -i :{release_inc} --os-name debian --os-not-required', 'git-bloom-generate -y rosrpm --prefix release/:{ros_distro}' + ' :{ros_distro} -i :{release_inc}', + 'git-bloom-generate -y rosconda --prefix release/:{ros_distro}' ' :{ros_distro} -i :{release_inc}' ], [ diff --git a/bloom/generators/conda/__init__.py b/bloom/generators/conda/__init__.py new file mode 100644 index 00000000..05db8e81 --- /dev/null +++ b/bloom/generators/conda/__init__.py @@ -0,0 +1,4 @@ +from .generator import CondaGenerator +from .generator import sanitize_package_name + +__all__ = ['CondaGenerator', 'sanitize_package_name'] diff --git a/bloom/generators/conda/generate_cmd.py b/bloom/generators/conda/generate_cmd.py new file mode 100644 index 00000000..0d8ddebe --- /dev/null +++ b/bloom/generators/conda/generate_cmd.py @@ -0,0 +1,142 @@ +# Software License Agreement (BSD License) +# +# Copyright (c) 2013, Open Source Robotics Foundation, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Open Source Robotics Foundation, Inc. nor +# the names of its contributors may be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import print_function + +import os +import sys +import traceback + +from bloom.logging import debug +from bloom.logging import error +from bloom.logging import fmt +from bloom.logging import info + +from bloom.generators.conda.generator import generate_substitutions_from_package +from bloom.generators.conda.generator import place_template_files +from bloom.generators.conda.generator import process_template_files + +from bloom.util import get_distro_list_prompt + +try: + from rosdep2 import create_default_installer_context +except ImportError: + debug(traceback.format_exc()) + error("rosdep was not detected, please install it.", exit=True) + +try: + from catkin_pkg.packages import find_packages +except ImportError: + debug(traceback.format_exc()) + error("catkin_pkg was not detected, please install it.", exit=True) + + +def prepare_arguments(parser): + add = parser.add_argument + add('package_path', nargs='?', + help="path to or containing the package.xml of a package") + action = parser.add_mutually_exclusive_group(required=False) + add = action.add_argument + add('--place-template-files', action='store_true', + help="places conda/* template file(s) only") + add('--process-template-files', action='store_true', + help="processes templates in conda/* only") + add = parser.add_argument + add('--ros-distro', help="ROS distro, e.g. %s (used for rosdep)" % get_distro_list_prompt()) + return parser + + +def get_subs(pkg, ros_distro): + return generate_substitutions_from_package( + pkg, + ros_distro + ) + + +def main(args=None, get_subs_fn=None): + get_subs_fn = get_subs_fn or get_subs + _place_template_files = True + _process_template_files = True + package_path = os.getcwd() + if args is not None: + package_path = args.package_path or os.getcwd() + _place_template_files = args.place_template_files + _process_template_files = args.process_template_files + + pkgs_dict = find_packages(package_path) + if len(pkgs_dict) == 0: + sys.exit("No packages found in path: '{0}'".format(package_path)) + if len(pkgs_dict) > 1: + sys.exit("Multiple packages found, " + "this tool only supports one package at a time.") + + ros_distro = os.environ.get('ROS_DISTRO', 'indigo') + + # Allow args overrides + ros_distro = args.ros_distro or ros_distro + + # Summarize + info(fmt("@!@{gf}==> @|") + + fmt("Generating Conda recipes for package(s) %s" % + ([p.name for p in pkgs_dict.values()]))) + + for path, pkg in pkgs_dict.items(): + template_files = None + try: + subs = get_subs_fn(pkg, ros_distro) + if _place_template_files: + # Place template files + place_template_files(path, pkg.get_build_type()) + if _process_template_files: + # Just process existing template files + template_files = process_template_files(path, subs) + if not _place_template_files and not _process_template_files: + # If neither, do both + place_template_files(path, pkg.get_build_type()) + template_files = process_template_files(path, subs) + if template_files is not None: + for template_file in template_files: + os.remove(os.path.normpath(template_file)) + except Exception as exc: + debug(traceback.format_exc()) + error(type(exc).__name__ + ": " + str(exc), exit=True) + except (KeyboardInterrupt, EOFError): + sys.exit(1) + +# This describes this command to the loader +description = dict( + title='conda', + description="Generates conda recipes for a catkin package", + main=main, + prepare_arguments=prepare_arguments +) diff --git a/bloom/generators/conda/generator.py b/bloom/generators/conda/generator.py new file mode 100644 index 00000000..00a69ae9 --- /dev/null +++ b/bloom/generators/conda/generator.py @@ -0,0 +1,798 @@ +# Software License Agreement (BSD License) +# +# Copyright (c) 2013, Willow Garage, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Willow Garage, Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import print_function + +import collections +import datetime +import io +import json +import os +import pkg_resources +import re +import shutil +import sys +import traceback +import textwrap + +# Python 2/3 support. +try: + from configparser import SafeConfigParser +except ImportError: + from ConfigParser import SafeConfigParser +from dateutil import tz +from distutils.version import LooseVersion +from time import strptime + +from bloom.generators import BloomGenerator +from bloom.generators import GeneratorError +from bloom.generators import resolve_dependencies +from bloom.generators import update_rosdep + +from bloom.generators.common import default_fallback_resolver +from bloom.generators.common import invalidate_view_cache +from bloom.generators.common import evaluate_package_conditions +from bloom.generators.common import resolve_rosdep_key + +from bloom.git import inbranch +from bloom.git import get_branches +from bloom.git import get_commit_hash +from bloom.git import get_current_branch +from bloom.git import has_changes +from bloom.git import show +from bloom.git import tag_exists + +from bloom.logging import ansi +from bloom.logging import debug +from bloom.logging import enable_drop_first_log_prefix +from bloom.logging import error +from bloom.logging import fmt +from bloom.logging import info +from bloom.logging import warning + +from bloom.commands.git.patch.common import get_patch_config +from bloom.commands.git.patch.common import set_patch_config + +from bloom.packages import get_package_data + +from bloom.util import code +from bloom.util import execute_command +from bloom.util import maybe_continue + +try: + import rosdistro +except ImportError as err: + debug(traceback.format_exc()) + error("rosdistro was not detected, please install it.", exit=True) + +try: + import em +except ImportError: + debug(traceback.format_exc()) + error("empy was not detected, please install it.", exit=True) + +# Drop the first log prefix for this command +enable_drop_first_log_prefix(True) + +TEMPLATE_EXTENSION = '.em' + + +def __place_template_folder(group, src, dst, gbp=False): + template_files = pkg_resources.resource_listdir(group, src) + # For each template, place + for template_file in template_files: + template_path = os.path.join(src, template_file) + template_dst = os.path.join(dst, template_file) + if pkg_resources.resource_isdir(group, template_path): + debug("Recursing on folder '{0}'".format(template_path)) + __place_template_folder(group, template_path, template_dst, gbp) + else: + try: + debug("Placing template '{0}'".format(template_path)) + template = pkg_resources.resource_string(group, template_path) + template_abs_path = pkg_resources.resource_filename(group, template_path) + except IOError as err: + error("Failed to load template " + "'{0}': {1}".format(template_file, str(err)), exit=True) + if not os.path.exists(dst): + os.makedirs(dst) + if os.path.exists(template_dst): + debug("Removing existing file '{0}'".format(template_dst)) + os.remove(template_dst) + with open(template_dst, 'w') as f: + if not isinstance(template, str): + template = template.decode('utf-8') + f.write(template) + shutil.copystat(template_abs_path, template_dst) + + +def place_template_files(path, build_type, gbp=False): + info(fmt("@!@{bf}==>@| Placing templates files in the 'conda' folder.")) + recipe_path = os.path.join(path, 'conda') + # Create/Clean the conda folder + if not os.path.exists(recipe_path): + os.makedirs(recipe_path) + # Place template files + group = 'bloom.generators.conda' + templates = os.path.join('templates', build_type) + __place_template_folder(group, templates, recipe_path, gbp) + + +def summarize_dependency_mapping(data, deps, build_deps, resolved_deps): + if len(deps) == 0 and len(build_deps) == 0: + return + info("Package '" + data['Package'] + "' has dependencies:") + header = " " + ansi('boldoff') + ansi('ulon') + \ + "rosdep key => " + data['OSName'] + ' ' + \ + data['Distribution'] + " key" + ansi('reset') + template = " " + ansi('cyanf') + "{0:<20} " + ansi('purplef') + \ + "=> " + ansi('cyanf') + "{1}" + ansi('reset') + if len(deps) != 0: + info(ansi('purplef') + "Run Dependencies:" + + ansi('reset')) + info(header) + for key in [d.name for d in deps]: + info(template.format(key, resolved_deps[key])) + if len(build_deps) != 0: + info(ansi('purplef') + + "Build and Build Tool Dependencies:" + ansi('reset')) + info(header) + for key in [d.name for d in build_deps]: + info(template.format(key, resolved_deps[key])) + + +def format_depends(depends, resolved_deps): + versions = { + 'version_lt': '<', + 'version_lte': '<=', + 'version_eq': '=', + 'version_gte': '>=', + 'version_gt': '>' + } + formatted = [] + for d in depends: + for resolved_dep in resolved_deps[d.name]: + version_depends = [k + for k in versions.keys() + if getattr(d, k, None) is not None] + if not version_depends: + formatted.append(resolved_dep) + else: + for v in version_depends: + formatted.append("{0} {1} {2}".format( + resolved_dep, versions[v], getattr(d, v))) + return formatted + + +def missing_dep_resolver(key, peer_packages): + if key in peer_packages: + return [sanitize_package_name(key)] + return default_fallback_resolver(key, peer_packages) + + +def generate_substitutions_from_package( + package, + ros_distro, + conda_inc=0, + peer_packages=None, + releaser_history=None, + fallback_resolver=None, + skip_keys=None +): + peer_packages = peer_packages or [] + skip_keys = skip_keys or set() + data = {} + # Name, Version, Description + data['Name'] = package.name + data['Version'] = package.version + data['Description'] = rpmify_string(package.description) + # License + if not package.licenses or not package.licenses[0]: + error("No license set for package '{0}', aborting.".format(package.name), exit=True) + data['License'] = package.licenses[0] + # Websites + websites = [str(url) for url in package.urls if url.type == 'website'] + data['Homepage'] = websites[0] if websites else '' + if data['Homepage'] == '': + warning("No homepage set") + # RPM Increment Number + data['CONDAInc'] = conda_inc + # Package name + data['Package'] = sanitize_package_name(package.name) + # Resolve dependencies + evaluate_package_conditions(package, ros_distro) + depends = [ + dep for dep in (package.run_depends + package.buildtool_export_depends) + if dep.evaluated_condition is not False and dep.name not in skip_keys] + build_depends = [ + dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) + if dep.evaluated_condition is not False and dep.name not in skip_keys] + replaces = [ + dep for dep in package.replaces + if dep.evaluated_condition is not False] + conflicts = [ + dep for dep in package.conflicts + if dep.evaluated_condition is not False] + unresolved_keys = depends + build_depends + replaces + conflicts + # The installer key is not considered here, but it is checked when the keys are checked before this + resolved_deps = resolve_dependencies(unresolved_keys, ros_distro, + peer_packages + [d.name for d in (replaces + conflicts)], + fallback_resolver) + data['Depends'] = sorted( + set(format_depends(depends, resolved_deps)) + ) + data['BuildDepends'] = sorted( + set(format_depends(build_depends, resolved_deps)) + ) + data['Replaces'] = sorted( + set(format_depends(replaces, resolved_deps)) + ) + data['Conflicts'] = sorted( + set(format_depends(conflicts, resolved_deps)) + ) + data['Provides'] = [] + data['Supplements'] = [] + + # Build-type specific substitutions. + build_type = package.get_build_type() + if build_type == 'catkin': + pass + elif build_type == 'cmake': + pass + elif build_type == 'ament_cmake': + pass + elif build_type == 'ament_python': + pass + else: + error( + "Build type '{}' is not supported by this version of bloom.". + format(build_type), exit=True) + + # Use the time stamp to set the date strings + stamp = datetime.datetime.now(tz.tzlocal()) + data['Date'] = stamp.strftime('%a %b %d %Y') + # Maintainers + maintainers = [] + for m in package.maintainers: + maintainers.append(str(m)) + data['Maintainer'] = maintainers[0] + data['Maintainers'] = ', '.join(maintainers) + # Changelog + if releaser_history: + sorted_releaser_history = sorted(releaser_history, + key=lambda k: LooseVersion(k), reverse=True) + sorted_releaser_history = sorted(sorted_releaser_history, + key=lambda k: strptime(releaser_history.get(k)[0], '%a %b %d %Y'), + reverse=True) + changelogs = [(v, releaser_history[v]) for v in sorted_releaser_history] + else: + # Ensure at least a minimal changelog + changelogs = [] + if package.version + '-' + str(conda_inc) not in [x[0] for x in changelogs]: + changelogs.insert(0, ( + package.version + '-' + str(conda_inc), ( + data['Date'], + package.maintainers[0].name, + package.maintainers[0].email + ) + )) + exported_tags = [e.tagname for e in package.exports] + data['NoArch'] = 'metapackage' in exported_tags or 'architecture_independent' in exported_tags + data['changelogs'] = changelogs + # Summarize dependencies + summarize_dependency_mapping(data, depends, build_depends, resolved_deps) + + def convertToUnicode(obj): + if sys.version_info.major == 2: + if isinstance(obj, str): + return unicode(obj.decode('utf8')) + elif isinstance(obj, unicode): + return obj + else: + if isinstance(obj, bytes): + return str(obj.decode('utf8')) + elif isinstance(obj, str): + return obj + if isinstance(obj, list): + for i, val in enumerate(obj): + obj[i] = convertToUnicode(val) + return obj + elif isinstance(obj, type(None)): + return None + elif isinstance(obj, tuple): + obj_tmp = list(obj) + for i, val in enumerate(obj_tmp): + obj_tmp[i] = convertToUnicode(obj_tmp[i]) + return tuple(obj_tmp) + elif isinstance(obj, int): + return obj + elif isinstance(obj, int): + return obj + raise RuntimeError('need to deal with type %s' % (str(type(obj)))) + + for item in data.items(): + data[item[0]] = convertToUnicode(item[1]) + + return data + + +def __process_template_folder(path, subs): + items = os.listdir(path) + processed_items = [] + for item in list(items): + item = os.path.abspath(os.path.join(path, item)) + if os.path.basename(item) in ['.', '..', '.git', '.svn']: + continue + if os.path.isdir(item): + sub_items = __process_template_folder(item, subs) + processed_items.extend([os.path.join(item, s) for s in sub_items]) + if not item.endswith(TEMPLATE_EXTENSION): + continue + with open(item, 'r') as f: + template = f.read() + # Remove extension + template_path = item[:-len(TEMPLATE_EXTENSION)] + # Expand template + info("Expanding '{0}' -> '{1}'".format( + os.path.relpath(item), + os.path.relpath(template_path))) + result = em.expand(template, **subs) + # Write the result + with io.open(template_path, 'w', encoding='utf-8') as f: + if sys.version_info.major == 2: + result = result.decode('utf-8') + f.write(result) + # Copy the permissions + shutil.copymode(item, template_path) + processed_items.append(item) + return processed_items + + +def process_template_files(path, subs): + info(fmt("@!@{bf}==>@| In place processing templates in 'conda' folder.")) + recipe_dir = os.path.join(path, 'conda') + if not os.path.exists(recipe_dir): + sys.exit("No conda directory found at '{0}', cannot process templates." + .format(recipe_dir)) + return __process_template_folder(recipe_dir, subs) + + +def match_branches_with_prefix(prefix, get_branches, prune=False): + debug("match_branches_with_prefix(" + str(prefix) + ", " + + str(get_branches()) + ")") + branches = [] + # Match branches + existing_branches = get_branches() + for branch in existing_branches: + if branch.startswith('remotes/origin/'): + branch = branch.split('/', 2)[-1] + if branch.startswith(prefix): + branches.append(branch) + branches = list(set(branches)) + if prune: + # Prune listed branches by packages in latest upstream + with inbranch('upstream'): + pkg_names, version, pkgs_dict = get_package_data('upstream') + for branch in branches: + if branch.split(prefix)[-1].strip('/') not in pkg_names: + branches.remove(branch) + return branches + + +def get_package_from_branch(branch): + with inbranch(branch): + try: + package_data = get_package_data(branch) + except SystemExit: + return None + if type(package_data) not in [list, tuple]: + # It is a ret code + CondaGenerator.exit(package_data) + names, version, packages = package_data + if type(names) is list and len(names) > 1: + CondaGenerator.exit( + "RPM generator does not support generating " + "from branches with multiple packages in them, use " + "the release generator first to split packages into " + "individual branches.") + if type(packages) is dict: + return list(packages.values())[0] + + +def rpmify_string(value): + markup_remover = re.compile(r'<.*?>') + value = markup_remover.sub('', value) + value = re.sub('\s+', ' ', value) + value = '\n'.join([v.strip() for v in + textwrap.TextWrapper(width=80, break_long_words=False, replace_whitespace=False).wrap(value)]) + return value + + +def sanitize_package_name(name): + return name.replace('_', '-') + + +class CondaGenerator(BloomGenerator): + title = 'conda' + description = "Generates conda recipes from the catkin meta data" + has_run_rosdep = os.environ.get('BLOOM_SKIP_ROSDEP_UPDATE', '0').lower() not in ['0', 'f', 'false', 'n', 'no'] + rosdistro = os.environ.get('ROS_DISTRO', 'indigo') + + def prepare_arguments(self, parser): + # Add command line arguments for this generator + add = parser.add_argument + add('-i', '--conda-inc', help="Conda increment number", default='0') + add('-p', '--prefix', required=True, + help="branch prefix to match, and from which create conda recipes" + " hint: if you want to match 'release/foo' use 'release'") + add('-a', '--match-all', default=False, action="store_true", + help="match all branches with the given prefix, " + "even if not in current upstream") + add('--skip-keys', nargs='+', required=False, default=[], + help="dependency keys which should be skipped and" + " discluded from the conda dependencies") + + def handle_arguments(self, args): + self.interactive = args.interactive + self.conda_inc = args.conda_inc + self.skip_keys = args.skip_keys or set() + self.prefix = args.prefix + self.branches = match_branches_with_prefix(self.prefix, get_branches, prune=not args.match_all) + if len(self.branches) == 0: + error( + "No packages found, check your --prefix or --src arguments.", + exit=True + ) + self.packages = {} + self.tag_names = {} + self.names = [] + self.branch_args = [] + self.conda_branches = [] + for branch in self.branches: + package = get_package_from_branch(branch) + if package is None: + # This is an ignored package + continue + self.packages[package.name] = package + self.names.append(package.name) + args = self.generate_branching_arguments(package, branch) + # First branch is conda/[/] + self.conda_branches.append(args[0][0]) + self.branch_args.extend(args) + + def summarize(self): + info("Generating source conda recipes for the packages: " + str(self.names)) + info("Conda Incremental Version (Build Number): " + str(self.conda_inc)) + + def get_branching_arguments(self): + return self.branch_args + + def update_rosdep(self): + update_rosdep() + self.has_run_rosdep = True + + def _check_all_keys_are_valid(self, peer_packages, rosdistro): + keys_to_resolve = set() + key_to_packages_which_depends_on = collections.defaultdict(list) + keys_to_ignore = set() + for package in self.packages.values(): + evaluate_package_conditions(package, rosdistro) + depends = [ + dep for dep in (package.run_depends + package.buildtool_export_depends) + if dep.evaluated_condition is not False] + build_depends = [ + dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) + if dep.evaluated_condition is not False] + unresolved_keys = [ + dep for dep in (depends + build_depends + package.replaces + package.conflicts) + if dep.evaluated_condition is not False] + keys_to_ignore = { + dep for dep in keys_to_ignore.union(package.replaces + package.conflicts) + if dep.evaluated_condition is not False} + keys = [d.name for d in unresolved_keys] + keys_to_resolve.update(keys) + for key in keys: + key_to_packages_which_depends_on[key].append(package.name) + + for skip_key in self.skip_keys: + try: + keys_to_resolve.remove(skip_key) + except KeyError: + warning("Key '{0}' specified by --skip-keys was not found".format(skip_key)) + else: + warning("Skipping dependency key '{0}' per --skip-keys".format(skip_key)) + + rosdistro = self.rosdistro + all_keys_valid = True + for key in sorted(keys_to_resolve): + try: + extended_peer_packages = peer_packages + [d.name for d in keys_to_ignore] + rule, installer_key, default_installer_key = \ + resolve_rosdep_key(key, rosdistro, extended_peer_packages, + retry=False) + if rule is None: + continue + if installer_key != default_installer_key: + error("Key '{0}' resolved to '{1}' with installer '{2}', " + "which does not match the default installer '{3}'." + .format(key, rule, installer_key, default_installer_key)) + BloomGenerator.exit( + "The RPM generator does not support dependencies " + "which are installed with the '{0}' installer." + .format(installer_key), + returncode=code.GENERATOR_INVALID_INSTALLER_KEY) + except (GeneratorError, RuntimeError) as e: + print(fmt("Failed to resolve @{cf}@!{key}@| with: {e}") + .format(**locals())) + print(fmt("@{cf}@!{0}@| is depended on by these packages: ").format(key) + + str(list(set(key_to_packages_which_depends_on[key])))) + print(fmt("@{kf}@!<== @{rf}@!Failed@|")) + all_keys_valid = False + return all_keys_valid + + def pre_modify(self): + info("\nPre-verifying RPM dependency keys...") + # Run rosdep update is needed + if not self.has_run_rosdep: + self.update_rosdep() + + peer_packages = [p.name for p in self.packages.values()] + + while not self._check_all_keys_are_valid(peer_packages, self.rosdistro): + error("Some of the dependencies for packages in this repository could not be resolved by rosdep.") + if not self.interactive: + sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) + error("You can try to address the issues which appear above and try again if you wish, " + "or continue without releasing into RPM-based distributions (e.g. Fedora 24).") + try: + if not maybe_continue(msg="Would you like to try again?"): + error("User aborted after rosdep keys were not resolved.") + sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) + except (KeyboardInterrupt, EOFError): + error("\nUser quit.", exit=True) + update_rosdep() + invalidate_view_cache() + + info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") + + for package in self.packages.values(): + if not package.licenses or not package.licenses[0]: + error("No license set for package '{0}', aborting.".format(package.name), exit=True) + + def pre_branch(self, destination, source): + if destination in self.conda_branches: + return + # Run rosdep update is needed + if not self.has_run_rosdep: + self.update_rosdep() + # Determine the current package being generated + name = destination.split('/')[-1] + distro = destination.split('/')[-2] + # Retrieve the package + package = self.packages[name] + # Report on this package + self.summarize_package(package, distro) + + def pre_rebase(self, destination): + # Get the stored configs is any + patches_branch = 'patches/' + destination + config = self.load_original_config(patches_branch) + if config is not None: + curr_config = get_patch_config(patches_branch) + if curr_config['parent'] == config['parent']: + set_patch_config(patches_branch, config) + + def post_rebase(self, destination): + name = destination.split('/')[-1] + # Retrieve the package + package = self.packages[name] + # Handle differently if this is an rpm vs distro branch + if destination in self.rpm_branches: + info("Placing RPM template files into '{0}' branch." + .format(destination)) + # Then this is an rpm branch + # Place the raw template files + self.place_template_files(package.get_build_type()) + else: + # This is a distro specific rpm branch + # Determine the current package being generated + distro = destination.split('/')[-2] + # Create RPMs for each distro + with inbranch(destination): + data = self.generate_conda_recipe(package) + # Create the tag name for later + self.tag_names[destination] = self.generate_tag_name(data) + # Update the patch configs + patches_branch = 'patches/' + destination + config = get_patch_config(patches_branch) + # Store it + self.store_original_config(config, patches_branch) + # Modify the base so import/export patch works + current_branch = get_current_branch() + if current_branch is None: + error("Could not determine current branch.", exit=True) + config['base'] = get_commit_hash(current_branch) + # Set it + set_patch_config(patches_branch, config) + + def post_patch(self, destination, color='bluef'): + if destination in self.conda_branches: + return + # Tag after patches have been applied + with inbranch(destination): + # Tag + tag_name = self.tag_names[destination] + if tag_exists(tag_name): + if self.interactive: + warning("Tag exists: " + tag_name) + warning("Do you wish to overwrite it?") + if not maybe_continue('y'): + error("Answered no to continue, aborting.", exit=True) + else: + warning("Overwriting tag: " + tag_name) + else: + info("Creating tag: " + tag_name) + execute_command('git tag -f ' + tag_name) + # Report of success + name = destination.split('/')[-1] + package = self.packages[name] + distro = destination.split('/')[-2] + info(ansi(color) + "####" + ansi('reset'), use_prefix=False) + info( + ansi(color) + "#### " + ansi('greenf') + "Successfully" + + ansi(color) + " generated '" + ansi('boldon') + + + distro + ansi('boldoff') + "' RPM for package" + " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + + " at version '" + ansi('boldon') + package.version + + "-" + str(self.conda_inc) + ansi('boldoff') + "'" + + ansi('reset'), + use_prefix=False + ) + info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False) + + def store_original_config(self, config, patches_branch): + with inbranch(patches_branch): + with open('conda.store', 'w+') as f: + f.write(json.dumps(config)) + execute_command('git add conda.store') + if has_changes(): + execute_command('git commit -m "Store original patch config"') + + def load_original_config(self, patches_branch): + config_store = show(patches_branch, 'conda.store') + if config_store is None: + return config_store + return json.loads(config_store) + + def place_template_files(self, build_type, recipe_dir='conda'): + # Create/Clean the recipe folder + if os.path.exists(recipe_dir): + if self.interactive: + warning("recipe directory exists: " + recipe_dir) + warning("Do you wish to overwrite it?") + if not maybe_continue('y'): + error("Answered no to continue, aborting.", exit=True) + else: + warning("Overwriting recipe directory: " + recipe_dir) + execute_command('git rm -rf ' + recipe_dir) + execute_command('git commit -m "Clearing previous recipe folder"') + if os.path.exists(recipe_dir): + shutil.rmtree(recipe_dir) + # Use generic place template files command + place_template_files('.', build_type, gbp=True) + # Commit results + execute_command('git add ' + recipe_dir) + execute_command('git commit -m "Placing conda template files"') + + def get_releaser_history(self): + # Assumes that this is called in the target branch + patches_branch = 'patches/' + get_current_branch() + raw = show(patches_branch, 'releaser_history.json') + return None if raw is None else json.loads(raw) + + def set_releaser_history(self, history): + # Assumes that this is called in the target branch + patches_branch = 'patches/' + get_current_branch() + debug("Writing release history to '{0}' branch".format(patches_branch)) + with inbranch(patches_branch): + with open('releaser_history.json', 'w') as f: + f.write(json.dumps(history)) + execute_command('git add releaser_history.json') + if has_changes(): + execute_command('git commit -m "Store releaser history"') + + def get_subs(self, package, releaser_history=None): + return generate_substitutions_from_package( + package, + self.rosdistro, + self.install_prefix, + self.conda_inc, + [p.name for p in self.packages.values()], + releaser_history=releaser_history, + fallback_resolver=missing_dep_resolver, + skip_keys=self.skip_keys + ) + + def generate_conda_recipe(self, package, recipe_dir='conda'): + info("Generating Conda recipes...") + # Try to retrieve the releaser_history + releaser_history = self.get_releaser_history() + # Generate substitution values + subs = self.get_subs(package, releaser_history) + # Use subs to create and store releaser history + self.set_releaser_history(dict(subs['changelogs'])) + # Template files + template_files = process_template_files('.', subs) + # Remove any residual template files + execute_command('git rm -rf ' + ' '.join("'{}'".format(t) for t in template_files)) + # Add marker file to tell mock to archive the sources + open('.write_tar', 'a').close() + # Add marker file changes to the conda folder + execute_command('git add .write_tar ' + recipe_dir) + # Commit changes + execute_command('git commit -m "Generated Conda recipes') + # Rename the template spec file + execute_command('git mv ' + recipe_dir + '/template.spec ' + recipe_dir + '/' + subs['Package'] + '.spec') + # Commit changes + execute_command('git commit -m "Renamed Conda spec file') + # Return the subs for other use + return subs + + def generate_tag_name(self, data): + tag_name = '{Package}-{Version}-{CONDAInc}' + tag_name = 'conda/' + tag_name.format(**data) + return tag_name + + def generate_branching_arguments(self, package, branch): + n = package.name + # conda branch + conda_branch = 'conda/' + n + # Branch first to the conda branch + args = [[conda_branch, branch, False]] + # Then for each RPM distro, branch from the base conda branch + # args.extend([ + # ['conda/' + d + '/' + n, rpm_branch, False] for d in self.distros + # ]) + return args + + def summarize_package(self, package, distro, color='bluef'): + info(ansi(color) + "\n####" + ansi('reset'), use_prefix=False) + info( + ansi(color) + "#### Generating '" + ansi('boldon') + + ' ' + distro + ansi('boldoff') + "' RPM for package" + " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + + " at version '" + ansi('boldon') + package.version + + "-" + str(self.conda_inc) + ansi('boldoff') + "'" + + ansi('reset'), + use_prefix=False + ) + info(ansi(color) + "####" + ansi('reset'), use_prefix=False) diff --git a/bloom/generators/conda/templates/ament_cmake/template.spec.em b/bloom/generators/conda/templates/ament_cmake/template.spec.em new file mode 100644 index 00000000..3c8f1654 --- /dev/null +++ b/bloom/generators/conda/templates/ament_cmake/template.spec.em @@ -0,0 +1,39 @@ +package: + name: @(Package) + version: @(Version) +source: + path: @(Package)/src/work + +build: + script: + sel(win): bld_ament_cmake.bat + sel(unix): build_ament_cmake.sh + number: @(CONDAInc) +about: + @[if Homepage and Homepage != '']home: @(Homepage)@\n@[end if]@ + license: @(License) + summary: ROS @(Name) package + +extra: + recipe-maintainers: + - ros-forge + +requirements: + build: + - "{{ compiler('cxx') }}" + - "{{ compiler('c') }}" + - ninja + - sel(unix): make + - sel(osx): tapi + - cmake + - sel(build_platform != target_platform): python + - sel(build_platform != target_platform): cross-python_{{ target_platform }} + - sel(build_platform != target_platform): cython + host: + - python + @[for p in Depends] - @p@\n@[end for]@ + @[for p in BuildDepends] - @p@\n@[end for]@ + run: + - python + @[for p in Depends] - @p@\n@[end for]@ + - sel(osx and x86_64): __osx >={{ MACOSX_DEPLOYMENT_TARGET|default('10.14') }} diff --git a/bloom/generators/conda/templates/ament_python/template.spec.em b/bloom/generators/conda/templates/ament_python/template.spec.em new file mode 100644 index 00000000..e69de29b diff --git a/bloom/generators/conda/templates/catkin/template.spec.em b/bloom/generators/conda/templates/catkin/template.spec.em new file mode 100644 index 00000000..e69de29b diff --git a/bloom/generators/conda/templates/cmake/template.spec.em b/bloom/generators/conda/templates/cmake/template.spec.em new file mode 100644 index 00000000..e69de29b diff --git a/bloom/generators/rosconda.py b/bloom/generators/rosconda.py new file mode 100644 index 00000000..7a0d1bc2 --- /dev/null +++ b/bloom/generators/rosconda.py @@ -0,0 +1,165 @@ +# Software License Agreement (BSD License) +# +# Copyright (c) 2021, Open Source Robotics Foundation, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Open Source Robotics Foundation, Inc. nor +# the names of its contributors may be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import print_function + +from bloom.generators.common import default_fallback_resolver + +from bloom.generators.conda.generator import sanitize_package_name + +from bloom.generators.conda import CondaGenerator +from bloom.generators.conda.generator import generate_substitutions_from_package +from bloom.generators.conda.generate_cmd import main as conda_main +from bloom.generators.conda.generate_cmd import prepare_arguments + +from bloom.logging import info + +from bloom.rosdistro_api import get_index + +from bloom.util import get_distro_list_prompt + + +class RosCondaGenerator(CondaGenerator): + title = 'rosconda' + description = "Generates conda recipes tailored for the given rosdistro" + + def prepare_arguments(self, parser): + # Add command line arguments for this generator + add = parser.add_argument + add('rosdistro', help="ROS distro to target (%s, etc.)" % get_distro_list_prompt()) + return CondaGenerator.prepare_arguments(self, parser) + + def handle_arguments(self, args): + self.rosdistro = args.rosdistro + ret = CondaGenerator.handle_arguments(self, args) + return ret + + def summarize(self): + ret = CondaGenerator.summarize(self) + info("Releasing for rosdistro: " + self.rosdistro) + return ret + + def get_subs(self, package, releaser_history): + def fallback_resolver(key, peer_packages, rosdistro=self.rosdistro): + if key in peer_packages: + return [sanitize_package_name(rosify_package_name(key, rosdistro))] + return default_fallback_resolver(key, peer_packages) + subs = generate_substitutions_from_package( + package, + self.rosdistro, + self.conda_inc, + [p.name for p in self.packages.values()], + releaser_history=releaser_history, + fallback_resolver=fallback_resolver, + skip_keys=self.skip_keys + ) + subs['Rosdistro'] = self.rosdistro + subs['Package'] = rosify_package_name(subs['Package'], self.rosdistro) + + # Virtual packages + subs['Provides'] += [ + '%%{name}-%s = %%{version}-%%{release}' % subpackage for subpackage in [ + 'devel', 'doc', 'runtime']] + + # Group membership + subs['Provides'].extend( + sanitize_package_name(rosify_package_name(g.name, self.rosdistro)) + + '(member)' for g in package.member_of_groups) + subs['Supplements'].extend( + sanitize_package_name(rosify_package_name(g.name, self.rosdistro)) + + '(all)' for g in package.member_of_groups) + + # ROS 2 specific bloom extensions. + ros2_distros = [ + name for name, values in get_index().distributions.items() + if values.get('distribution_type') == 'ros2'] + if self.rosdistro in ros2_distros: + # Add ros-workspace package as a dependency to any package other + # than ros_workspace and its dependencies. + if package.name not in ['ament_cmake_core', 'ament_package', 'ros_workspace']: + workspace_pkg_name = rosify_package_name('ros-workspace', self.rosdistro) + subs['BuildDepends'].append(workspace_pkg_name) + subs['Depends'].append(workspace_pkg_name) + + # Add packages necessary to build vendor typesupport for rosidl_interface_packages to their + # build dependencies. + if self.rosdistro in ros2_distros and \ + 'rosidl_interface_packages' in [p.name for p in package.member_of_groups]: + ROS2_VENDOR_TYPESUPPORT_DEPENDENCIES = [ + 'rosidl-typesupport-fastrtps-c', + 'rosidl-typesupport-fastrtps-cpp', + ] + + subs['BuildDepends'] += [ + rosify_package_name(name, self.rosdistro) for name in ROS2_VENDOR_TYPESUPPORT_DEPENDENCIES] + return subs + + def generate_branching_arguments(self, package, branch): + conda_branch = 'conda/' + self.rosdistro + '/' + package.name + args = [[conda_branch, branch, False]] + n, r, b, ds = package.name, self.rosdistro, conda_branch, self.distros + args.extend([ + ['conda/' + r + '/' + d + '/' + n, b, False] for d in ds + ]) + return args + + def get_release_tag(self, data): + return 'release/{0}/{1}/{2}-{3}'\ + .format(self.rosdistro, data['Name'], data['Version'], self.conda_inc) + + +def rosify_package_name(name, rosdistro): + return 'ros-{0}-{1}'.format(rosdistro, name) + + +def get_subs(pkg, ros_distro): + # No fallback_resolver provided because peer packages not considered. + subs = generate_substitutions_from_package( + pkg, + ros_distro + ) + subs['Package'] = rosify_package_name(subs['Package'], ros_distro) + return subs + + +def main(args=None): + conda_main(args, get_subs) + + +# This describes this command to the loader +description = dict( + title='rosconda', + description="Generates ROS style Conda recipes for a catkin package", + main=main, + prepare_arguments=prepare_arguments +) diff --git a/setup.py b/setup.py index 2a6208f7..64209a1b 100755 --- a/setup.py +++ b/setup.py @@ -31,6 +31,9 @@ ], 'bloom.generators.rpm': [ 'bloom/generators/rpm/templates/*' + ], + 'bloom.generators.conda': [ + 'bloom/generators/conda/templates/*' ] }, include_package_data=True, @@ -71,13 +74,17 @@ 'debian = bloom.generators.debian:DebianGenerator', 'rosdebian = bloom.generators.rosdebian:RosDebianGenerator', 'rpm = bloom.generators.rpm:RpmGenerator', - 'rosrpm = bloom.generators.rosrpm:RosRpmGenerator' + 'rosrpm = bloom.generators.rosrpm:RosRpmGenerator', + 'conda = bloom.generators.conda:CondaGenerator', + 'rosconda = bloom.generators.rosrpm:RosCondaGenerator' ], 'bloom.generate_cmds': [ 'debian = bloom.generators.debian.generate_cmd:description', 'rosdebian = bloom.generators.rosdebian:description', 'rpm = bloom.generators.rpm.generate_cmd:description', - 'rosrpm = bloom.generators.rosrpm:description' + 'rosrpm = bloom.generators.rosrpm:description', + 'conda = bloom.generators.conda.generate_cmd:description', + 'rosconda = bloom.generators.rosconda:description' ] } )