Be My Ardent-tine (ros-infrastructure#135)

* Add ardent, and change the ROSDISTRO_INDEX_URL to point to the ros2 version.

* Index ROS1 and ROS2.

* Separate (for more modularity) several parts of the ebuild text generation, as well as parse the build_type tag from the export line of the package.xml.

* Fix syntax, centralize the ros2_distro list.

* Extract tag with a regex, because the other way is disgustingly hackey, and lint.

* Rename class to reflect overlay.

* Begin working on ROS2 patches.

* Fix build_type detection logic.

* Add mapping for uncrustify license.

* Added the eclass.

* Add cmake build_type mapping.

* Add LGPL-2.1 test detection test (and fix).

* Change distros to file in sorted order so as to make testing more consistent.

superflore/PackageMetadata.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import re
+
 from catkin_pkg.package import parse_package_string
 
 
@@ -38,3 +40,11 @@ def __init__(self, pkg_xml):
         self.upstream_name = [
             author.name for author in pkg.maintainers
         ][0]
+        tag_remover = re.compile('<.*?>')
+        build_type = [
+            re.sub(tag_remover, '', str(e))
+            for e in pkg.exports if 'build_type' in str(e)
+        ]
+        self.build_type = 'catkin'
+        if build_type:
+            self.build_type = build_type[0]

superflore/exceptions.py

@@ -36,3 +36,9 @@ def __init__(self, message):
 class NoGitHubAuthToken(Exception):
     def __init__(self, message):
         self.message = message
+
+
+class UnknownBuildType(Exception):
+    """Raised when we don't know what to inherit to build the package"""
+    def __init__(self, msg):
+        self.message = msg

superflore/generate_installers.py

@@ -14,6 +14,7 @@
 
 from rosinstall_generator.distro import get_distro
 from rosinstall_generator.distro import get_package_names
+from superflore.exceptions import UnknownBuildType
 from superflore.exceptions import UnknownLicense
 from superflore.utils import err
 from superflore.utils import get_pkg_version
@@ -75,6 +76,13 @@ def generate_installers(
             err("{0}%: Unknown License '{1}'.".format(percent, str(ul)))
             bad_installers.append(pkg)
             failed = failed + 1
+        except UnknownBuildType as ub:
+            err(
+                "{0}%: Unknown Build type '{1}' for package '{2}'".format(
+                    percent, str(ub), pkg
+                )
+            )
+            failed = failed + 1
         except KeyError:
             failed_msg = 'Failed to generate installer'
             err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg))

superflore/generators/ebuild/ebuild.py

@@ -14,6 +14,7 @@
 
 from time import gmtime, strftime
 
+from superflore.exceptions import UnknownBuildType
 from superflore.exceptions import UnresolvedDependency
 from superflore.utils import get_license
 from superflore.utils import resolve_dep
@@ -67,7 +68,10 @@ def __init__(self):
         self.unresolved_deps = list()
         self.name = None
         self.has_patches = False
+        self.build_type = 'catkin'
+        self.is_ros2 = False
         self.python_3 = True
+        self.patches = list()
         self.illegal_desc_chars = '()[]{}|^$\\#\t\n\r\v\f\'\"\`'
 
     def add_build_depend(self, depend, internal=True):
@@ -97,25 +101,54 @@ def add_test_depend(self, tdepend, internal=True):
     def add_keyword(self, keyword, stable=False):
         self.keys.append(ebuild_keyword(keyword, stable))
 
-    def get_ebuild_text(self, distributor, license_text):
-        """
-        Generate the ebuild in text, given the distributor line
-        and the license text.
-        """
+    def get_license_line(self, distributor, license_text):
         ret = "# Copyright " + strftime("%Y", gmtime()) + " "
         ret += distributor + "\n"
         ret += "# Distributed under the terms of the " + license_text
         ret += " license\n\n"
+        return ret
+
+    def get_eapi_line(self):
+        return 'EAPI=%s\n' % self.eapi
+
+    def get_python_compat(self, python_versions):
+        ver_string = ''
+        if len(python_versions) > 1:
+            ver_string = '{' + ','.join(python_versions) + '}'
+        else:
+            ver_string = python_versions[0]
+        return 'PYTHON_COMPAT=( python%s )\n\n' % ver_string
 
+    def get_inherit_line(self):
+        # if we are using catkin, we just inherit ros-cmake
+        if self.build_type in ['catkin', 'cmake']:
+            return 'inherit ros-cmake\n\n'
+        elif self.build_type == 'ament_python':
+            return 'inherit ament-python\n\n'
+        elif self.build_type == 'ament_cmake':
+            return 'inherit ament-cmake\n\n'
+        else:
+            raise UnknownBuildType(self.build_type)
+
+    def get_ebuild_text(self, distributor, license_text):
+        """
+        Generate the ebuild in text, given the distributor line
+        and the license text.
+        """
         # EAPI=<eapi>
-        ret += "EAPI=" + self.eapi + "\n"
-        if self.python_3:
-            ret += "PYTHON_COMPAT=( python{2_7,3_5} )\n\n"
+        ret = self.get_license_line(distributor, license_text)
+        ret += self.get_eapi_line()
+        if self.python_3 and not self.is_ros2:
+            # enable python 2.7 and python 3.5
+            ret += self.get_python_compat(['2_7', '3_5'])
+        elif self.python_3:
+            # only use 3.5, 3.6 for ROS 2
+            ret += self.get_python_compat(['3_5', '3_6'])
         else:
-            ret += "PYTHON_COMPAT=( python2_7 )\n\n"
+            # fallback to python 2.7
+            ret += self.get_python_compat(['2_7'])
         # inherits
-        ret += "inherit ros-cmake\n\n"
-
+        ret += self.get_inherit_line()
         # description, homepage, src_uri
         self.description =\
             sanitize_string(self.description, self.illegal_desc_chars)
@@ -198,12 +231,14 @@ def get_ebuild_text(self, distributor, license_text):
 
         # Patch source if needed.
         if self.has_patches:
+            # TODO(allenh1): explicitly list patches
             ret += "\nsrc_prepare() {\n"
             ret += "    cd ${P}\n"
             ret += "    EPATCH_SOURCE=\"${FILESDIR}\""
             ret += " EPATCH_SUFFIX=\"patch\" \\\n"
             ret += "    EPATCH_FORCE=\"yes\" epatch\n"
-            ret += "    ros-cmake_src_prepare\n"
+            if self.build_type in ['catkin', 'cmake']:
+                ret += "    ros-cmake_src_prepare\n"
             ret += "}\n"
 
         # source configuration

superflore/generators/ebuild/gen_packages.py

@@ -28,6 +28,7 @@
 from superflore.utils import get_pkg_version
 from superflore.utils import make_dir
 from superflore.utils import ok
+from superflore.utils import ros2_distros
 from superflore.utils import warn
 
 # TODO(allenh1): This is a blacklist of things that
@@ -48,9 +49,14 @@ def regenerate_pkg(overlay, pkg, distro, preserve_existing=False):
     ebuild_name = overlay.repo.repo_dir + ebuild_name
     patch_path = '/ros-{}/{}/files'.format(distro.name, pkg)
     patch_path = overlay.repo.repo_dir + patch_path
+    is_ros2 = distro.name in ros2_distros
     has_patches = os.path.exists(patch_path)
     pkg_names = get_package_names(distro)[0]
-
+    patches = None
+    if os.path.exists(patch_path):
+        patches = [
+            f for f in glob.glob('%s/*.patch' % patch_path)
+        ]
     if pkg not in pkg_names:
         raise RuntimeError("Unknown package '%s'" % (pkg))
     # otherwise, remove a (potentially) existing ebuild.
@@ -70,6 +76,8 @@ def regenerate_pkg(overlay, pkg, distro, preserve_existing=False):
     try:
         current = gentoo_installer(distro, pkg, has_patches)
         current.ebuild.name = pkg
+        current.ebuild.patches = patches
+        current.ebuild.is_ros2 = is_ros2
     except Exception as e:
         err('Failed to generate installer for package {}!'.format(pkg))
         raise e
@@ -176,6 +184,7 @@ def _gen_ebuild_for_package(
     pkg_ebuild.upstream_license = pkg.upstream_license
     pkg_ebuild.description = pkg.description
     pkg_ebuild.homepage = pkg.homepage
+    pkg_ebuild.build_type = pkg.build_type
     return pkg_ebuild
 
 

superflore/generators/ebuild/overlay_instance.py

@@ -40,6 +40,7 @@ def commit_changes(self, distro):
             'lunar': 'regenerate ros-lunar, ',
             'indigo': 'regenerate ros-indigo, ',
             'kinetic': 'regenerate ros-kinetic, ',
+            'ardent': 'regenerate ros2-ardent, ',
         }[distro or 'update'] + time.ctime()
         self.repo.git.commit(m='{0}'.format(commit_msg))
 

superflore/generators/ebuild/run.py

@@ -23,6 +23,7 @@
 from superflore.parser import get_parser
 from superflore.repo_instance import RepoInstance
 from superflore.TempfileManager import TempfileManager
+from superflore.utils import active_distros
 from superflore.utils import clean_up
 from superflore.utils import err
 from superflore.utils import file_pr
@@ -31,12 +32,16 @@
 from superflore.utils import info
 from superflore.utils import load_pr
 from superflore.utils import ok
+from superflore.utils import ros2_distros
 from superflore.utils import save_pr
 from superflore.utils import url_to_repo_org
 from superflore.utils import warn
 
-# Modify if a new distro is added
-active_distros = ['indigo', 'kinetic', 'lunar']
+# TODO(allenh1): It would be super nice make this a configuration option.
+ros2_index =\
+    'https://raw.githubusercontent.com/ros2/rosdistro/ros2/index.yaml'
+ros1_index =\
+    'https://raw.githubusercontent.com/ros/rosdistro/master/index.yaml'
 
 
 def main():
@@ -51,6 +56,7 @@ def main():
         preserve_existing = False
     elif args.ros_distro:
         selected_targets = [args.ros_distro]
+        set_index_for_distro(args.ros_distro)
         preserve_existing = False
     elif args.dry_run and args.pr_only:
         parser.error('Invalid args! cannot dry-run and file PR')
@@ -68,7 +74,7 @@ def main():
             err('reason: {0}'.format(e))
             sys.exit(1)
     if not selected_targets:
-        selected_targets = active_distros
+        selected_targets = active_distros + ros2_distros
     repo_org = 'ros'
     repo_name = 'ros-overlay'
     if args.upstream_repo:
@@ -137,6 +143,7 @@ def main():
             sys.exit(0)
 
         for distro in selected_targets:
+            set_index_for_distro(distro)
             distro_installers, distro_broken, distro_changes =\
                 generate_installers(
                     distro_name=distro,
@@ -179,3 +186,11 @@ def main():
 
         clean_up()
         ok('Successfully synchronized repositories!')
+
+
+def set_index_for_distro(distro):
+    if distro in ros2_distros:
+        # Add ROS2 to rosdistro
+        os.environ['ROSDISTRO_INDEX_URL'] = ros2_index
+    else:
+        os.environ['ROSDISTRO_INDEX_URL'] = ros1_index

superflore/utils.py

@@ -25,6 +25,10 @@
 from superflore.rosdep_support import resolve_rosdep_key
 from termcolor import colored
 
+# Modify if a new distro is added
+active_distros = ['indigo', 'kinetic', 'lunar']
+ros2_distros = ['ardent']
+
 
 def warn(string):  # pragma: no cover
     print(colored('>>>> {0}'.format(string), 'yellow'))
@@ -144,8 +148,10 @@ def trim_string(string, length=80):
 
 def get_license(l):
     bsd_re = '^(BSD)((.)*([124]))?'
-    gpl_re = '(GPL)((.)*([123]))?'
-    lgpl_re = '^(LGPL)((.)*([23]|2\\.1))?'
+    gpl_re = '((([^L])*(GPL)([^0-9]*))|'\
+        '(GNU(.)*GENERAL(.)*PUBLIC(.)*LICENSE([^0-9])*))([0-9])?'
+    lgpl_re = '(((LGPL)([^0-9]*))|'\
+        '(GNU(.)*Lesser(.)*Public(.)*License([^0-9])*))([0-9]?\\.[0-9])?'
     apache_re = '^(Apache)((.)*(1\\.0|1\\.1|2\\.0|2))?'
     cc_re = '^(Creative(.)?Commons)((.)*)'
     cc_nc_nd_re = '^((Creative(.)?Commons)|CC)((.)*)' +\
@@ -168,12 +174,16 @@ def get_license(l):
             return 'BSD-{0}'.format(version)
         return 'BSD'
     elif re.search(lgpl_re, l, f):
-        version = re.search(lgpl_re, l, f).group(4)
+        version = re.search(lgpl_re, l, f)
+        grp = len(version.groups())
+        version = version.group(grp)
         if version:
             return 'LGPL-{0}'.format(version)
         return 'LGPL-2'
     elif re.search(gpl_re, l, f):
-        version = re.search(gpl_re, l, f).group(4)
+        version = re.search(gpl_re, l, f)
+        grp = len(version.groups())
+        version = version.group(grp)
         if version:
             return 'GPL-{0}'.format(version)
         return 'GPL-1'
@@ -241,7 +251,7 @@ def gen_delta_msg(total_changes):
     """Return string of changes for the PR message."""
     delta = "Changes:\n"
     delta += "========\n"
-    for distro in total_changes:
+    for distro in sorted(total_changes):
         if not total_changes[distro]:
             continue
         delta += "%s Changes:\n" % distro.title()

tests/test_utils.py

@@ -28,6 +28,10 @@
 
 
 class TestUtils(unittest.TestCase):
+    def set_lang_env(self):
+        os.environ['LANG'] = 'en_US.UTF-8'
+        os.environ['LC_ALL'] = 'en_US.UTF-8'
+
     def test_sanitize(self):
         """Test sanitize string function"""
         # test with an empty string
@@ -94,27 +98,33 @@ def test_get_license(self):
         self.assertEqual(ret, 'public_domain')
         ret = get_license('GPL')
         self.assertEqual(ret, 'GPL-1')
+        ret = get_license('GNU GENERAL PUBLIC LICENSE Version 3')
+        self.assertEqual(ret, 'GPL-3')
+        ret = get_license('GNU Lesser Public License 2.1')
+        self.assertEqual(ret, 'LGPL-2.1')
 
     def test_delta_msg(self):
         """Test the delta message generated for the PR"""
+        self.set_lang_env()
         total_changes = dict()
         total_changes['hydro'] = ['foo', 'bar']
         total_changes['boxturtle'] = ['baz']
         total_changes['C'] = []
         expect = 'Changes:\n'\
                  '========\n'\
+                 'Boxturtle Changes:\n'\
+                 '---------------\n'\
+                 '* baz\n\n'\
                  'Hydro Changes:\n'\
                  '---------------\n'\
                  '* bar\n'\
-                 '* foo\n\n'\
-                 'Boxturtle Changes:\n'\
-                 '---------------\n'\
-                 '* baz\n\n'
+                 '* foo\n\n'
         got = gen_delta_msg(total_changes)
         self.assertEqual(expect, got)
 
     def test_missing_deps_msg(self):
         """Test the missing dependencies list"""
+        self.set_lang_env()
         self.assertEqual(
             gen_missing_deps_msg([]), 'No missing dependencies.\n'
         )