Get rid of imp, started giving deprecation warning every time in Python 3.7 in Ubuntu 19.04.

Please python stop torturing me with refactors.

Make ./run -u blow up if executable not found, otherwise I go crazy.

Get ./test-gdb back to life after the ./run relative path refactor, forgot to test this.

Author: cirosantilli

README.adoc

@@ -3910,20 +3910,10 @@ Similarly to <<qemu-user-mode-does-not-show-stdout-immediately>>, QEMU error mes
 In particular, it does not say anything if you pass it a non-existing executable:
 
 ....
-./build-userland --clean userland/c/hello.c
-./run --userland userland/c/hello.c
-echo $?
-....
-
-does not output anything, except for the `1` exit status.
-
-If you run however the raw command without a pipe manually, it shows a helpful error message:
-
-....
-Error while loading /path/to/linux-kernel-module-cheat/out/userland/default/x86_64/c/hello.out: No such file or directory
+qemu-x86_64 asdf | cat
 ....
 
-Tested in de77c62c091f6418e73b64e8a0a19639c587a103 + 1.
+So we just check ourselves manually
 
 == Kernel module utilities
 

bisect-linux-boot-gem5

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
 
-import imp
 import os
 import shutil
 import sys
 
 import common
-build_linux = imp.load_source('build-linux', os.path.join(kwargs['root_dir'], 'build_linux'))
-run = imp.load_source('run', os.path.join(kwargs['root_dir'], 'run'))
+
+build_linux = common.import_path_relative_root('build-linux')
+run = common.import_path_relative_root('run')
 
 parser = self.get_argparse(
     argparse_args={

build

@@ -403,7 +403,7 @@ Which components to build. Default: qemu-buildroot
             args = self.get_common_args()
             args.update(extra_args)
             args['show_time'] = False
-            self.import_path_main(component_file)(**args)
+            common.import_path_main(component_file)(**args)
         return f
 
     def timed_main(self):

build-baremetal

@@ -133,7 +133,7 @@ Build the baremetal examples with crosstool-NG.
                     in_name, in_ext = os.path.splitext(in_basename)
                     if  (
                         os.path.isfile(in_path) and
-                        in_ext in (self.env['c_ext'], self.env['asm_ext'])
+                        in_ext in self.env['build_in_exts']
                     ):
                         out = os.path.join(out_dir, in_name + self.env['baremetal_build_ext'])
                         src = os.path.join(self.env['baremetal_source_dir'], in_path)

build-userland

@@ -88,7 +88,7 @@ Default: build all examples that have their package dependencies met, e.g.:
                     for path, in_dirnames, in_filenames in self.sh.walk(target):
                         for in_filename in in_filenames:
                             in_ext = os.path.splitext(in_filename)[1]
-                            if not in_ext in self.env['userland_in_exts']:
+                            if not in_ext in self.env['build_in_exts']:
                                 continue
                             in_path = os.path.join(path, in_filename)
                             error = thread_pool.submit({

build-userland-in-tree

@@ -1,18 +1,11 @@
 #!/usr/bin/env python3
 
-import imp
 import os
 import subprocess
 
-git_root = subprocess.check_output([
-    'git',
-    'rev-parse',
-    '--show-toplevel',
-]).decode().rstrip()
-build_userland = imp.load_source(
-    'build_userland',
-    os.path.join(git_root, 'build-userland')
-)
+import common
+
+build_userland = common.import_path_relative_root('build-userland')
 
 class Main(build_userland.Main):
     def __init__(self):

cli_function.py

@@ -11,10 +11,11 @@
 import argparse
 import bisect
 import collections
-import imp
 import os
 import sys
 
+import common
+
 class _Argument:
     def __init__(
             self,
@@ -188,7 +189,7 @@ def _get_args(self, kwargs):
         if config_file is not None:
             if os.path.exists(config_file):
                 config_configs = {}
-                config = imp.load_source('config', config_file)
+                config = common.import_path(config_file)
                 if self.extra_config_params is None:
                     config.set_args(config_configs)
                 else:

common.py

@@ -8,7 +8,7 @@
 import enum
 import functools
 import glob
-import imp
+import importlib
 import inspect
 import itertools
 import json
@@ -105,7 +105,7 @@ def git_sha(repo_path):
 consts['header_ext'] = '.h'
 consts['kernel_module_ext'] = '.ko'
 consts['obj_ext'] = '.o'
-consts['userland_in_exts'] = [
+consts['build_in_exts'] = [
     consts['asm_ext'],
     consts['c_ext'],
     consts['cxx_ext'],
@@ -136,6 +136,32 @@ def git_sha(repo_path):
     consts['emulator_choices'].add(consts['emulator_short_to_long_dict'][key])
 consts['host_arch'] = platform.processor()
 
+def import_path(path):
+    '''
+    https://stackoverflow.com/questions/2601047/import-a-python-module-without-the-py-extension
+    https://stackoverflow.com/questions/31773310/what-does-the-first-argument-of-the-imp-load-source-method-do
+    '''
+    module_name = os.path.basename(path).replace('-', '_')
+    spec = importlib.util.spec_from_loader(
+        module_name,
+        importlib.machinery.SourceFileLoader(module_name, path)
+    )
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    sys.modules[module_name] = module
+    return module
+
+def import_path_relative_root(basename):
+    return import_path(os.path.join(consts['root_dir'], basename))
+
+def import_path_main(basename):
+    '''
+    Import an object of the Main class of a given file.
+
+    By convention, we call the main object of all our CLI scripts as Main.
+    '''
+    return import_path_relative_root(basename).Main()
+
 class ExitLoop(Exception):
     pass
 
@@ -905,6 +931,8 @@ def join(*paths):
                         env['source_path'] = source_path
                         break
             env['image'] = path
+        elif env['userland'] is not None:
+            env['image'] = self.resolve_userland_executable(env['userland'])
         else:
             if env['emulator'] == 'gem5':
                 env['image'] = env['vmlinux']
@@ -1084,24 +1112,6 @@ def github_make_request(
             _json = {}
         return _json
 
-    def import_path(self, basename):
-        '''
-        https://stackoverflow.com/questions/2601047/import-a-python-module-without-the-py-extension
-        https://stackoverflow.com/questions/31773310/what-does-the-first-argument-of-the-imp-load-source-method-do
-        '''
-        return imp.load_source(
-            basename.replace('-', '_'),
-            os.path.join(self.env['root_dir'], basename)
-        )
-
-    def import_path_main(self, path):
-        '''
-        Import an object of the Main class of a given file.
-
-        By convention, we call the main object of all our CLI scripts as Main.
-        '''
-        return self.import_path(path).Main()
-
     def is_arch_supported(self, arch):
         return self.supported_archs is None or arch in self.supported_archs
 
@@ -1270,6 +1280,8 @@ def resolve_executable(
 
         If the input path is a file, add the executable extension automatically.
         '''
+        if not self.env['dry_run'] and not os.path.exists(in_path):
+            raise Exception('Input path does not exist: ' + in_path)
         if self.is_subpath(in_path, magic_in_dir):
             # Abspath needed to remove the trailing `/.` which makes e.g. rmrf fail.
             out = os.path.abspath(os.path.join(

run

@@ -404,12 +404,12 @@ Extra options to append at the end of the emulator command line.
             if not self.env['dry_run']:
                 raise Exception('Root filesystem not found. Did you build it? ' \
                                 'Tried to use: ' + self.env['disk_image'])
-        def raise_image_not_found():
+        def raise_image_not_found(image):
             if not self.env['dry_run']:
                 raise Exception('Executable image not found. Did you build it? ' \
-                                'Tried to use: ' + self.env['image'])
-        if self.env['image'] is None:
-            raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths))
+                                'Tried to use: ' + image)
+        if not os.path.exists(self.env['image']):
+            raise_image_not_found(self.env['image'])
         cmd = debug_vm.copy()
         if self.env['emulator'] == 'gem5':
             if self.env['quiet']:
@@ -430,7 +430,7 @@ Extra options to append at the end of the emulator command line.
                     self.env['userland'] is None
                 ):
                     if not os.path.exists(self.env['linux_image']):
-                        raise_image_not_found()
+                        raise_image_not_found(self.env['image'])
                     self.sh.run_cmd([os.path.join(self.env['extract_vmlinux'], self.env['linux_image'])])
             os.makedirs(os.path.dirname(self.env['gem5_readfile']), exist_ok=True)
             self.sh.write_string_to_file(self.env['gem5_readfile'], self.env['gem5_readfile'])
@@ -457,7 +457,7 @@ Extra options to append at the end of the emulator command line.
             if self.env['userland'] is not None:
                 cmd.extend([
                     self.env['gem5_se_file'], LF,
-                    '--cmd', self.resolve_userland_executable(self.env['userland']), LF,
+                    '--cmd', self.env['image'], LF,
                 ])
                 if self.env['userland_args'] is not None:
                     cmd.extend(['--options', self.env['userland_args'], LF])
@@ -551,8 +551,6 @@ Extra options to append at the end of the emulator command line.
                     debug_args
                 )
             else:
-                if not os.path.exists(self.env['image']):
-                    raise_image_not_found()
                 extra_emulator_args.extend(extra_qemu_args)
                 self.make_run_dirs()
                 if self.env['debug_vm']:
@@ -725,7 +723,7 @@ Extra options to append at the end of the emulator command line.
         cmd.extend(self.env['extra_emulator_args'])
         if self.env['userland'] and self.env['emulator'] in ('qemu', 'native'):
             # The program and arguments must come at the every end of the CLI.
-            cmd.extend([self.resolve_userland_executable(self.env['userland']), LF])
+            cmd.extend([self.env['image'], LF])
             if self.env['userland_args'] is not None:
                 cmd.extend(self.sh.shlex_split(self.env['userland_args']))
         if debug_vm or self.env['terminal']:

run-gdb

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import imp
 import os
 import signal
 import subprocess
@@ -34,7 +33,7 @@ class GdbTestcase:
         self.child.setecho(False)
         self.child.waitnoecho()
         self.child.expect(self.prompt)
-        test = imp.load_source('test', test_script_path)
+        test = common.import_path(test_script_path)
         exception = None
         try:
             test.test(self)