SSH runner: created RemoteConnection and copy conan config to remote

conan/internal/runner/ssh.py

@@ -1,14 +1,19 @@
 from pathlib import Path
+from typing import Iterable, Optional
+import fnmatch
 import pathlib
 import tempfile
 
 from conan.api.output import Color, ConanOutput
 from conan.errors import ConanException
 
+import errno
 import os
 from io import BytesIO
 import sys
 
+from conan.internal.runner.output import RunnerOutput
+
 def ssh_info(msg, error=False):
     fg=Color.BRIGHT_MAGENTA
     if error:
@@ -51,9 +56,14 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar
         if ssh_config and ssh_config.lookup(hostname):
             hostname = ssh_config.lookup(hostname)['hostname']
 
-        self.client = SSHClient()
-        self.client.load_system_host_keys()
-        self.client.connect(hostname)
+        # this self.client manages the main ssh connection
+        self.ssh_client = SSHClient()
+        self.ssh_client.load_system_host_keys()
+        self.ssh_client.connect(hostname)
+        self.runner_output = RunnerOutput(hostname)
+        # This client manages just the sftp transfers
+        # TODO: Integrate both client calls in one
+        self.sftp_client = RemoteConnection(self.ssh_client, self.runner_output)
 
 
     def run(self, use_cache=True):
@@ -92,10 +102,10 @@ def ensure_runner_environment(self):
         has_python3_command = False
         python_is_python3 = False
 
-        _, _stdout, _stderr = self.client.exec_command("python3 --version")
+        _, _stdout, _stderr = self.ssh_client.exec_command("python3 --version")
         has_python3_command = _stdout.channel.recv_exit_status() == 0
         if not has_python3_command:
-            _, _stdout, _stderr = self.client.exec_command("python --version")
+            _, _stdout, _stderr = self.ssh_client.exec_command("python --version")
             if _stdout.channel.recv_exit_status() == 0 and "Python 3" in _stdout.read().decode():
                 python_is_python3 = True
 
@@ -106,14 +116,14 @@ def ensure_runner_environment(self):
             raise ConanException("Unable to locate working Python 3 executable in remote SSH environment")
 
         # Determine if remote host is Windows
-        _, _stdout, _ = self.client.exec_command(f'{python_command} -c "import os; print(os.name)"')
+        _, _stdout, _ = self.ssh_client.exec_command(f'{python_command} -c "import os; print(os.name)"')
         if _stdout.channel.recv_exit_status() != 0:
             raise ConanException("Unable to determine remote OS type")
         is_windows = _stdout.read().decode().strip() == "nt"
         self.remote_is_windows = is_windows
 
         # Get remote user home folder
-        _, _stdout, _ = self.client.exec_command(f'{python_command} -c "from pathlib import Path; print(Path.home())"')
+        _, _stdout, _ = self.ssh_client.exec_command(f'{python_command} -c "from pathlib import Path; print(Path.home())"')
         if _stdout.channel.recv_exit_status() != 0:
             raise ConanException("Unable to determine remote home user folder")
         home_folder = _stdout.read().decode().strip()
@@ -129,7 +139,7 @@ def ensure_runner_environment(self):
 
         # Ensure remote folders exist
         for folder in [remote_folder, remote_conan_home]:
-            _, _stdout, _stderr = self.client.exec_command(f"""{python_command} -c "import os; os.makedirs('{folder}', exist_ok=True)""")
+            _, _stdout, _stderr = self.ssh_client.exec_command(f"""{python_command} -c "import os; os.makedirs('{folder}', exist_ok=True)""")
             if _stdout.channel.recv_exit_status() != 0:
                 ssh_info(f"Error creating remote folder: {_stderr.read().decode()}")
                 raise ConanException(f"Unable to create remote workfolder at {folder}")
@@ -144,17 +154,10 @@ def ensure_runner_environment(self):
         ssh_info(f"Expected remote conan command: {conan_cmd}")
 
         # Check if remote Conan executable exists, otherwise invoke pip inside venv
-        sftp = self.client.open_sftp()
-        try:
-            sftp.stat(conan_cmd)
-            has_remote_conan = True
-        except FileNotFoundError:
-            has_remote_conan = False
-        finally:
-            sftp.close()
+        has_remote_conan = self.sftp_client.check_file_exists(conan_cmd)
 
         if not has_remote_conan:
-            _, _stdout, _stderr = self.client.exec_command(f"{python_command} -m venv {conan_venv}")
+            _, _stdout, _stderr = self.ssh_client.exec_command(f"{python_command} -m venv {conan_venv}")
             if _stdout.channel.recv_exit_status() != 0:
                 ssh_info(f"Unable to create remote venv: {_stderr.read().decode().strip()}")
 
@@ -163,7 +166,7 @@ def ensure_runner_environment(self):
             else:
                 python_command = remote_folder + "/venv" + "/bin" + "/python"
 
-            _, _stdout, _stderr = self.client.exec_command(f"{python_command} -m pip install git+https://github.com/conan-io/conan@feature/docker_wrapper")
+            _, _stdout, _stderr = self.ssh_client.exec_command(f"{python_command} -m pip install git+https://github.com/conan-io/conan@feature/docker_wrapper")
             if _stdout.channel.recv_exit_status() != 0:
                 # Note: this may fail on windows
                 ssh_info(f"Unable to install conan in venv: {_stderr.read().decode().strip()}")
@@ -178,37 +181,33 @@ def ensure_runner_environment(self):
             conan_bat_contents = f"""@echo off\n{env_lines}\n{conan_cmd} %*\n"""
             conan_bat = remote_folder + "/conan.bat"
             try:
-                sftp = self.client.open_sftp()
+                sftp = self.ssh_client.open_sftp()
                 sftp.putfo(BytesIO(conan_bat_contents.encode()), conan_bat)
             except:
                 raise ConanException("unable to set up Conan remote script")
             finally:
                 sftp.close()
 
             self.remote_conan = conan_bat
-        _, _stdout, _stderr = self.client.exec_command(f"{self.remote_conan} config home")
+        _, _stdout, _stderr = self.ssh_client.exec_command(f"{self.remote_conan} config home")
         ssh_info(f"Remote conan config home returned: {_stdout.read().decode().strip()}")
-        _, _stdout, _stderr = self.client.exec_command(f"{self.remote_conan} profile detect --force")
-        self._copy_profiles()
-
-
-    def _copy_profiles(self):
-        sftp = self.client.open_sftp()
-
-        # TODO: very questionable choices here
-        try:
-            profiles = {
-                self.args.profile_host[0]: self.host_profile.dumps(),
-                self.args.profile_build[0]: self.build_profile.dumps()
-            }
-
-            for name, contents in profiles.items():
-                dest_filename = self.remote_conan_home + f"/profiles/{name}"
-                sftp.putfo(BytesIO(contents.encode()), dest_filename)
-        except:
-            raise ConanException("Unable to copy profiles to remote")
-        finally:
-            sftp.close()
+        _, _stdout, _stderr = self.ssh_client.exec_command(f"{self.remote_conan} profile detect --force")
+        self._sync_conan_config()
+
+    def _sync_conan_config(self):
+        # Transfer conan config to remote
+        cache_home = Path(self.conan_api.config.home())
+        cache_remote = Path(self.remote_conan_home)
+        # TODO: inspect each file and determine if references to local paths are present -> inform
+        # user that that profile/configuration file will not work on remote
+        self.sftp_client.put_dir(
+            cache_home / "profiles",
+            cache_remote / "profiles",
+            exclude_patterns=(".conan.db", "*.pyc", "__pycache__", ".DS_Store", ".git")
+        )
+        for file in ("settings.yml", "settings_user.yml", "global.conf", "remotes.json"):
+            if (cache_home / file).exists():
+                self.sftp_client.put(cache_home / file, cache_remote / file)
 
     def copy_working_conanfile_path(self):
         resolved_path = Path(self.args.path).resolve()
@@ -220,14 +219,15 @@ def copy_working_conanfile_path(self):
 
         # Create temporary destination directory
         temp_dir_create_cmd = f"""{self.remote_python_command} -c "import tempfile; print(tempfile.mkdtemp(dir='{self.remote_workspace}'))"""
-        _, _stdout, _ = self.client.exec_command(temp_dir_create_cmd)
+        _, _stdout, _ = self.ssh_client.exec_command(temp_dir_create_cmd)
         if _stdout.channel.recv_exit_status() != 0:
             raise ConanException("Unable to create remote temporary directory")
         self.remote_create_dir = _stdout.read().decode().strip().replace("\\", '/')
 
         # Copy current folder to destination using sftp
+        # self.remote_conn.put_dir(resolved_path.as_posix(), self.remote_create_dir)
         _Path = pathlib.PureWindowsPath if self.remote_is_windows else pathlib.PurePath
-        sftp = self.client.open_sftp()
+        sftp = self.ssh_client.open_sftp()
         for root, dirs, files in os.walk(resolved_path.as_posix()):
             relative_root = Path(root).relative_to(resolved_path)
             for dir in dirs:
@@ -245,7 +245,7 @@ def _run_command(self, command):
             ensure we pass width and height that matches the current
             terminal
         '''
-        channel = self.client.get_transport().open_session()
+        channel = self.ssh_client.get_transport().open_session()
         if sys.stdout.isatty():
             width, height = os.get_terminal_size()
             channel.get_pty(width=width, height=height)
@@ -261,18 +261,69 @@ def update_local_cache(self, json_result):
         _Path = pathlib.PureWindowsPath if self.remote_is_windows else pathlib.PurePath
         pkg_list_json = _Path(self.remote_create_dir).joinpath("pkg_list.json").as_posix()
         pkg_list_command = f"{self.remote_conan} list --graph={json_result} --graph-binaries=build --format=json > {pkg_list_json}"
-        _, stdout, _ = self.client.exec_command(pkg_list_command)
+        _, stdout, _ = self.ssh_client.exec_command(pkg_list_command)
         if stdout.channel.recv_exit_status() != 0:
             raise ConanException("Unable to generate remote package list")
 
         conan_cache_tgz = _Path(self.remote_create_dir).joinpath("cache.tgz").as_posix()
         cache_save_command = f"{self.remote_conan} cache save --list {pkg_list_json} --file {conan_cache_tgz}"
-        _, stdout, _ = self.client.exec_command(cache_save_command)
+        _, stdout, _ = self.ssh_client.exec_command(cache_save_command)
         if stdout.channel.recv_exit_status() != 0:
             raise ConanException("Unable to save remote conan cache state")
 
-        sftp = self.client.open_sftp()
         with tempfile.TemporaryDirectory() as tmp:
             local_cache_tgz = os.path.join(tmp, 'cache.tgz')
-            sftp.get(conan_cache_tgz, local_cache_tgz)
-            package_list = self.conan_api.cache.restore(local_cache_tgz)
+            self.sftp_client.get(conan_cache_tgz, local_cache_tgz)
+            self.conan_api.cache.restore(local_cache_tgz)
+
+
+class RemoteConnection:
+    def __init__(self, ssh_client, runner_output: RunnerOutput):
+        from paramiko.client import SSHClient
+        self.client: SSHClient = ssh_client
+        # No need to close connection as it will be closed on destroy of RemoteConnection
+        self.sftp = self.client.open_sftp()
+        self.runner_output = runner_output
+
+    def put(self, src: Path, dst: Path) -> None:
+        try:
+            self.sftp.put(src.as_posix(), dst.as_posix())
+            self.runner_output.verbose(f"Copying file {src.as_posix()} to {dst}")
+        except IOError as e:
+            self.runner_output.error(f"Unable to copy {src} to {dst}:\n{e}")
+
+    def put_dir(self, source_folder: Path, destination_folder: Path, exclude_patterns: Optional[Iterable[str]] = None) -> None:
+        for item in source_folder.iterdir():
+            dest_item = destination_folder / item.name
+            # Check if item matches any exclude pattern
+            if exclude_patterns and any(fnmatch.fnmatch(item.name, pattern) for pattern in exclude_patterns):
+                continue
+            if item.is_file():
+                self.runner_output.verbose(f"Copying file {item.as_posix()} to {dest_item}")
+                self.put(item, dest_item)
+            elif item.is_dir():
+                self.runner_output.verbose(f"Copying directory {item.as_posix()} to {dest_item}")
+                self.mkdir(dest_item, ignore_existing=True)
+                self.put_dir(item, dest_item, exclude_patterns)
+
+    def get(self, src: str, dst: str) -> None:
+        try:
+            self.sftp.get(src, dst)
+        except IOError as e:
+            self.runner_output.error(f"Unable to copy from remote {src} to {dst}:\n{e}")
+
+    def mkdir(self, folder: Path, ignore_existing=False) -> None:
+        try:
+            self.sftp.mkdir(folder.as_posix())
+        except IOError as e:
+            if e.errno == errno.EEXIST and ignore_existing:
+                pass
+            else:
+                raise
+
+    def check_file_exists(self, file: str) -> bool:
+        try:
+            self.sftp.stat(file)
+            return True
+        except FileNotFoundError:
+            return False