changes so installing/updating goes smooth

Author: Tmonster

.github/workflows/RunBenchmark.yml

@@ -40,6 +40,7 @@ jobs:
       CC: gcc-10
       CXX: g++-10
       GEN: ninja
+      MACHINE_TYPE="c6id.4xlarge"
     runs-on: self-hosted
     steps:
       - uses: actions/checkout@v3
@@ -53,11 +54,13 @@ jobs:
         run: |
           ./_setup_utils/mount.sh
 
-      - name: Upgrade all solutions
+      - name: Install or Upgrade all solutions
         shell: bash
         working-directory: /var/lib/mount/db-benchmark-metal
         run: |
-          python3 _setup_utils/install_all_solutions.py all
+          python3 _setup_utils/install_all_solutions.py all --exclude clickhouse
+          # clickhouse needs sudo priviledges
+          sudo python3 _setup_utils/install_all_solutions clickhouse
 
       - name: Run the benchmark
         shell: bash

_setup_utils/install_all_solutions.py

@@ -1,52 +1,83 @@
 import os
 import csv
 import sys
+import subprocess
 
 SOLUTIONS_FILENAME = "_control/solutions.csv"
 
 
+INCLUDE = set()
+
 def install_solution(solution_name):
     min_setup_file_name = f"./{solution_name}/min-setup-{solution_name}.sh"
     setup_file_name = f"./{solution_name}/setup-{solution_name}.sh"
+    upgrade_file_name = f"./{solution_name}/upg-{solution_name}.sh"
+    get_version_filename = f"./{solution_name}/ver-{solution_name}.sh"
     print(f"Installing {solution_name}")
-    if os.path.exists(min_setup_file_name):
-        os.system(min_setup_file_name)
-    elif os.path.exists(setup_file_name):
-        os.system(setup_file_name)
+    do_install = False
+    try:
+        subprocess.call([get_version_filename], stdout="/dev/null")
+    except Exception as e:
+        do_install = True
+
+    if do_install:
+        if os.path.exists(min_setup_file_name):
+            subprocess.call([min_setup_file_name])
+        elif os.path.exists(setup_file_name):
+            subprocess.call([setup_file_name])
+        else:
+            # print(f"no script for {setup_file_name} or {min_setup_file_name}")
+            raise Exception(f"No script to install {solution_name}")
     else:
-        # print(f"no script for {setup_file_name} or {min_setup_file_name}")
-        raise Exception(f"No script to install {solution_name}")
+        subprocess.call([upgrade_file_name])
 
 # based on the name of the solution, run the {{solution}}/min-setup-{{solution}}.sh file.
 # if there is no min-setup-{{solution}}.sh, then run setup-{{solution}}.sh.
 # if error, exit with an error
 # else don't
-def install_all_solutions():
-    install_solutions = set()
+def include_all_solutions():
+    global INCLUDE
     with open(SOLUTIONS_FILENAME, newline="") as solutions_file:
         solutions = csv.DictReader(solutions_file, delimiter=',')
         for row in solutions:
             if row['solution'] == "data.table":
-                install_solutions.add("datatable")
+                INCLUDE.add("datatable")
             else:
-                install_solutions.add(row['solution'])
-    for solution in install_solutions:
-        install_solution(solution)
+                INCLUDE.add(row['solution'])
 
 if len(sys.argv) == 0:
-    print("Usage: python3 install_all_solutions.py solution_name solution_name ...")
+    print("""
+Usage: python3 install_all_solutions.py solution_name solution_name ...
+       python3 install_all_solutions.py all --exclude clickhouse polars
+""")
     exit(1)
 
 # first argument is file name
-for solution in sys.argv[1:]:
-    if solution.strip() == "all":
-        install_all_solutions()
-    else:
-        if solution == "data.table":
-            install_solution("datatable")
-        elif solution == "clickhouse":
-            install_solution("clickhouse")
-            install_solution("polars")
+
+def main():
+    global INCLUDE
+    including = True
+    for solution in sys.argv[1:]:
+        if solution.strip() == "all":
+            include_all_solutions()
+        elif solution.strip() == "--exclude":
+            including = False
         else:
-            install_solution(solution)
-        
+            if including:
+                if solution == "data.table":
+                    INCLUDE.add("datatable")
+                elif solution == "clickhouse":
+                    INCLUDE.add("clickhouse")
+                else:
+                    INCLUDE.add(solution)
+            else:
+                sol = solution.strip()
+                INCLUDE.remove(sol)
+
+    for solution in INCLUDE:
+        install_solution(solution)
+
+
+if __name__ == "__main__":
+    main()
+    

clickhouse/setup-clickhouse.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # install
 sudo apt-get install -y apt-transport-https ca-certificates curl gnupg
 curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' | sudo gpg --dearmor -o /usr/share/keyrings/clickhouse-keyring.gpg

clickhouse/upg-clickhouse.sh

@@ -3,4 +3,4 @@ set -e
 
 # upgrade to latest released
 echo 'upgrading clickhouse-server clickhouse-client...'
-apt-get install --only-upgrade clickhouse-server clickhouse-client
+sudo apt-get install --only-upgrade clickhouse-server clickhouse-client

datatable/setup-datatable.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # install devel data.table
 mkdir -p ./datatable/r-datatable
 Rscript -e 'install.packages("data.table", repos="https://Rdatatable.gitlab.io/data.table", method="curl", lib="./datatable/r-datatable")'

juliadf/setup-juliadf.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # install julia
 
 wget https://julialang-s3.julialang.org/bin/linux/x64/1.10/julia-1.10.5-linux-x86_64.tar.gz

juliads/setup-juliads.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # install julia
 
 wget https://julialang-s3.julialang.org/bin/linux/x64/1.10/julia-1.10.5-linux-x86_64.tar.gz