Validate runtime image environments in CI testing

.github/workflows/build.yml

@@ -172,6 +172,18 @@ jobs:
       - name: Build
         run: make docs
 
+  validate-image-env:
+    name: Validate Image Environment
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9, "3.10"]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Validate image environment
+        run: make PYTHON_VERSION=${{ matrix.python-version }} elyra-image-env
+
+
   validate-images:
     name: Validate Images
     runs-on: ubuntu-latest

Makefile

@@ -21,18 +21,25 @@
 .PHONY: install install-all install-dev install-examples install-gitlab-dependency check-install watch release
 .PHONY: test-dependencies pytest test-server test-ui-unit test-integration test-integration-debug test-ui test
 .PHONY: docs-dependencies docs
-.PHONY: elyra-image publish-elyra-image kf-notebook-image publish-kf-notebook-image
+.PHONY: elyra-image elyra-image-env publish-elyra-image kf-notebook-image publish-kf-notebook-image
 .PHONY: container-images publish-container-images validate-runtime-images
+
+.ONESHELL:
+
 SHELL:=/bin/bash
 
 # Python execs
 PYTHON?=python3
 PYTHON_PIP=$(PYTHON) -m pip
+PYTHON_VERSION?=3.9
+
+CONDA_ACTIVATE = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate
 
 ELYRA_VERSION:=$$(grep __version__ elyra/_version.py | cut -d"\"" -f2)
 TAG:=dev
 ELYRA_IMAGE=elyra/elyra:$(TAG)
 ELYRA_IMAGE_LATEST=elyra/elyra:latest
+ELYRA_IMAGE_ENV?=elyra-image-env
 KF_NOTEBOOK_IMAGE=elyra/kf-notebook:$(TAG)
 KF_NOTEBOOK_IMAGE_LATEST=elyra/kf-notebook:latest
 
@@ -189,6 +196,20 @@ watch: ## Watch packages. For use alongside jupyter lab --watch
 
 release: yarn-install build-ui build-server ## Build wheel file for release
 
+
+elyra-image-env: ## Creates a conda env consisting of the dependencies used in images
+	conda env remove -y -n $(ELYRA_IMAGE_ENV)
+	conda create -y -n $(ELYRA_IMAGE_ENV) python=$(PYTHON_VERSION)
+	if [ "$(PYTHON_VERSION)" == "3.7" ]; then \
+		$(CONDA_ACTIVATE) $(ELYRA_IMAGE_ENV) && \
+		$(PYTHON_PIP) install -r etc/generic/requirements-elyra-py37.txt && \
+		conda deactivate; \
+	else \
+		$(CONDA_ACTIVATE) $(ELYRA_IMAGE_ENV) && \
+		$(PYTHON_PIP) install -r etc/generic/requirements-elyra.txt && \
+		conda deactivate; \
+	fi
+
 ## Test targets
 
 test-dependencies:
@@ -308,6 +329,26 @@ validate-runtime-images: # Validates delivered runtime-images meet minimum crite
 				echo ERROR: Image $$image did not meet criteria for command: $$cmd ; \
 				fail=1; \
 			fi; \
+			if [ $$cmd == "python3" ]; then \
+				IMAGE_PYTHON3_MINOR_VERSION=`docker run --rm $$image $$cmd --version | cut -d' ' -f2 | cut -d'.' -f2` ; \
+				if [[ $$IMAGE_PYTHON3_MINOR_VERSION -lt 8 ]]; then \
+					echo WARNING: Image $$image requires at Python 3.8 or greater for latest generic component dependency installation; \
+					docker run -v $$(pwd)/etc/generic:/opt/elyra/ --rm $$image python3 -m pip install -r /opt/elyra/requirements-elyra-py37.txt > /dev/null ; \
+					if [ $$? -ne 0 ]; then \
+						echo ERROR: Image $$image did not meet python requirements criteria in requirements-elyra-py37.txt ; \
+						fail=1; \
+					fi; \
+				elif [[ $$IMAGE_PYTHON3_MINOR_VERSION -ge 8 ]]; then \
+					docker run -v $$(pwd)/etc/generic:/opt/elyra/ --rm $$image python3 -m pip install -r /opt/elyra/requirements-elyra.txt > /dev/null ; \
+					if [ $$? -ne 0 ]; then \
+						echo ERROR: Image $$image did not meet python requirements criteria in requirements-elyra.txt ; \
+						fail=1; \
+					fi; \
+				else \
+					echo ERROR: Image $$image unable to parse python version ; \
+					fail=1; \
+				fi; \
+			fi; \
 		done; \
 		if [ $(REMOVE_RUNTIME_IMAGE) -eq 1 ]; then \
 			echo Removing image $$image... ; \

etc/generic/requirements-elyra-py37.txt

@@ -0,0 +1,65 @@
+# This is a comprehensive list of python dependencies that Elyra requires to execute Jupyter notebooks.
+ipykernel==6.13.0
+ipython==7.33.0
+jinja2==3.1.2
+jupyter-client==7.3.1
+jupyter-core==4.10.0
+minio==7.1.8
+nbclient==0.6.3
+nbconvert==5.6.1
+nbformat==5.4.0
+papermill==2.3.4
+pyzmq==22.3.0
+prompt-toolkit==3.0.29
+requests==2.27.1
+tornado==6.1
+traitlets==5.2.0
+urllib3==1.26.9
+#
+# These excluded are transitive dependencies of the included python packages.
+#ansiwrap==0.8.4
+#appnope==0.1.3
+#attrs==21.4.0
+#backcall==0.2.0
+#beautifulsoup4==4.11.1
+#bleach==5.0.0
+#certifi==2021.10.8
+#charset-normalizer==2.0.12
+#click==8.1.3
+#debugpy==1.6.0
+#decorator==5.1.1
+#defusedxml==0.7.1
+#entrypoints==0.4
+#fastjsonschema==2.15.3
+#idna==3.3
+#importlib-metadata==4.11.3
+#importlib-resources==5.7.1
+#jedi==0.18.1
+#jsonschema==4.5.1
+#jupyterlab-pygments==0.2.2
+#MarkupSafe==2.1.1
+#matplotlib-inline==0.1.3
+#mistune==0.8.4
+#nest-asyncio==1.5.5
+#packaging==21.3
+#pandocfilters==1.5.0
+#parso==0.8.3
+#pexpect==4.8.0
+#pickleshare==0.7.5
+#psutil==5.9.0
+#ptyprocess==0.7.0
+#Pygments==2.12.0
+#pyparsing==3.0.9
+#pyrsistent==0.18.1
+#python-dateutil==2.8.2
+#PyYAML==6.0
+#six==1.16.0
+#soupsieve==2.3.2.post1
+#tenacity==8.0.1
+#textwrap3==0.9.2
+#tinycss2==1.1.1
+#tqdm==4.64.0
+#typing_extensions==4.2.0
+#wcwidth==0.2.5
+#webencodings==0.5.1
+#zipp==3.8.0

etc/generic/requirements-elyra.txt

@@ -1,67 +1,69 @@
 # This is a comprehensive list of python dependencies that Elyra requires to execute Jupyter notebooks.
-ipykernel==6.9.1
-ipython==8.0.1
+ipykernel==6.13.0
+ipython==8.3.0
 ipython-genutils==0.2.0
-jinja2==3.0.3
-jupyter-client==7.1.2
-jupyter-core==4.9.2
+jinja2==3.1.2
+jupyter-client==7.3.1
+jupyter-core==4.10.0
 MarkupSafe==2.1.1
-minio==7.1.2
-nbclient==0.4.1
-nbconvert==6.4.5
-nbformat==5.1.3
+minio==7.1.8
+nbclient==0.6.3
+nbconvert==6.5.0
+nbformat==5.4.0
 papermill==2.3.4
-pyzmq==19.0.1
-prompt-toolkit==3.0.5
-requests==2.25.1
-tornado==6.0.4
+pyzmq==22.3.0
+prompt-toolkit==3.0.29
+requests==2.27.1
+tornado==6.1
 traitlets==5.1.1
-urllib3==1.26.5
+urllib3==1.26.9
 #
 # These excluded are transitive dependencies of the included python packages.
 #ansiwrap==0.8.4
-#appdirs==1.4.4
-#appnope==0.1.0
-#async-generator==1.10
-#attrs==19.3.0
+#appnope==0.1.3
+#asttokens==2.0.5
+#attrs==21.4.0
 #backcall==0.2.0
-#black==19.10b0
-#bleach==3.1.5
-#certifi==2020.4.5.2
-#chardet==3.0.4
-#click==7.1.2
-#configparser==5.0.0
-#decorator==4.4.2
-#defusedxml==0.6.0
-#entrypoints==0.3
-#idna==2.9
-#importlib-metadata==1.6.1
-#jedi==0.17.0
-#jsonschema==3.2.0
+#beautifulsoup4==4.11.1
+#bleach==5.0.0
+#certifi==2021.10.8
+#charset-normalizer==2.0.12
+#click==8.1.3
+#debugpy==1.6.0
+#decorator==5.1.1
+#defusedxml==0.7.1
+#entrypoints==0.4
+#executing==0.8.3
+#fastjsonschema==2.15.3
+#idna==3.3
+#importlib-resources==5.7.1
+#jedi==0.18.1
+#jsonschema==4.5.1
+#jupyterlab-pygments==0.2.2
+#matplotlib-inline==0.1.3
 #mistune==0.8.4
-#nest-asyncio==1.3.3
-#packaging==20.4
-#pandocfilters==1.4.2
-#parso==0.7.0
-#pathspec==0.8.0
+#nest-asyncio==1.5.5
+#packaging==21.3
+#pandocfilters==1.5.0
+#parso==0.8.3
 #pexpect==4.8.0
 #pickleshare==0.7.5
-#ptyprocess==0.6.0
-#Pygments==2.6.1
-#pyparsing==2.4.7
-#pyrsistent==0.16.0
-#python-dateutil==2.8.1
-#pytz==2020.1
-#PyYAML==5.3.1
-#regex==2020.6.8
-#six==1.15.0
-#tenacity==6.2.0
-#testpath==0.4.4
+#psutil==5.9.0
+#ptyprocess==0.7.0
+#pure-eval==0.2.2
+#Pygments==2.12.0
+#pyparsing==3.0.9
+#pyrsistent==0.18.1
+#python-dateutil==2.8.2
+#PyYAML==6.0
+#six==1.16.0
+#soupsieve==2.3.2.post1
+#stack-data==0.2.0
+#tenacity==8.0.1
 #textwrap3==0.9.2
-#toml==0.10.1
-#tqdm==4.46.1
-#typed-ast==1.4.1
-#wcwidth==0.2.4
+#tinycss2==1.1.1
+#tqdm==4.64.0
+#wcwidth==0.2.5
 #webencodings==0.5.1
-#zipp==3.1.0
+#zipp==3.8.0
 #