diff --git a/plugins/module_utils/sap_launchpad_software_center_download_runner.py b/plugins/module_utils/sap_launchpad_software_center_download_runner.py index 86cc55b..d4914fd 100644 --- a/plugins/module_utils/sap_launchpad_software_center_download_runner.py +++ b/plugins/module_utils/sap_launchpad_software_center_download_runner.py @@ -10,6 +10,7 @@ from . import constants as C from .sap_api_common import _request, https_session from .sap_id_sso import _get_sso_endpoint_meta +from .sap_launchpad_software_center_download_search_fuzzy import * logger = logging.getLogger(__name__) @@ -17,27 +18,77 @@ MAX_RETRY_TIMES = 3 -def search_software_filename(name, deduplicate): - """Return a single software that matched the filename +def search_software_filename(name, deduplicate, search_alternatives): """ - search_results = _search_software(name) - softwares = [r for r in search_results if r['Title'] == name or r['Description'] == name] - num_files=len(softwares) - if num_files == 0: - raise ValueError(f'no result found for {name}') - elif num_files > 1 and deduplicate == '': - names = [s['Title'] for s in softwares] - raise ValueError('more than one results were found: %s. ' + Execute search for SAP Software or its alternative when search_alternatives is true. + + Args: + name: The filename name to check (e.g. 'SAPCAR_1115-70006178.EXE'). + deduplicate: Select deduplication logic from 'first', 'last' + search_alternatives: Boolean for enabling fuzzy search. + + Returns: + download_link: Download link of matched SAP Software. + filename: File name of matched SAP Software. + alternative_found: True if alternative search was successful. + """ + + alternative_found = False + software_search = _search_software(name) + software_filtered = [r for r in software_search if r['Title'] == name or r['Description'] == name] + + files_count=len(software_filtered) + if files_count == 0: + # Run fuzzy search if search_alternatives was selected + if search_alternatives: + software_fuzzy_found = search_software_fuzzy(name) + software_fuzzy_filtered, suggested_filename = filter_fuzzy_search(software_fuzzy_found, name) + if len(software_fuzzy_filtered) == 0: + raise ValueError(f'File {name} is not available to download and has no alternatives') + + software_fuzzy_alternatives = software_fuzzy_filtered[0].get('Title') + + # Search has to be filtered again, because API call can get + # duplicates like 70SWPM10SP43_2-20009701.sar for SWPM10SP43_2-20009701.SAR + software_search_alternatives = _search_software(software_fuzzy_alternatives) + software_search_alternatives_filtered = [ + file for file in software_search_alternatives + if file.get('Title', '').startswith(suggested_filename) + ] + alternatives_count=len(software_search_alternatives_filtered) + if alternatives_count == 0: + raise ValueError(f'File {name} is not available to download and has no alternatives') + elif alternatives_count > 1 and deduplicate == '': + names = [s['Title'] for s in software_search_alternatives_filtered] + raise ValueError('More than one results were found: %s. ' + 'please use the correct full filename' % names) + elif alternatives_count > 1 and deduplicate == 'first': + software_found = software_search_alternatives_filtered[0] + alternative_found = True + elif alternatives_count > 1 and deduplicate == 'last': + software_found = software_search_alternatives_filtered[alternatives_count-1] + alternative_found = True + else: + software_found = software_search_alternatives_filtered[0] + alternative_found = True + else: + raise ValueError(f'File {name} is not available to download. Enable "search_alternatives" to search for alternatives.') + + elif files_count > 1 and deduplicate == '': + names = [s['Title'] for s in software_filtered] + raise ValueError('More than one results were found: %s. ' 'please use the correct full filename' % names) - elif num_files > 1 and deduplicate == 'first': - software = softwares[0] - elif num_files > 1 and deduplicate == 'last': - software = softwares[num_files-1] + elif files_count > 1 and deduplicate == 'first': + software_found = software_filtered[0] + elif files_count > 1 and deduplicate == 'last': + software_found = software_filtered[files_count-1] else: - software = softwares[0] + software_found = software_filtered[0] - download_link, filename = software['DownloadDirectLink'], name - return (download_link, filename) + download_link = software_found['DownloadDirectLink'] + filename = _get_valid_filename(software_found) + + return (download_link, filename, alternative_found) def download_software(download_link, filename, output_dir, retry=0): @@ -151,6 +202,7 @@ def download_software_via_legacy_api(username, password, download_link, def _search_software(keyword): + url = C.URL_SOFTWARE_CENTER_SERVICE + '/SearchResultSet' params = { 'SEARCH_MAX_RESULT': 500, @@ -167,7 +219,7 @@ def _search_software(keyword): j = json.loads(res.text) results = j['d']['results'] except json.JSONDecodeError: - # When use has no authority to search some specified softwares, + # When use has no authority to search some specified files, # it will return non-json response, which is actually expected. # So just return an empty list. logger.warning('Non-JSON response returned for software searching') @@ -246,3 +298,27 @@ def _is_checksum_matched(f, etag): for chunk in iter(lambda: f.read(4096 * hash.block_size), b""): hash.update(chunk) return hash.hexdigest() == checksum + + +def _get_valid_filename(software_found): + """ + Ensure that CD Media have correct filenames from description. + Example: S4CORE105_INST_EXPORT_1.zip downloads as 19118000000000004323 + + Args: + software_found: List[0] with dictionary of file. + + Returns: + Valid filename for CD Media files, where applicable. + """ + + # Check if Title contains filename and extension + if re.match(r'^[^/\\\0]+\.[^/\\\0]+$', software_found['Title']): + return software_found['Title'] + else: + # Check if Description contains filename and extension + if re.match(r'^[^/\\\0]+\.[^/\\\0]+$', software_found['Description']): + return software_found['Description'] + else: + # Default to Title if Description does not help + return software_found['Title'] diff --git a/plugins/module_utils/sap_launchpad_software_center_download_search_fuzzy.py b/plugins/module_utils/sap_launchpad_software_center_download_search_fuzzy.py index 98be35f..9fba4cf 100644 --- a/plugins/module_utils/sap_launchpad_software_center_download_search_fuzzy.py +++ b/plugins/module_utils/sap_launchpad_software_center_download_search_fuzzy.py @@ -1,5 +1,7 @@ import csv import logging +import os +import re import requests @@ -8,16 +10,38 @@ def search_software_fuzzy(query, max=None, csv_filename=None): - """Returns a list of dict for the software results. """ - results = _search_software(query) + Execute fuzzy search using Unique Software ID instead of name. + ID is unique to Product and Platform combination. + Example of shared ID 80002616: + - SYBCTRL_1440-80002616.SAR + - SYBCTRL_1436-80002616.SAR + + Args: + query: The filename name to check (e.g. 'SYBCTRL_1440-80002616.SAR'). + + Returns: + The list of dict for the software results. + Empty list is returned if query does not contain ID. + """ + # Format query to split filename. + filename_base = os.path.splitext(query)[0] # Remove extension + + # Ensure that fuzzy search is run only for valid IDs. + # This excludes unique files without ID like: S4CORE105_INST_EXPORT_1.zip + if '-' in filename_base: + filename_id = filename_base.split('-')[-1] # Split id from filename + else: + return [] + + results = _search_software(filename_id) num = 0 - softwares = [] + fuzzy_results = [] while True: for r in results: r = _remove_useless_keys(r) - softwares.append(r) + fuzzy_results.append(r) num += len(results) # quit if no results or results number reach the max if num == 0 or (max and num >= max): @@ -35,9 +59,216 @@ def search_software_fuzzy(query, max=None, csv_filename=None): break if csv_filename: - _write_software_results(softwares, csv_filename) + _write_software_results(fuzzy_results, csv_filename) return - return softwares + return fuzzy_results + + +def filter_fuzzy_search(fuzzy_results, filename): + """ + Filter fuzzy search output using filename. + + Args: + fuzzy_results: Output of search_software_fuzzy. + filename: The filename name to check + + Returns: + fuzzy_results_sorted: The list of files that match the filter criteria, sorted by 'Title' in descending order. + suggested_filename: Return generated keyword for further reuse after API call. + """ + + # Prepare filtered list for specific SPS + suggested_filename = _prepare_search_filename_specific(filename) + + fuzzy_results_filtered = [ + file for file in fuzzy_results + if file.get('Title', '').startswith(suggested_filename) + ] + + # Repeat filtering without specific SPS + if len(fuzzy_results_filtered) == 0: + suggested_filename = _prepare_search_filename_nonspecific(filename) + + fuzzy_results_filtered = [ + file for file in fuzzy_results + if file.get('Title', '').startswith(suggested_filename) + ] + + # fuzzy_results_sorted = sorted(fuzzy_results_filtered, key=lambda item: item.get('Title', ''), reverse=True) + fuzzy_results_sorted =_sort_fuzzy_results(fuzzy_results_filtered, filename) + + return fuzzy_results_sorted, suggested_filename + + +def _prepare_search_filename_specific(filename): + """ + Prepare suggested search keyword for known products specific to SPS version. + + Args: + filename: The filename name to check + + Returns: + Suggested filename to filter fuzzy search. + """ + + # Format query to split filename. + filename_base = os.path.splitext(filename)[0] # Remove extension + filename_name = filename_base.rsplit('_', 1)[0] # Split software name from version + # Following filenames will be processed using default filename_name split. + # Return SYBCTRL for SYBCTRL_1436-80002616.SAR + # Return SMDA720 for SMDA720_SP11_22-80003641.SAR + + + for swpm_version in ("70SWPM1", "70SWPM2", "SWPM1", "SWPM2"): + if filename_base.startswith(swpm_version): + return swpm_version + + # Return SUM11SP04 for SUM11SP04_2-80006858.SAR + if filename_base.startswith('SUM'): + return filename.split('-')[0].split('_')[0] + + # Return DBATL740O11 for DBATL740O11_48-80002605.SAR + elif filename_base.startswith('DBATL'): + return filename.split('-')[0].split('_')[0] + + # Return IMDB_AFL20_077 for IMDB_AFL20_077_0-80002045.SAR + # Return IMDB_AFL100_102P for IMDB_AFL100_102P_41-10012328.SAR + elif filename_base.startswith('IMDB_AFL'): + return "_".join(filename.split('-')[0].split('_')[:3]) + + # Return IMDB_CLIENT20_021 for IMDB_CLIENT20_021_31-80002082.SAR + elif filename_base.startswith('IMDB_CLIENT'): + return "_".join(filename.split('-')[0].split('_')[:3]) + + # IMDB_LCAPPS for SAP HANA 1.0 + # Return IMDB_LCAPPS_122 for IMDB_LCAPPS_122P_3300-20010426.SAR + elif filename_base.startswith('IMDB_LCAPPS_1'): + filename_parts = filename.split('-')[0].rsplit('_', 2) + return f"{filename_parts[0]}_{filename_parts[1][:3]}" + + # IMDB_LCAPPS for SAP HANA 2.0 + # Return IMDB_LCAPPS_206 for IMDB_LCAPPS_2067P_400-80002183.SAR + elif filename_base.startswith('IMDB_LCAPPS_2'): + filename_parts = filename.split('-')[0].rsplit('_', 2) + return f"{filename_parts[0]}_{filename_parts[1][:3]}" + + # Return IMDB_SERVER20_06 (SPS06) for IMDB_SERVER20_067_4-80002046.SAR + elif filename_base.startswith('IMDB_SERVER'): + filename_parts = filename.split('-')[0].rsplit('_', 2) + return f"{filename_parts[0]}_{filename_parts[1][:2]}" + + # Return SAPEXE_100 for SAPEXE_100-80005374.SAR + elif filename_base.startswith('SAPEXE'): + return filename_base.split('-')[0] + + # Return SAPHANACOCKPIT02 (SPS02) for SAPHANACOCKPIT02_0-70002300.SAR + elif filename_base.startswith('SAPHANACOCKPIT'): + return filename_base.split('-')[0].rsplit('_', 1)[0] + + # Return unchanged filename_name + else: + return filename_name + + +def _prepare_search_filename_nonspecific(filename): + """ + Prepare suggested search keyword for known products nonspecific to SPS version. + + Args: + filename: The filename name to check + + Returns: + Suggested filename to filter fuzzy search. + """ + + # Format query to split filename. + filename_base = os.path.splitext(filename)[0] # Remove extension + filename_name = filename_base.rsplit('_', 1)[0] # Split software name from version + + # Return SUM11 for SUM11SP04_2-80006858.SAR + if filename_base.startswith('SUM'): + if filename_base.startswith('SUMHANA'): + return 'SUMHANA' + elif filename_base[3:5].isdigit(): # Allow only SUM and 2 digits + return filename_base[:5] + + # Return DBATL740O11 for DBATL740O11_48-80002605.SAR + elif filename_base.startswith('DBATL'): + return filename.split('-')[0].split('_')[0] + + # Return IMDB_AFL20 for IMDB_AFL20_077_0-80002045.SAR + # Return IMDB_AFL100 for IMDB_AFL100_102P_41-10012328.SAR + elif filename_base.startswith('IMDB_AFL'): + return "_".join(filename.split('-')[0].split('_')[:2]) + + # Return IMDB_CLIENT for IMDB_CLIENT20_021_31-80002082.SAR + elif filename_base.startswith('IMDB_CLIENT'): + return 'IMDB_CLIENT' + + # Return IMDB_LCAPPS for IMDB_LCAPPS_122P_3300-20010426.SAR + elif filename_base.startswith('IMDB_LCAPPS'): + return "_".join(filename.split('-')[0].split('_')[:2]) + + # Return IMDB_SERVER20 for IMDB_SERVER20_067_4-80002046.SAR + elif filename_base.startswith('IMDB_SERVER'): + return "_".join(filename.split('-')[0].split('_')[:2]) + + # Return SAPHANACOCKPIT for SAPHANACOCKPIT02_0-70002300.SAR + elif filename_base.startswith('SAPHANACOCKPIT'): + return 'SAPHANACOCKPIT' + + # Return SAPHOSTAGENT for SAPHOSTAGENT61_61-80004831.SAR + elif filename_base.startswith('SAPHOSTAGENT'): + return 'SAPHOSTAGENT' + + # Return unchanged filename_name + else: + return filename_name + + +def _sort_fuzzy_results(fuzzy_results_filtered, filename): + """ + Sort results of fuzzy search for known nonstandard versions. + Example: + IMDB_LCAPPS_122P_3500-20010426.SAR, IMDB_LCAPPS_122P_600-70001332.SAR + + Args: + fuzzy_results_filtered: The list of filtered fuzzy results. + filename: The filename name to check. + + Returns: + Ordered list of fuzzy results, based on known nonstandard versions. + """ + + if _get_numeric_search_keyword(filename): + software_fuzzy_sorted = sorted( + fuzzy_results_filtered, + key= lambda item: _get_numeric_search_keyword(item.get('Title', '')), + reverse=True, + ) + else: + software_fuzzy_sorted = sorted( + fuzzy_results_filtered, + key=lambda item: item.get('Title', ''), + reverse=True, + ) + + return software_fuzzy_sorted + + +def _get_numeric_search_keyword(filename): + """ + Extract integer value of version from filename. + + Args: + filename: The filename name to check. + + """ + match = re.search(r'_(\d+)-', filename) + if match: + return int(match.group(1)) + else: + return None def _search_software(keyword, remove_useless_keys=False): diff --git a/plugins/modules/software_center_download.py b/plugins/modules/software_center_download.py index 952af26..f6d9002 100644 --- a/plugins/modules/software_center_download.py +++ b/plugins/modules/software_center_download.py @@ -26,6 +26,14 @@ required: true type: str softwarecenter_search_query: + description: + - "Deprecated. Use 'search_query' instead." + required: false + type: str + deprecated: + alternative: search_query + removed_in: "1.2.0" + search_query: description: - Filename of the SAP software to download. required: false @@ -40,9 +48,9 @@ - Download filename of the SAP software. required: false type: str - dest: + download_path: description: - - Destination folder. + - Destination folder path. required: true type: str deduplicate: @@ -50,8 +58,18 @@ - How to handle multiple search results. required: false type: str + search_alternatives: + description: + - Enable search for alternative packages, when filename is not available. + required: false + type: bool + dry_run: + description: + - Check availability of SAP Software without downloading. + required: false + type: bool author: - - Lab for SAP Solutions + - SAP LinuxLab ''' @@ -60,16 +78,16 @@ community.sap_launchpad.sap_launchpad_software_center_download: suser_id: 'SXXXXXXXX' suser_password: 'password' - softwarecenter_search_query: + search_query: - 'SAPCAR_1324-80000936.EXE' - dest: "/tmp/" + download_path: "/tmp/" - name: Download using direct link and filename community.sap_launchpad.software_center_download: suser_id: 'SXXXXXXXX' suser_password: 'password' download_link: 'https://softwaredownloads.sap.com/file/0010000000048502015' download_filename: 'IW_FNDGC100.SAR' - dest: "/tmp/" + download_path: "/tmp/" ''' RETURN = r''' @@ -77,6 +95,14 @@ description: the status of the process returned: always type: str +filename: + description: the name of the original or alternative file found to download. + returned: always + type: str +alternative: + description: true if alternative file was found + returned: always + type: bool ''' @@ -98,17 +124,13 @@ def run_module(): suser_id=dict(type='str', required=True), suser_password=dict(type='str', required=True, no_log=True), softwarecenter_search_query=dict(type='str', required=False, default=''), + search_query=dict(type='str', required=False, default=''), download_link=dict(type='str', required=False, default=''), download_filename=dict(type='str', required=False, default=''), - dest=dict(type='str', required=True), + download_path=dict(type='str', required=True), dry_run=dict(type='bool', required=False, default=False), - deduplicate=dict(type='str', required=False, default='') - ) - - # Define result dictionary objects to be passed back to Ansible - result = dict( - changed=False, - msg='' + deduplicate=dict(type='str', required=False, default=''), + search_alternatives=dict(type='bool', required=False, default=False) ) # Instantiate module @@ -117,52 +139,128 @@ def run_module(): supports_check_mode=True ) - # Check mode - if module.check_mode: - module.exit_json(**result) - # Define variables based on module inputs username = module.params.get('suser_id') password = module.params.get('suser_password') - query = module.params.get('softwarecenter_search_query') + + if module.params['search_query'] != '': + query = module.params['search_query'] + elif module.params['softwarecenter_search_query'] != '': + query = module.params['softwarecenter_search_query'] + module.warn("The 'softwarecenter_search_query' is deprecated and will be removed in a future version. Use 'search_query' instead.") + else: + query = None + + download_path = module.params['download_path'] download_link= module.params.get('download_link') download_filename= module.params.get('download_filename') - dest = module.params.get('dest') dry_run = module.params.get('dry_run') deduplicate = module.params.get('deduplicate') + search_alternatives = module.params.get('search_alternatives') + + # Define result dictionary objects to be passed back to Ansible + result = dict( + changed=False, + msg='', + filename=download_filename, + alternative=False, + skipped=False, + failed=False + ) + + # Check mode + if module.check_mode: + module.exit_json(**result) - # Main run + # Main try: - # Search directory and subdirectories for filename without file extension - filename = query if query else download_filename - pattern = dest + '/**/' + os.path.splitext(filename)[0] + '*' - for file in glob.glob(pattern, recursive=True): - if os.path.exists(file): - module.exit_json(skipped=True, msg="file {} already exists".format(file)) + # Ensure that required parameters are provided + if not (query or (download_link and download_filename)): + module.fail_json( + msg="Either 'search_query' or both 'download_link' and 'download_filename' must be provided." + ) + # Search for existing files using exact filename + filename = query if query else download_filename + filename_exact = os.path.join(download_path, filename) + result['filename'] = filename + + if os.path.exists(filename_exact): + result['skipped'] = True + result['msg'] = f"File already exists: {filename}" + module.exit_json(**result) + else: + # Exact file not found, search with pattern + # pattern = download_path + '/**/' + os.path.splitext(filename)[0] + '*' # old pattern + filename_base = os.path.splitext(filename)[0] + filename_ext = os.path.splitext(filename)[1] + filename_pattern = os.path.join(download_path, "**", filename_base + "*" + filename_ext) + filename_similar = glob.glob(filename_pattern, recursive=True) + + # Skip if similar files were found and search_alternatives was not set. + if filename_similar and not (query and search_alternatives): + filename_similar_names = [os.path.basename(f) for f in filename_similar] + result['skipped'] = True + result['msg'] = f"Similar file(s) already exist: {', '.join(filename_similar_names)}" + module.exit_json(**result) + # Initiate login with given credentials sap_sso_login(username, password) - # EXEC: query - # execute search_software_filename first to get download link and filename + # Execute search_software_filename first to get download link and filename + alternative_found = False # True if search_alternatives was successful if query: - download_link, download_filename = search_software_filename(query,deduplicate) - - # execute download_software + download_link, download_filename, alternative_found = search_software_filename(query,deduplicate,search_alternatives) + + # Search for existing files again with alternative filename + if search_alternatives and alternative_found: + result['filename'] = download_filename + result['alternative'] = True + + filename_alternative_exact = os.path.join(download_path, download_filename) + if os.path.exists(filename_alternative_exact): + result['skipped'] = True + result['msg'] = f"Alternative file already exists: {download_filename} - original file {query} is not available to download" + module.exit_json(**result) + else: + filename_alternative_base = os.path.splitext(download_filename)[0] + filename_alternative_ext = os.path.splitext(download_filename)[1] + filename_alternative_pattern = os.path.join(download_path, "**", filename_alternative_base + "*" + filename_alternative_ext) + filename_alternative_similar = glob.glob(filename_alternative_pattern, recursive=True) + + # Skip if similar files were found and search_alternatives was not set. + if filename_alternative_similar: + filename_alternative_similar_names = [os.path.basename(f) for f in filename_alternative_similar] + result['skipped'] = True + result['msg'] = f"Similar alternative file(s) already exist: {', '.join(filename_alternative_similar_names)}" + module.exit_json(**result) + + # Ensure that download_link is provided when query was not provided + if not download_link: + module.fail_json(msg="Missing parameters 'query' or 'download_link'.") + + # Exit module before download when dry_run is true if dry_run: available = is_download_link_available(download_link) - if available: - module.exit_json(changed=False, msg="download link {} is available".format(download_link)) + if available and query and not alternative_found: + result['msg'] = f"SAP Software is available to download: {download_filename}" + module.exit_json(**result) + elif available and query and alternative_found: + result['msg'] = f"Alternative SAP Software is available to download: {download_filename} - original file {query} is not available to download" + module.exit_json(**result) else: - module.fail_json(msg="download link {} is not available".format(download_link)) + module.fail_json(msg="Download link {} is not available".format(download_link)) - download_software(download_link, download_filename, dest) + download_software(download_link, download_filename, download_path) - # Process return dictionary for Ansible + # Update final results json result['changed'] = True - result['msg'] = "SAP software download successful" + if query and alternative_found: + result['msg'] = f"Successfully downloaded alternative SAP software: {download_filename} - original file {query} is not available to download" + else: + result['msg'] = f"Successfully downloaded SAP software: {download_filename}" except requests.exceptions.HTTPError as e: # module.fail_json(msg='SAP SSO authentication failed' + str(e), **result)