Skip to content

add script to upload packages to web server #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
orenreiss wants to merge 3 commits into from
Closed

add script to upload packages to web server #1

Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0815e1e
add script to upload packages to web server
orenreiss Dec 20, 2022
8c47a70
fixes in script that upload packages to web server following code review
orenreiss Dec 21, 2022
c21a38f
fixes in script that upload packages to web server following code review
orenreiss Dec 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 21 additions & 32 deletions scripts/populate_file_web_server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
#!/usr/bin/python3
Copy link
Copy Markdown

@dgsudharsan dgsudharsan Dec 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#!/usr/bin/env python3

Copy link
Copy Markdown
Owner Author

@orenreiss orenreiss Dec 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done


import sys
import os
import time
Expand Down Expand Up @@ -72,40 +74,27 @@ def __str__(self):

# Helper functions

def info_print(msg):
if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[PRINT_LEVEL_INFO]:
print(msg)

def info_print_in_place(msg):
if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[PRINT_LEVEL_INFO]:
print(msg, end='\r')

def verbose_print(msg):
if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[PRINT_LEVEL_VERBOSE]:
print(msg)
def print_msg(print_level, msg, print_in_place):
Copy link
Copy Markdown

@dgsudharsan dgsudharsan Dec 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can have default value for print_in_place=False. In the invocations below other than when it needs to be true, you can ignore this parameter.

Copy link
Copy Markdown
Owner Author

@orenreiss orenreiss Dec 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[print_level]:
if True == print_in_place:
print(msg, end='\r')
else:
print(msg)

def warn_print(msg):
if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[PRINT_LEVEL_WARN]:
print(msg)

def error_print(msg):
if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[PRINT_LEVEL_ERROR]:
print(msg)

def create_dir_if_not_exist(dir):
if not os.path.exists(dir):
try:
os.mkdir(dir)
except:
warn_print("Cannot create directory " + dir)
print_msg(PRINT_LEVEL_WARN, "Cannot create directory " + dir, False)
Copy link
Copy Markdown

@dgsudharsan dgsudharsan Dec 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please remove False after setting the default value. Applies to other changes below

Copy link
Copy Markdown
Owner Author

@orenreiss orenreiss Dec 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done


def delete_file_if_exist(file):

if os.path.exists(file):
try:
os.remove(file)
except:
warn_print("Cannot delete " + file)
print_msg(PRINT_LEVEL_WARN, "Cannot delete " + file, False)

# Logic functions

Expand All @@ -124,7 +113,7 @@ def generate_output_file(resources, dest_url_valid, dest_url, output_file_name):
line += resource.name + "==" + resource.hash
f.write(line + '\n')
except:
warn_print(output_file_name + " cannot be created")
print_msg(PRINT_LEVEL_WARN, output_file_name + " cannot be created", False)
return RET_CODE_CANNOT_CREATE_FILE

f.close()
Expand Down Expand Up @@ -158,7 +147,7 @@ def upload_resource_to_server(resource_path, resource_name, user, key, server_ur
# Check if server reports checksum, if so compare reported sum and the one
# specified in filename
if reported_md5 != None and reported_md5 != file_md5:
warn_print(f"Server reported file's chsum {reported_md5}, expected {file_md5}")
print_msg(PRINT_LEVEL_WARN, f"Server reported file's chsum {reported_md5}, expected {file_md5}", False)


return RET_CODE_SUCCESS
Expand All @@ -174,7 +163,7 @@ def download_external_resouce(resource, cache_path):
f.write(r.content)
f.close()
except:
error_print("Cannot write " + resource_path_in_cache + " to cache")
print_msg(PRINT_LEVEL_ERROR, "Cannot write " + resource_path_in_cache + " to cache", False)
resource_path_in_cache = "" #report error

return resource_path_in_cache
Expand All @@ -189,7 +178,7 @@ def get_resources_list(resource_files_list):
for line in f:
resource_list.append(Resource(line, file_name))
except:
warn_print(file_name + " cannot be opened")
print_msg(PRINT_LEVEL_WARN, file_name + " cannot be opened", False)

return resource_list

Expand All @@ -210,7 +199,7 @@ def get_resource_files_list(serach_path):
for file in files:
if False == filter_out_dir(subdir) and RESOURCES_FILE_NAME == file:
Copy link
Copy Markdown

@dgsudharsan dgsudharsan Dec 21, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How are the version-web files going to be maintained? Will it be populated manually? Can you please provide some insights?

Copy link
Copy Markdown
Owner Author

@orenreiss orenreiss Dec 21, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

version-web are generated as part of the flow of first "SONiC reproducible build". Later they are being used on any compilation. The script is only parsing the existing files in the specified path

file_full_path = os.path.join(subdir, file)
verbose_print("Found resource file :" + file_full_path)
print_msg(PRINT_LEVEL_VERBOSE, "Found resource file :" + file_full_path, False)
resource_files_list.append(file_full_path)

return resource_files_list
Expand Down Expand Up @@ -264,14 +253,14 @@ def main():
if not unique_name in resource_dict.keys():
resource_dict[unique_name] = resource

info_print("Found " + str(len(resource_files_list)) + " version files and " + str(len(resource_dict.keys())) + " unique resources")
print_msg(PRINT_LEVEL_INFO, "Found " + str(len(resource_files_list)) + " version files and " + str(len(resource_dict.keys())) + " unique resources", False)

if args.dest != DEFAULT_INVALID_INPUT:
upload_files_to_server = True
info_print("Upload files to URL - " + args.dest)
print_msg(PRINT_LEVEL_INFO, "Upload files to URL - " + args.dest, False)
else:
upload_files_to_server = False
info_print("Skipping files upload to server")
print_msg(PRINT_LEVEL_INFO, "Skipping files upload to server", False)

#create cache directory if not exist
create_dir_if_not_exist(args.cache)
Expand All @@ -281,7 +270,7 @@ def main():

resource = resource_dict[unique_name]

verbose_print(resource)
print_msg(PRINT_LEVEL_VERBOSE, resource, False)

resource_counter += 1.0

Expand All @@ -300,12 +289,12 @@ def main():
if True == g_delete_resources_in_cache:
delete_file_if_exist(file_in_cache)

info_print_in_place("Downloading Data. Progress " + str(int(100.0*resource_counter/len(resource_dict.keys()))) + "%") #print progress bar
print_msg(PRINT_LEVEL_INFO, "Downloading Data. Progress " + str(int(100.0*resource_counter/len(resource_dict.keys()))) + "%", True) #print progress bar

# generate version output file as needed
if args.output != DEFAULT_INVALID_INPUT:
ret_val = generate_output_file(resource_dict, upload_files_to_server, args.dest, args.output)
info_print("Generate output file " + args.output)
print_msg(PRINT_LEVEL_INFO, "Generate output file " + args.output, False)

return ret_val

Expand Down