From fd2e72ce66dab72bc3e52795eb6e43b19f0ab14a Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Mon, 26 Aug 2019 14:26:07 -0700 Subject: [PATCH 1/8] [build_debian] Generate checksum of ASIC config files * Adds script to generate checksums for ASIC config files * Adds step to build_debian that copies ASIC config checksum into SONiC filesystem Signed-off-by: Danny Allen daall@microsoft.com --- build_debian.sh | 8 +++ .../generate_asic_config_checksum.py | 69 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 files/build_scripts/generate_asic_config_checksum.py diff --git a/build_debian.sh b/build_debian.sh index 126a8ee329e..d4e20334f6f 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -458,6 +458,14 @@ build_number: ${BUILD_NUMBER:-0} built_by: $USER@$BUILD_HOSTNAME EOF +## Copy ASIC config checksum +python files/build_scripts/generate_asic_config_checksum.py +if [ ! -f asic_config_checksum ]; then + echo 'asic_config_checksum not found' + exit 1 +fi +sudo cp asic_config_checksum $FILESYSTEM_ROOT/etc/sonic/asic_config_checksum + if [ -f sonic_debian_extension.sh ]; then ./sonic_debian_extension.sh $FILESYSTEM_ROOT $PLATFORM_DIR fi diff --git a/files/build_scripts/generate_asic_config_checksum.py b/files/build_scripts/generate_asic_config_checksum.py new file mode 100644 index 00000000000..92d922fecab --- /dev/null +++ b/files/build_scripts/generate_asic_config_checksum.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +import syslog +import errno +import hashlib + +SYSLOG_IDENTIFIER = 'asic_config_checksum' + +CHUNK_SIZE = 8192 + +SAMPLE_PATH = 'src/sonic-swss/swssconfig/sample/' +SAMPLE_FILES = ['netbouncer.json', '00-copp.config.json'] + +OUTPUT_PATH = './' +OUTPUT_FILE = 'asic_config_checksum' + +def log_info(msg): + syslog.openlog(SYSLOG_IDENTIFIER) + syslog.syslog(syslog.LOG_INFO, msg) + syslog.closelog() + +def log_error(msg): + syslog.openlog(SYSLOG_IDENTIFIER) + syslog.syslog(syslog.LOG_ERR, msg) + syslog.closelog() + +def get_samples(): + ''' + Gathers up ASIC config file names from the appropriately named 'sample' + directory in SWSS + ''' + checksum_files = [] + for file in SAMPLE_FILES: + checksum_files.append(SAMPLE_PATH + file) + return checksum_files + +def generate_checksum(checksum_files): + ''' + Iterates through all of the given ASIC config file, reads their contents, + and generates a checksum. NOTE: The checksum is performed in the order + provided. This function does NOT do any re-ordering of the files before + creating the checksum. + ''' + checksum = hashlib.md5() + for file in checksum_files: + try: + with open(file, 'r') as f: + for chunk in iter(lambda: f.read(CHUNK_SIZE), b""): + checksum.update(chunk) + except IOError as e: + log_error('Error processing ASIC config file ' + file + ':' + e.strerror) + return None + + return checksum.hexdigest() + +def main(): + checksum_files = [] + checksum_files += get_samples() + + checksum_files.sort() + checksum = generate_checksum(checksum_files) + if checksum == None: + exit(1) + + with open(OUTPUT_FILE, 'w') as output: + output.write(checksum + '\n') + +if __name__ == '__main__': + main() From 5e97d25cbebc08343a12e10e313d2f28a21ca1e7 Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Mon, 26 Aug 2019 16:18:56 -0700 Subject: [PATCH 2/8] Fix typo --- build_debian.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_debian.sh b/build_debian.sh index d4e20334f6f..2abd8c6a0fe 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -460,7 +460,7 @@ EOF ## Copy ASIC config checksum python files/build_scripts/generate_asic_config_checksum.py -if [ ! -f asic_config_checksum ]; then +if [[ ! -f './asic_config_checksum' ]]; then echo 'asic_config_checksum not found' exit 1 fi From 34b5de3242e9d7a4d92a295d63541908b71802c3 Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Mon, 26 Aug 2019 17:19:51 -0700 Subject: [PATCH 3/8] Remove unused variable --- files/build_scripts/generate_asic_config_checksum.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/files/build_scripts/generate_asic_config_checksum.py b/files/build_scripts/generate_asic_config_checksum.py index 92d922fecab..64ef0b314d9 100644 --- a/files/build_scripts/generate_asic_config_checksum.py +++ b/files/build_scripts/generate_asic_config_checksum.py @@ -11,8 +11,7 @@ SAMPLE_PATH = 'src/sonic-swss/swssconfig/sample/' SAMPLE_FILES = ['netbouncer.json', '00-copp.config.json'] -OUTPUT_PATH = './' -OUTPUT_FILE = 'asic_config_checksum' +OUTPUT_FILE = './asic_config_checksum' def log_info(msg): syslog.openlog(SYSLOG_IDENTIFIER) From facb4f2f0bcca24c0d7393227289ad12b0c5c5f1 Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Wed, 28 Aug 2019 01:15:49 -0700 Subject: [PATCH 4/8] Incorporate feedbacK --- .../generate_asic_config_checksum.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/files/build_scripts/generate_asic_config_checksum.py b/files/build_scripts/generate_asic_config_checksum.py index 64ef0b314d9..36c6d9870cf 100644 --- a/files/build_scripts/generate_asic_config_checksum.py +++ b/files/build_scripts/generate_asic_config_checksum.py @@ -1,17 +1,18 @@ #!/usr/bin/env python import syslog -import errno +import os import hashlib SYSLOG_IDENTIFIER = 'asic_config_checksum' CHUNK_SIZE = 8192 -SAMPLE_PATH = 'src/sonic-swss/swssconfig/sample/' -SAMPLE_FILES = ['netbouncer.json', '00-copp.config.json'] +CONFIG_FILES = { + os.path.abspath('./src/sonic-swss/swssconfig/sample/'): ['netbouncer.json', '00-copp.config.json'] +} -OUTPUT_FILE = './asic_config_checksum' +OUTPUT_FILE = os.path.abspath('./asic_config_checksum') def log_info(msg): syslog.openlog(SYSLOG_IDENTIFIER) @@ -23,14 +24,15 @@ def log_error(msg): syslog.syslog(syslog.LOG_ERR, msg) syslog.closelog() -def get_samples(): +def get_config_files(): ''' Gathers up ASIC config file names from the appropriately named 'sample' directory in SWSS ''' checksum_files = [] - for file in SAMPLE_FILES: - checksum_files.append(SAMPLE_PATH + file) + for path, files in CONFIG_FILES.items(): + for file in files: + checksum_files.append(os.path.join(path, file)) return checksum_files def generate_checksum(checksum_files): @@ -40,7 +42,7 @@ def generate_checksum(checksum_files): provided. This function does NOT do any re-ordering of the files before creating the checksum. ''' - checksum = hashlib.md5() + checksum = hashlib.sha1() for file in checksum_files: try: with open(file, 'r') as f: @@ -53,11 +55,8 @@ def generate_checksum(checksum_files): return checksum.hexdigest() def main(): - checksum_files = [] - checksum_files += get_samples() - - checksum_files.sort() - checksum = generate_checksum(checksum_files) + config_files = sorted(get_config_files()) + checksum = generate_checksum(config_files) if checksum == None: exit(1) From 669db820eea8d0f8919ec127d6c9a137b676675f Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Wed, 28 Aug 2019 10:39:01 -0700 Subject: [PATCH 5/8] Incorporate feedback --- build_debian.sh | 2 +- .../generate_asic_config_checksum.py | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/build_debian.sh b/build_debian.sh index 2abd8c6a0fe..7c7fdce577b 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -464,7 +464,7 @@ if [[ ! -f './asic_config_checksum' ]]; then echo 'asic_config_checksum not found' exit 1 fi -sudo cp asic_config_checksum $FILESYSTEM_ROOT/etc/sonic/asic_config_checksum +sudo cp ./asic_config_checksum $FILESYSTEM_ROOT/etc/sonic/asic_config_checksum if [ -f sonic_debian_extension.sh ]; then ./sonic_debian_extension.sh $FILESYSTEM_ROOT $PLATFORM_DIR diff --git a/files/build_scripts/generate_asic_config_checksum.py b/files/build_scripts/generate_asic_config_checksum.py index 36c6d9870cf..1de0930d538 100644 --- a/files/build_scripts/generate_asic_config_checksum.py +++ b/files/build_scripts/generate_asic_config_checksum.py @@ -24,23 +24,23 @@ def log_error(msg): syslog.syslog(syslog.LOG_ERR, msg) syslog.closelog() -def get_config_files(): +def get_config_files(config_file_map): ''' - Gathers up ASIC config file names from the appropriately named 'sample' - directory in SWSS + Generates a list of absolute paths to ASIC config files. ''' - checksum_files = [] - for path, files in CONFIG_FILES.items(): + config_files = [] + for path, files in config_file_map.items(): for file in files: - checksum_files.append(os.path.join(path, file)) - return checksum_files + config_files.append(os.path.join(path, file)) + return config_files def generate_checksum(checksum_files): ''' - Iterates through all of the given ASIC config file, reads their contents, - and generates a checksum. NOTE: The checksum is performed in the order - provided. This function does NOT do any re-ordering of the files before - creating the checksum. + Generates a checksum for a given list of files. Returns None if an error + occurs while reading the files. + + NOTE: The checksum is performed in the order provided. This function does + NOT do any re-ordering of the files before creating the checksum. ''' checksum = hashlib.sha1() for file in checksum_files: @@ -55,7 +55,7 @@ def generate_checksum(checksum_files): return checksum.hexdigest() def main(): - config_files = sorted(get_config_files()) + config_files = sorted(get_config_files(CONFIG_FILES)) checksum = generate_checksum(config_files) if checksum == None: exit(1) From 30499a5f577aedd5682a94d6197811ae297c6454 Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Wed, 28 Aug 2019 17:34:34 -0700 Subject: [PATCH 6/8] Incorporate feedback --- files/build_scripts/generate_asic_config_checksum.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/files/build_scripts/generate_asic_config_checksum.py b/files/build_scripts/generate_asic_config_checksum.py index 1de0930d538..ca83f4f26a6 100644 --- a/files/build_scripts/generate_asic_config_checksum.py +++ b/files/build_scripts/generate_asic_config_checksum.py @@ -30,8 +30,8 @@ def get_config_files(config_file_map): ''' config_files = [] for path, files in config_file_map.items(): - for file in files: - config_files.append(os.path.join(path, file)) + for config_file in files: + config_files.append(os.path.join(path, config_file)) return config_files def generate_checksum(checksum_files): @@ -43,13 +43,13 @@ def generate_checksum(checksum_files): NOT do any re-ordering of the files before creating the checksum. ''' checksum = hashlib.sha1() - for file in checksum_files: + for checksum_file in checksum_files: try: - with open(file, 'r') as f: + with open(checksum_file, 'r') as f: for chunk in iter(lambda: f.read(CHUNK_SIZE), b""): checksum.update(chunk) except IOError as e: - log_error('Error processing ASIC config file ' + file + ':' + e.strerror) + log_error('Error processing ASIC config file ' + checksum_file + ':' + e.strerror) return None return checksum.hexdigest() From badf2fe36963789e6d458339cef62f6267e7bd7f Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Tue, 3 Sep 2019 14:47:32 -0700 Subject: [PATCH 7/8] Advance submodule --- src/sonic-utilities | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-utilities b/src/sonic-utilities index cb0e745d6ad..4024019594b 160000 --- a/src/sonic-utilities +++ b/src/sonic-utilities @@ -1 +1 @@ -Subproject commit cb0e745d6ad105f5bc486ec77955f06a7c25912e +Subproject commit 4024019594bc9174374ed3a3b594d8a43299dd07 From e72e52db90ef2a405f0e366f0a386d61338482cf Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Tue, 3 Sep 2019 15:01:02 -0700 Subject: [PATCH 8/8] Revert "Advance submodule" This reverts commit badf2fe36963789e6d458339cef62f6267e7bd7f. --- src/sonic-utilities | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-utilities b/src/sonic-utilities index 4024019594b..cb0e745d6ad 160000 --- a/src/sonic-utilities +++ b/src/sonic-utilities @@ -1 +1 @@ -Subproject commit 4024019594bc9174374ed3a3b594d8a43299dd07 +Subproject commit cb0e745d6ad105f5bc486ec77955f06a7c25912e