|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# Copyright (c) 2020 The Bitcoin Core developers |
| 3 | +# Distributed under the MIT software license, see the accompanying |
| 4 | +# file COPYING or http://www.opensource.org/licenses/mit-license.php. |
| 5 | +"""Parse message capture binary files. |
| 6 | +To be used in conjunction with -capturemessages. |
| 7 | +""" |
| 8 | + |
| 9 | +import argparse |
| 10 | +import os |
| 11 | +import shutil |
| 12 | +import sys |
| 13 | +from io import BytesIO |
| 14 | +import json |
| 15 | +from pathlib import Path |
| 16 | +from typing import Any, Dict, List, Optional |
| 17 | + |
| 18 | +sys.path.append( |
| 19 | + os.path.join( |
| 20 | + os.path.dirname(__file__), |
| 21 | + '../../test/functional')) |
| 22 | + |
| 23 | +from test_framework.messages import ser_uint256 # noqa: E402 |
| 24 | +from test_framework.p2p import MESSAGEMAP # noqa: E402 |
| 25 | + |
| 26 | +TIME_SIZE = 8 |
| 27 | +LENGTH_SIZE = 4 |
| 28 | +MSGTYPE_SIZE = 12 |
| 29 | + |
| 30 | +# The test framework classes stores hashes as large ints in many cases. |
| 31 | +# These are variables of type uint256 in core. |
| 32 | +# There isn't a way to distinguish between a large int and a large int that is actually a blob of bytes. |
| 33 | +# As such, they are itemized here. |
| 34 | +# Any variables with these names that are of type int are actually uint256 variables. |
| 35 | +# (These can be easily found by looking for calls to deser_uint256, deser_uint256_vector, and uint256_from_str in messages.py) |
| 36 | +HASH_INTS = [ |
| 37 | + "blockhash", |
| 38 | + "block_hash", |
| 39 | + "hash", |
| 40 | + "hashMerkleRoot", |
| 41 | + "hashPrevBlock", |
| 42 | + "hashstop", |
| 43 | + "limited_proofid", |
| 44 | + "prev_header", |
| 45 | + "sha256", |
| 46 | + "stop_hash", |
| 47 | +] |
| 48 | + |
| 49 | +HASH_INT_VECTORS = [ |
| 50 | + "hashes", |
| 51 | + "headers", |
| 52 | + "vHave", |
| 53 | + "vHash", |
| 54 | +] |
| 55 | + |
| 56 | + |
| 57 | +class ProgressBar: |
| 58 | + def __init__(self, total: float): |
| 59 | + self.total = total |
| 60 | + self.running = 0 |
| 61 | + |
| 62 | + def set_progress(self, progress: float): |
| 63 | + cols = shutil.get_terminal_size()[0] |
| 64 | + if cols <= 12: |
| 65 | + return |
| 66 | + max_blocks = cols - 9 |
| 67 | + num_blocks = int(max_blocks * progress) |
| 68 | + print('\r[ {}{} ] {:3.0f}%' |
| 69 | + .format('#' * num_blocks, |
| 70 | + ' ' * (max_blocks - num_blocks), |
| 71 | + progress * 100), |
| 72 | + end='') |
| 73 | + |
| 74 | + def update(self, more: float): |
| 75 | + self.running += more |
| 76 | + self.set_progress(self.running / self.total) |
| 77 | + |
| 78 | + |
| 79 | +def to_jsonable(obj: Any) -> Any: |
| 80 | + if hasattr(obj, "__dict__"): |
| 81 | + return obj.__dict__ |
| 82 | + elif hasattr(obj, "__slots__"): |
| 83 | + ret: Dict[str, Any] = {} |
| 84 | + for slot in obj.__slots__: |
| 85 | + val = getattr(obj, slot, None) |
| 86 | + if slot in HASH_INTS and isinstance(val, int): |
| 87 | + ret[slot] = ser_uint256(val).hex() |
| 88 | + elif slot in HASH_INT_VECTORS and isinstance(val[0], int): |
| 89 | + ret[slot] = [ser_uint256(a).hex() for a in val] |
| 90 | + else: |
| 91 | + ret[slot] = to_jsonable(val) |
| 92 | + return ret |
| 93 | + elif isinstance(obj, list): |
| 94 | + return [to_jsonable(a) for a in obj] |
| 95 | + elif isinstance(obj, bytes): |
| 96 | + return obj.hex() |
| 97 | + else: |
| 98 | + return obj |
| 99 | + |
| 100 | + |
| 101 | +def process_file(path: str, messages: List[Any], recv: bool, |
| 102 | + progress_bar: Optional[ProgressBar]) -> None: |
| 103 | + with open(path, 'rb') as f_in: |
| 104 | + if progress_bar: |
| 105 | + bytes_read = 0 |
| 106 | + |
| 107 | + while True: |
| 108 | + if progress_bar: |
| 109 | + # Update progress bar |
| 110 | + diff = f_in.tell() - bytes_read - 1 |
| 111 | + progress_bar.update(diff) |
| 112 | + bytes_read = f_in.tell() - 1 |
| 113 | + |
| 114 | + # Read the Header |
| 115 | + tmp_header_raw = f_in.read(TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) |
| 116 | + if not tmp_header_raw: |
| 117 | + break |
| 118 | + tmp_header = BytesIO(tmp_header_raw) |
| 119 | + time = int.from_bytes(tmp_header.read(TIME_SIZE), "little") |
| 120 | + msgtype: bytes = tmp_header.read(MSGTYPE_SIZE).split(b'\x00', 1)[0] |
| 121 | + length = int.from_bytes(tmp_header.read(LENGTH_SIZE), "little") |
| 122 | + |
| 123 | + # Start converting the message to a dictionary |
| 124 | + msg_dict = {} |
| 125 | + msg_dict["direction"] = "recv" if recv else "sent" |
| 126 | + msg_dict["time"] = time |
| 127 | + # "size" is less readable here, but more readable in the output |
| 128 | + msg_dict["size"] = length |
| 129 | + |
| 130 | + msg_ser = BytesIO(f_in.read(length)) |
| 131 | + |
| 132 | + # Determine message type |
| 133 | + if msgtype not in MESSAGEMAP: |
| 134 | + # Unrecognized message type |
| 135 | + try: |
| 136 | + msgtype_tmp = msgtype.decode() |
| 137 | + if not msgtype_tmp.isprintable(): |
| 138 | + raise UnicodeDecodeError |
| 139 | + msg_dict["msgtype"] = msgtype_tmp |
| 140 | + except UnicodeDecodeError: |
| 141 | + msg_dict["msgtype"] = "UNREADABLE" |
| 142 | + msg_dict["body"] = msg_ser.read().hex() |
| 143 | + msg_dict["error"] = "Unrecognized message type." |
| 144 | + messages.append(msg_dict) |
| 145 | + print( |
| 146 | + f"WARNING - Unrecognized message type {msgtype} in {path}", |
| 147 | + file=sys.stderr) |
| 148 | + continue |
| 149 | + |
| 150 | + # Deserialize the message |
| 151 | + msg = MESSAGEMAP[msgtype]() |
| 152 | + msg_dict["msgtype"] = msgtype.decode() |
| 153 | + |
| 154 | + try: |
| 155 | + msg.deserialize(msg_ser) |
| 156 | + except KeyboardInterrupt: |
| 157 | + raise |
| 158 | + except Exception: |
| 159 | + # Unable to deserialize message body |
| 160 | + msg_ser.seek(0, os.SEEK_SET) |
| 161 | + msg_dict["body"] = msg_ser.read().hex() |
| 162 | + msg_dict["error"] = "Unable to deserialize message." |
| 163 | + messages.append(msg_dict) |
| 164 | + print( |
| 165 | + f"WARNING - Unable to deserialize message in {path}", |
| 166 | + file=sys.stderr) |
| 167 | + continue |
| 168 | + |
| 169 | + # Convert body of message into a jsonable object |
| 170 | + if length: |
| 171 | + msg_dict["body"] = to_jsonable(msg) |
| 172 | + messages.append(msg_dict) |
| 173 | + |
| 174 | + if progress_bar: |
| 175 | + # Update the progress bar to the end of the current file |
| 176 | + # in case we exited the loop early |
| 177 | + # Go to end of file |
| 178 | + f_in.seek(0, os.SEEK_END) |
| 179 | + diff = f_in.tell() - bytes_read - 1 |
| 180 | + progress_bar.update(diff) |
| 181 | + |
| 182 | + |
| 183 | +def main(): |
| 184 | + parser = argparse.ArgumentParser( |
| 185 | + description=__doc__, |
| 186 | + epilog="EXAMPLE \n\t{0} -o out.json <data-dir>/message_capture/**/*.dat".format( |
| 187 | + sys.argv[0]), |
| 188 | + formatter_class=argparse.RawTextHelpFormatter) |
| 189 | + parser.add_argument( |
| 190 | + "capturepaths", |
| 191 | + nargs='+', |
| 192 | + help="binary message capture files to parse.") |
| 193 | + parser.add_argument( |
| 194 | + "-o", "--output", |
| 195 | + help="output file. If unset print to stdout") |
| 196 | + parser.add_argument( |
| 197 | + "-n", "--no-progress-bar", |
| 198 | + action='store_true', |
| 199 | + help="disable the progress bar. Automatically set if the output is not a terminal") |
| 200 | + args = parser.parse_args() |
| 201 | + capturepaths = [Path.cwd() / Path(capturepath) |
| 202 | + for capturepath in args.capturepaths] |
| 203 | + output = Path.cwd() / Path(args.output) if args.output else False |
| 204 | + use_progress_bar = (not args.no_progress_bar) and sys.stdout.isatty() |
| 205 | + |
| 206 | + messages: List[Any] = [] |
| 207 | + if use_progress_bar: |
| 208 | + total_size = sum(capture.stat().st_size for capture in capturepaths) |
| 209 | + progress_bar = ProgressBar(total_size) |
| 210 | + else: |
| 211 | + progress_bar = None |
| 212 | + |
| 213 | + for capture in capturepaths: |
| 214 | + process_file( |
| 215 | + str(capture), |
| 216 | + messages, |
| 217 | + "recv" in capture.stem, |
| 218 | + progress_bar) |
| 219 | + |
| 220 | + messages.sort(key=lambda msg: msg['time']) |
| 221 | + |
| 222 | + if use_progress_bar: |
| 223 | + progress_bar.set_progress(1) |
| 224 | + jsonrep = json.dumps(messages) |
| 225 | + if output: |
| 226 | + with open(str(output), 'w+', encoding="utf8") as f_out: |
| 227 | + f_out.write(jsonrep) |
| 228 | + else: |
| 229 | + print(jsonrep) |
| 230 | + |
| 231 | + |
| 232 | +if __name__ == "__main__": |
| 233 | + main() |
0 commit comments