import argparse import lzma import os import subprocess import tempfile from typing import List, Dict, Optional import requests import rpmfile from debian import debfile DWARF2JSON = './dwarf2json' class Downloader: def __init__(self, url_lists: List[List[str]]) -> None: self.url_lists = url_lists def download_lists(self, keep = False): for url_list in self.url_lists: print("Downloading files...") files_for_processing = self.download_list(url_list) self.process_files(files_for_processing) if not keep: for fname in files_for_processing.values(): if fname: os.unlink(fname) def download_list(self, urls: List[str]) -> Dict[str, str]: processed_files = {} for url in urls: print(f" - Downloading {url}") data = requests.get(url) with tempfile.NamedTemporaryFile() as archivedata: archivedata.write(data.content) archivedata.seek(0) if url.endswith('.rpm'): processed_files[url] = self.process_rpm(archivedata) elif url.endswith('.deb'): processed_files[url] = self.process_deb(archivedata) return processed_files def process_rpm(self, archivedata) -> Optional[str]: rpm = rpmfile.RPMFile(fileobj = archivedata) member = None extracted = None for member in rpm.getmembers(): if 'vmlinux' in member.name or 'System.map' in member.name: print(f" - Extracting {member.name}") extracted = rpm.extractfile(member) break if not member or not extracted: return None with tempfile.NamedTemporaryFile(delete = False, prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: print(f" - Writing to {output.name}") output.write(extracted.read()) return output.name def process_deb(self, archivedata) -> Optional[str]: deb = debfile.DebFile(fileobj = archivedata) member = None extracted = None for member in deb.data.tgz().getmembers(): if member.name.endswith('vmlinux') or 'System.map' in member.name: print(f" - Extracting {member.name}") extracted = deb.data.get_file(member.name) break if not member or not extracted: return None with tempfile.NamedTemporaryFile(delete = False, prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: print(f" - Writing to {output.name}") output.write(extracted.read()) return output.name def process_files(self, named_files: Dict[str, str]): """Runs the dwarf2json binary across the files""" print("Processing Files...") for i in named_files: if named_files[i] is None: print(f"FAILURE: None encountered for {i}") return args = [DWARF2JSON, 'linux'] output_filename = 'unknown-kernel.json' for named_file in named_files: prefix = '--system-map' if 'System' not in named_files[named_file]: prefix = '--elf' output_filename = './' + '-'.join((named_file.split('/')[-1]).split('-')[2:])[:-4] + '.json.xz' args += [prefix, named_files[named_file]] print(f" - Running {args}") proc = subprocess.run(args, capture_output = True) print(f" - Writing to {output_filename}") with lzma.open(output_filename, 'w') as f: f.write(proc.stdout) if __name__ == '__main__': parser = argparse.ArgumentParser(description = "Takes a list of URLs for Centos and downloads them") parser.add_argument("-f", "--file", dest = 'filename', metavar = "FILENAME", help = "Filename to be read", required = True) parser.add_argument("-d", "--dwarf2json", dest = 'dwarfpath', metavar = "PATH", default = DWARF2JSON, help = "Path to the dwarf2json binary", required = True) parser.add_argument("-k", "--keep", dest = 'keep', action = 'store_true', help = 'Keep extracted temporary files after completion', default = False) args = parser.parse_args() DWARF2JSON = args.dwarfpath with open(args.filename) as f: lines = f.readlines() urls = [] for i in range(len(lines) // 2): urls += [[lines[2 * i].strip(), lines[(2 * i) + 1].strip()]] d = Downloader(urls) d.download_lists(keep = args.keep)