From 84e9b92d7170d493da062e9b4d53e51bae8ece0c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 25 May 2020 23:44:43 +0100 Subject: [PATCH] Development: Allow for debian packages in stock converter. --- development/debian-kernels.txt | 93 +++++++++++++++++++ .../{rpm-profiles.py => stock-linux-json.py} | 66 +++++++++---- 2 files changed, 143 insertions(+), 16 deletions(-) create mode 100644 development/debian-kernels.txt rename development/{rpm-profiles.py => stock-linux-json.py} (53%) diff --git a/development/debian-kernels.txt b/development/debian-kernels.txt new file mode 100644 index 000000000..8b5c3dfbc --- /dev/null +++ b/development/debian-kernels.txt @@ -0,0 +1,93 @@ +https://www.mirrorservice.org/sites/ftp.debian.org/debian/pool/main/l/linux/linux-image-3.16.0-6-amd64-dbg_3.16.56-1+deb8u1_amd64.deb +https://www.mirrorservice.org/sites/ftp.debian.org/debian/pool/main/l/linux/linux-image-3.16.0-6-amd64_3.16.56-1+deb8u1_amd64.deb +https://www.mirrorservice.org/sites/ftp.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-12-amd64-dbg_4.9.210-1_amd64.deb +https://www.mirrorservice.org/sites/ftp.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-12-amd64_4.9.210-1_amd64.deb +https://www.mirrorservice.org/sites/ftp.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-12-rt-amd64_4.9.210-1_amd64.deb +https://www.mirrorservice.org/sites/ftp.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-12-rt-amd64-dbg_4.9.210-1_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-amd64_2.6.32-48squeeze6_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-amd64-dbg_2.6.32-48squeeze6_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-amd64_2.6.32-48squeeze20_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-amd64-dbg_2.6.32-48squeeze20_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-xen-amd64_2.6.32-48squeeze6_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-xen-amd64-dbg_2.6.32-48squeeze6_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-xen-amd64_2.6.32-48squeeze20_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-xen-amd64-dbg_2.6.32-48squeeze20_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.2.0-4-amd64_3.2.78-1_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.2.0-4-amd64-dbg_3.2.78-1_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.2.0-4-rt-amd64_3.2.78-1_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.2.0-4-rt-amd64-dbg_3.2.78-1_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.16.0-0.bpo.4-amd64_3.16.39-1+deb8u1~bpo70+1_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.16.0-0.bpo.4-amd64-dbg_3.16.39-1+deb8u1~bpo70+1_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.16.0-6-amd64_3.16.56-1+deb8u1_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.16.0-6-amd64-dbg_3.16.56-1+deb8u1_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-0.bpo.6-amd64_4.9.88-1+deb9u1~bpo8+1_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-0.bpo.6-amd64-dbg_4.9.88-1+deb9u1~bpo8+1_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-0.bpo.6-rt-amd64_4.9.88-1+deb9u1~bpo8+1_amd64.deb +http://archive.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-0.bpo.6-rt-amd64-dbg_4.9.88-1+deb9u1~bpo8+1_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-3.16.0-10-amd64_3.16.81-1_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-3.16.0-10-amd64-dbg_3.16.81-1_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-amd64_4.9.189-3+deb9u2_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-amd64-dbg_4.9.189-3+deb9u2_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-rt-amd64_4.9.189-3+deb9u2_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-rt-amd64-dbg_4.9.189-3+deb9u2_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-0.bpo.6-amd64-dbg_4.19.67-2%2Bdeb10u2~bpo9%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-0.bpo.6-amd64_4.19.67-2%2Bdeb10u2~bpo9%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-0.bpo.6-cloud-amd64-dbg_4.19.67-2%2Bdeb10u2~bpo9%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-0.bpo.6-cloud-amd64_4.19.67-2%2Bdeb10u2~bpo9%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-0.bpo.6-rt-amd64-dbg_4.19.67-2%2Bdeb10u2~bpo9%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-0.bpo.6-rt-amd64_4.19.67-2%2Bdeb10u2~bpo9%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-0.bpo.8-amd64-dbg_4.19.98-1~bpo9%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-0.bpo.8-amd64_4.19.98-1~bpo9%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-0.bpo.8-cloud-amd64-dbg_4.19.98-1~bpo9%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-0.bpo.8-cloud-amd64_4.19.98-1~bpo9%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-0.bpo.8-rt-amd64-dbg_4.19.98-1~bpo9%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-0.bpo.8-rt-amd64_4.19.98-1~bpo9%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-9-amd64-dbg_4.19.118-2_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-9-amd64_4.19.118-2_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-9-cloud-amd64-dbg_4.19.118-2_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-9-cloud-amd64_4.19.118-2_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-9-rt-amd64-dbg_4.19.118-2_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-9-rt-amd64_4.19.118-2_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.2-amd64_5.4.8-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.2-amd64-dbg_5.4.8-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.2-cloud-amd64_5.4.8-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.2-cloud-amd64-dbg_5.4.8-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.2-rt-amd64_5.4.8-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.2-rt-amd64-dbg_5.4.8-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.3-amd64_5.4.13-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.3-amd64-dbg_5.4.13-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.3-cloud-amd64_5.4.13-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.3-cloud-amd64-dbg_5.4.13-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.3-rt-amd64_5.4.13-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.3-rt-amd64-dbg_5.4.13-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.4-amd64_5.4.19-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.4-amd64-dbg_5.4.19-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.4-cloud-amd64_5.4.19-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.4-cloud-amd64-dbg_5.4.19-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.4-rt-amd64-dbg_5.4.19-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.4-rt-amd64_5.4.19-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.5.0-0.bpo.2-amd64-dbg_5.5.17-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.5.0-0.bpo.2-amd64_5.5.17-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.5.0-0.bpo.2-cloud-amd64-dbg_5.5.17-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.5.0-0.bpo.2-cloud-amd64_5.5.17-1~bpo10%2B1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.6.0-1-amd64-dbg_5.6.7-1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.6.0-1-amd64_5.6.7-1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.6.0-1-cloud-amd64-dbg_5.6.7-1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.6.0-1-cloud-amd64_5.6.7-1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.6.0-1-rt-amd64-dbg_5.6.7-1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.6.0-1-rt-amd64_5.6.7-1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.7.0-rc5-amd64-dbg_5.7~rc5-1~exp1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.7.0-rc5-amd64_5.7~rc5-1~exp1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.7.0-rc5-cloud-amd64-dbg_5.7~rc5-1~exp1_amd64.deb +http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.7.0-rc5-cloud-amd64_5.7~rc5-1~exp1_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux-latest/linux-image-amd64-dbg_3.16+63+deb8u6_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux-latest/linux-image-amd64_3.16+63+deb8u6_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux-latest-4.9/linux-image-4.9-amd64-dbg_4.9+80+deb9u10~deb8u1_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux-latest-4.9/linux-image-4.9-amd64_4.9+80+deb9u10~deb8u1_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-rt-amd64-dbg_4.9.189-3+deb9u2_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-rt-amd64_4.9.189-3+deb9u2_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-amd64-dbg_4.9.189-3+deb9u2_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-amd64_4.9.189-3+deb9u2_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-3.16.0-10-amd64-dbg_3.16.81-1_amd64.deb +http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-3.16.0-10-amd64_3.16.81-1_amd64.deb + diff --git a/development/rpm-profiles.py b/development/stock-linux-json.py similarity index 53% rename from development/rpm-profiles.py rename to development/stock-linux-json.py index c7f18a8bd..f3d94d4c5 100644 --- a/development/rpm-profiles.py +++ b/development/stock-linux-json.py @@ -3,10 +3,11 @@ import lzma import os import subprocess import tempfile -from typing import List, Dict +from typing import List, Dict, Optional import requests import rpmfile +from debian import debfile DWARF2JSON = './dwarf2json' @@ -22,32 +23,65 @@ class Downloader: files_for_processing = self.download_list(url_list) self.process_files(files_for_processing) for fname in files_for_processing.values(): - os.unlink(fname) + if fname: + os.unlink(fname) def download_list(self, urls: List[str]) -> Dict[str, str]: processed_files = {} for url in urls: print(" - Downloading {}".format(url)) data = requests.get(url) - with tempfile.NamedTemporaryFile() as rpmdata: - rpmdata.write(data.content) - rpmdata.seek(0) - rpm = rpmfile.RPMFile(fileobj = rpmdata) - for member in rpm.getmembers(): - if 'vmlinux' in member.name or 'System.map' in member.name: - print(" - Extracting {}".format(member.name)) - extracted = rpm.extractfile(member) - break - with tempfile.NamedTemporaryFile( - delete = False, prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: - print(" - Writing to {}".format(output.name)) - output.write(extracted.read()) - processed_files[url] = output.name + with tempfile.NamedTemporaryFile() as archivedata: + archivedata.write(data.content) + archivedata.seek(0) + if url.endswith('.rpm'): + processed_files[url] = self.process_rpm(archivedata) + elif url.endswith('.deb'): + processed_files[url] = self.process_deb(archivedata) + return processed_files + def process_rpm(self, archivedata) -> Optional[str]: + rpm = rpmfile.RPMFile(fileobj = archivedata) + member = None + extracted = None + for member in rpm.getmembers(): + if 'vmlinux' in member.name or 'System.map' in member.name: + print(" - Extracting {}".format(member.name)) + extracted = rpm.extractfile(member) + break + if not member or not extracted: + return None + with tempfile.NamedTemporaryFile(delete = False, + prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: + print(" - Writing to {}".format(output.name)) + output.write(extracted.read()) + return output.name + + def process_deb(self, archivedata) -> Optional[str]: + deb = debfile.DebFile(fileobj = archivedata) + member = None + extracted = None + for member in deb.data.tgz().getmembers(): + if member.name.endswith('vmlinux') or 'System.map' in member.name: + print(" - Extracting {}".format(member.name)) + extracted = deb.data.get_file(member.name) + break + if not member or not extracted: + return None + with tempfile.NamedTemporaryFile(delete = False, + prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: + print(" - Writing to {}".format(output.name)) + output.write(extracted.read()) + return output.name + def process_files(self, named_files: Dict[str, str]): """Runs the dwarf2json binary across the files""" print("Processing Files...") + for i in named_files: + if named_files[i] is None: + print("FAILURE: None encountered for {}".format(i)) + return args = [DWARF2JSON, 'linux'] output_filename = 'unknown-kernel.json' for named_file in named_files: