Development: Allow for debian packages in stock converter.

This commit is contained in:
Mike Auty
2020-05-25 23:44:43 +01:00
parent 27660d62ee
commit 84e9b92d71
2 changed files with 143 additions and 16 deletions
+93
View File
@@ -0,0 +1,93 @@
https://www.mirrorservice.org/sites/ftp.debian.org/debian/pool/main/l/linux/linux-image-3.16.0-6-amd64-dbg_3.16.56-1+deb8u1_amd64.deb
https://www.mirrorservice.org/sites/ftp.debian.org/debian/pool/main/l/linux/linux-image-3.16.0-6-amd64_3.16.56-1+deb8u1_amd64.deb
https://www.mirrorservice.org/sites/ftp.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-12-amd64-dbg_4.9.210-1_amd64.deb
https://www.mirrorservice.org/sites/ftp.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-12-amd64_4.9.210-1_amd64.deb
https://www.mirrorservice.org/sites/ftp.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-12-rt-amd64_4.9.210-1_amd64.deb
https://www.mirrorservice.org/sites/ftp.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-12-rt-amd64-dbg_4.9.210-1_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-amd64_2.6.32-48squeeze6_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-amd64-dbg_2.6.32-48squeeze6_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-amd64_2.6.32-48squeeze20_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-amd64-dbg_2.6.32-48squeeze20_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-xen-amd64_2.6.32-48squeeze6_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-xen-amd64-dbg_2.6.32-48squeeze6_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-xen-amd64_2.6.32-48squeeze20_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux-2.6/linux-image-2.6.32-5-xen-amd64-dbg_2.6.32-48squeeze20_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.2.0-4-amd64_3.2.78-1_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.2.0-4-amd64-dbg_3.2.78-1_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.2.0-4-rt-amd64_3.2.78-1_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.2.0-4-rt-amd64-dbg_3.2.78-1_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.16.0-0.bpo.4-amd64_3.16.39-1+deb8u1~bpo70+1_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.16.0-0.bpo.4-amd64-dbg_3.16.39-1+deb8u1~bpo70+1_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.16.0-6-amd64_3.16.56-1+deb8u1_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux/linux-image-3.16.0-6-amd64-dbg_3.16.56-1+deb8u1_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-0.bpo.6-amd64_4.9.88-1+deb9u1~bpo8+1_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-0.bpo.6-amd64-dbg_4.9.88-1+deb9u1~bpo8+1_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-0.bpo.6-rt-amd64_4.9.88-1+deb9u1~bpo8+1_amd64.deb
http://archive.debian.org/debian/pool/main/l/linux/linux-image-4.9.0-0.bpo.6-rt-amd64-dbg_4.9.88-1+deb9u1~bpo8+1_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-3.16.0-10-amd64_3.16.81-1_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-3.16.0-10-amd64-dbg_3.16.81-1_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-amd64_4.9.189-3+deb9u2_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-amd64-dbg_4.9.189-3+deb9u2_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-rt-amd64_4.9.189-3+deb9u2_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-rt-amd64-dbg_4.9.189-3+deb9u2_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-0.bpo.6-amd64-dbg_4.19.67-2%2Bdeb10u2~bpo9%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-0.bpo.6-amd64_4.19.67-2%2Bdeb10u2~bpo9%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-0.bpo.6-cloud-amd64-dbg_4.19.67-2%2Bdeb10u2~bpo9%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-0.bpo.6-cloud-amd64_4.19.67-2%2Bdeb10u2~bpo9%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-0.bpo.6-rt-amd64-dbg_4.19.67-2%2Bdeb10u2~bpo9%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-0.bpo.6-rt-amd64_4.19.67-2%2Bdeb10u2~bpo9%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-0.bpo.8-amd64-dbg_4.19.98-1~bpo9%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-0.bpo.8-amd64_4.19.98-1~bpo9%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-0.bpo.8-cloud-amd64-dbg_4.19.98-1~bpo9%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-0.bpo.8-cloud-amd64_4.19.98-1~bpo9%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-0.bpo.8-rt-amd64-dbg_4.19.98-1~bpo9%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-0.bpo.8-rt-amd64_4.19.98-1~bpo9%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-9-amd64-dbg_4.19.118-2_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-9-amd64_4.19.118-2_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-9-cloud-amd64-dbg_4.19.118-2_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-9-cloud-amd64_4.19.118-2_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-4.19.0-9-rt-amd64-dbg_4.19.118-2_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-4.19.0-9-rt-amd64_4.19.118-2_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.2-amd64_5.4.8-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.2-amd64-dbg_5.4.8-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.2-cloud-amd64_5.4.8-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.2-cloud-amd64-dbg_5.4.8-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.2-rt-amd64_5.4.8-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.2-rt-amd64-dbg_5.4.8-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.3-amd64_5.4.13-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.3-amd64-dbg_5.4.13-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.3-cloud-amd64_5.4.13-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.3-cloud-amd64-dbg_5.4.13-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.3-rt-amd64_5.4.13-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.3-rt-amd64-dbg_5.4.13-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.4-amd64_5.4.19-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.4-amd64-dbg_5.4.19-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.4-cloud-amd64_5.4.19-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.4-cloud-amd64-dbg_5.4.19-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.4.0-0.bpo.4-rt-amd64-dbg_5.4.19-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.4.0-0.bpo.4-rt-amd64_5.4.19-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.5.0-0.bpo.2-amd64-dbg_5.5.17-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.5.0-0.bpo.2-amd64_5.5.17-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.5.0-0.bpo.2-cloud-amd64-dbg_5.5.17-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.5.0-0.bpo.2-cloud-amd64_5.5.17-1~bpo10%2B1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.6.0-1-amd64-dbg_5.6.7-1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.6.0-1-amd64_5.6.7-1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.6.0-1-cloud-amd64-dbg_5.6.7-1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.6.0-1-cloud-amd64_5.6.7-1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.6.0-1-rt-amd64-dbg_5.6.7-1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.6.0-1-rt-amd64_5.6.7-1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.7.0-rc5-amd64-dbg_5.7~rc5-1~exp1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.7.0-rc5-amd64_5.7~rc5-1~exp1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux/linux-image-5.7.0-rc5-cloud-amd64-dbg_5.7~rc5-1~exp1_amd64.deb
http://ftp.us.debian.org/debian/pool/main/l/linux-signed-amd64/linux-image-5.7.0-rc5-cloud-amd64_5.7~rc5-1~exp1_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux-latest/linux-image-amd64-dbg_3.16+63+deb8u6_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux-latest/linux-image-amd64_3.16+63+deb8u6_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux-latest-4.9/linux-image-4.9-amd64-dbg_4.9+80+deb9u10~deb8u1_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux-latest-4.9/linux-image-4.9-amd64_4.9+80+deb9u10~deb8u1_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-rt-amd64-dbg_4.9.189-3+deb9u2_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-rt-amd64_4.9.189-3+deb9u2_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-amd64-dbg_4.9.189-3+deb9u2_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-4.9.0-11-amd64_4.9.189-3+deb9u2_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-3.16.0-10-amd64-dbg_3.16.81-1_amd64.deb
http://security.debian.org/debian-security/pool/updates/main/l/linux/linux-image-3.16.0-10-amd64_3.16.81-1_amd64.deb
@@ -3,10 +3,11 @@ import lzma
import os
import subprocess
import tempfile
from typing import List, Dict
from typing import List, Dict, Optional
import requests
import rpmfile
from debian import debfile
DWARF2JSON = './dwarf2json'
@@ -22,32 +23,65 @@ class Downloader:
files_for_processing = self.download_list(url_list)
self.process_files(files_for_processing)
for fname in files_for_processing.values():
os.unlink(fname)
if fname:
os.unlink(fname)
def download_list(self, urls: List[str]) -> Dict[str, str]:
processed_files = {}
for url in urls:
print(" - Downloading {}".format(url))
data = requests.get(url)
with tempfile.NamedTemporaryFile() as rpmdata:
rpmdata.write(data.content)
rpmdata.seek(0)
rpm = rpmfile.RPMFile(fileobj = rpmdata)
for member in rpm.getmembers():
if 'vmlinux' in member.name or 'System.map' in member.name:
print(" - Extracting {}".format(member.name))
extracted = rpm.extractfile(member)
break
with tempfile.NamedTemporaryFile(
delete = False, prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output:
print(" - Writing to {}".format(output.name))
output.write(extracted.read())
processed_files[url] = output.name
with tempfile.NamedTemporaryFile() as archivedata:
archivedata.write(data.content)
archivedata.seek(0)
if url.endswith('.rpm'):
processed_files[url] = self.process_rpm(archivedata)
elif url.endswith('.deb'):
processed_files[url] = self.process_deb(archivedata)
return processed_files
def process_rpm(self, archivedata) -> Optional[str]:
rpm = rpmfile.RPMFile(fileobj = archivedata)
member = None
extracted = None
for member in rpm.getmembers():
if 'vmlinux' in member.name or 'System.map' in member.name:
print(" - Extracting {}".format(member.name))
extracted = rpm.extractfile(member)
break
if not member or not extracted:
return None
with tempfile.NamedTemporaryFile(delete = False,
prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output:
print(" - Writing to {}".format(output.name))
output.write(extracted.read())
return output.name
def process_deb(self, archivedata) -> Optional[str]:
deb = debfile.DebFile(fileobj = archivedata)
member = None
extracted = None
for member in deb.data.tgz().getmembers():
if member.name.endswith('vmlinux') or 'System.map' in member.name:
print(" - Extracting {}".format(member.name))
extracted = deb.data.get_file(member.name)
break
if not member or not extracted:
return None
with tempfile.NamedTemporaryFile(delete = False,
prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output:
print(" - Writing to {}".format(output.name))
output.write(extracted.read())
return output.name
def process_files(self, named_files: Dict[str, str]):
"""Runs the dwarf2json binary across the files"""
print("Processing Files...")
for i in named_files:
if named_files[i] is None:
print("FAILURE: None encountered for {}".format(i))
return
args = [DWARF2JSON, 'linux']
output_filename = 'unknown-kernel.json'
for named_file in named_files: