WEBDIENSTE/osmedeus
1
0
Fork 0
mirror of https://github.com/j3ssie/osmedeus.git synced 2026-09-29 13:05:04 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
df0683b60759df6e4fdbe0fb4e44fe68ae18d8db
osmedeus/test/testdata/sample-jsonl-output/vigolium-spider.jsonl
T
j3ssie 8ec1de4f84 feat(db): add db_import_vigolium importer and harden CLI run 
- Add db_import_vigolium JS function that routes vigolium JSONL records
  by envelope type: http_record -> assets, finding -> vulnerabilities
  (deduped on new finding_hash column), skipping scan/oast_interaction
- Add FindingHash field to Vulnerability model with idempotent ALTER
  TABLE migration and matching index
- Suppress run errors when using --silent --empty-target placeholder mode
- Retry transient fetchURLContent failures (network errors, 408/429/5xx)
  with exponential backoff; leave 4xx untouched so GitHub auth fallback
  can engage
- Add unit tests for the vigolium importer (import + idempotency) and
  the HTTP retry behavior, plus a vigolium juice-shop sample fixture
- Bump katana, naabu, kingfisher, bearer registry entries and add
  vigolium to the direct-fetch registry
2026-05-29 23:31:56 +08:00

33 lines
14 KiB
JSON
Raw Blame History

{"type":"http_record","data":{"uuid":"e0000000-0000-0000-0000-000000000001","scheme":"https","hostname":"ginandjuice.shop","port":443,"ip":"34.246.169.176","method":"POST","path":"/catalog/product/stock","url":"https://ginandjuice.shop/catalog/product/stock","http_version":"HTTP/1.1","status_code":400,"response_content_type":"application/json","response_content_length":52,"response_words":4,"has_response":true,"source":"spidering","created_at":"2026-05-27T17:05:00Z"}}
{"type":"http_record","data":{"uuid":"ba5b7034-259b-41ca-a756-fb99db80dcf9","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/api/Addresss","url":"http://localhost:3000/api/Addresss","http_version":"HTTP/1.1","status_code":401,"response_content_type":"application/json; charset=utf-8","response_content_length":83,"response_words":38,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:07:24Z"}}
{"type":"http_record","data":{"uuid":"1e6a1af0-9e9e-4125-bbea-03a93a918a21","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/api/Challenges/?name=Score%20Board","url":"http://localhost:3000/api/Challenges/?name=Score%20Board","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":449,"response_words":42,"has_response":true,"source":"spidering","created_at":"2026-05-27T15:55:38Z"}}
{"type":"http_record","data":{"uuid":"6cf73016-5631-42f7-be7b-69a452613d7b","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/api/Feedbacks/","url":"http://localhost:3000/api/Feedbacks/","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":1734,"response_words":127,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:11:42Z"}}
{"type":"http_record","data":{"uuid":"4387497e-d55a-4d4b-a1bf-a68ceaddd93f","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/api/Quantitys/","url":"http://localhost:3000/api/Quantitys/","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":6258,"response_words":34,"has_response":true,"source":"spidering","created_at":"2026-05-27T15:55:38Z"}}
{"type":"http_record","data":{"uuid":"34f70e32-f7da-4a53-bb2a-265de5c51e27","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/api/Recycles/","url":"http://localhost:3000/api/Recycles/","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":76,"response_words":37,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:07:24Z"}}
{"type":"http_record","data":{"uuid":"0ec45854-0a4d-4dd6-982f-f091fc91389f","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/api/SecurityQuestions/","url":"http://localhost:3000/api/SecurityQuestions/","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":1934,"response_words":100,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:06:31Z"}}
{"type":"http_record","data":{"uuid":"2554b71f-b41d-433e-8b1c-e3e6f1d72b68","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/assets/i18n/en.json","url":"http://localhost:3000/assets/i18n/en.json","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=UTF-8","response_content_length":33168,"response_words":3442,"has_response":true,"source":"spidering","created_at":"2026-05-27T15:55:38Z"}}
{"type":"http_record","data":{"uuid":"4ae1618e-ee26-43a0-b6d9-a55aa9b2b140","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/confetti-5NW67UGY.js","url":"http://localhost:3000/confetti-5NW67UGY.js","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/javascript; charset=UTF-8","response_content_length":11194,"response_words":307,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:06:02Z"}}
{"type":"http_record","data":{"uuid":"f63a2a80-9f39-4ff6-b710-a90f8e1f5c3c","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/admin/application-configuration","url":"http://localhost:3000/rest/admin/application-configuration","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":21770,"response_words":1497,"has_response":true,"source":"spidering","created_at":"2026-05-27T15:55:38Z"}}
{"type":"http_record","data":{"uuid":"3d9a43c0-458d-4017-b453-88af54f54008","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/admin/application-version","url":"http://localhost:3000/rest/admin/application-version","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":20,"response_words":32,"has_response":true,"source":"spidering","created_at":"2026-05-27T15:55:38Z"}}
{"type":"http_record","data":{"uuid":"882190d5-3e30-450d-b261-e1a536340a08","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/continue-code","url":"http://localhost:3000/rest/continue-code","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":79,"response_words":32,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:06:02Z"}}
{"type":"http_record","data":{"uuid":"45d6fa53-efe0-4fd9-8109-0835d8654244","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/languages","url":"http://localhost:3000/rest/languages","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":4235,"response_words":38,"has_response":true,"source":"spidering","created_at":"2026-05-27T15:55:38Z"}}
{"type":"http_record","data":{"uuid":"08e28f5e-3aa3-4367-881a-91e15916d552","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/1/reviews","url":"http://localhost:3000/rest/products/1/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":172,"response_words":35,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:10:03Z"}}
{"type":"http_record","data":{"uuid":"514aefcd-e6eb-4b02-8d79-2509d2dd90f7","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/22/reviews","url":"http://localhost:3000/rest/products/22/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":177,"response_words":36,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:08:32Z"}}
{"type":"http_record","data":{"uuid":"80dc3e0a-640a-4b3e-8b10-f9b986d173aa","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/24/reviews","url":"http://localhost:3000/rest/products/24/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":30,"response_words":32,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:07:22Z"}}
{"type":"http_record","data":{"uuid":"f013010e-e250-4135-8c39-5a3b259c37eb","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/25/reviews","url":"http://localhost:3000/rest/products/25/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":30,"response_words":32,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:07:18Z"}}
{"type":"http_record","data":{"uuid":"b13f5e8f-5689-4f59-a55c-632b832297a8","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/30/reviews","url":"http://localhost:3000/rest/products/30/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":187,"response_words":37,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:06:37Z"}}
{"type":"http_record","data":{"uuid":"65ca8ec7-935d-4d2c-8fcc-2bf489df0238","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/33/reviews","url":"http://localhost:3000/rest/products/33/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":30,"response_words":32,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:07:41Z"}}
{"type":"http_record","data":{"uuid":"e6f7f47c-defa-46a6-969f-6a77f672168a","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/34/reviews","url":"http://localhost:3000/rest/products/34/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":30,"response_words":32,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:09:59Z"}}
{"type":"http_record","data":{"uuid":"12450a1a-9954-48e9-967c-3179e5f31fa2","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/38/reviews","url":"http://localhost:3000/rest/products/38/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":352,"response_words":42,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:07:45Z"}}
{"type":"http_record","data":{"uuid":"8e7e0f8a-e49b-4f65-a87f-5afc27606e91","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/41/reviews","url":"http://localhost:3000/rest/products/41/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":284,"response_words":59,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:11:29Z"}}
{"type":"http_record","data":{"uuid":"30e13460-fc84-44ed-aa36-4adde82ec0a2","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/42/reviews","url":"http://localhost:3000/rest/products/42/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":413,"response_words":58,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:09:37Z"}}
{"type":"http_record","data":{"uuid":"9adbb7c6-483e-4330-9af8-4402ec87bfc7","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/43/reviews","url":"http://localhost:3000/rest/products/43/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":296,"response_words":54,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:08:54Z"}}
{"type":"http_record","data":{"uuid":"d97a0244-ae2b-42f2-bcfa-ffa2838b85c4","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/5/reviews","url":"http://localhost:3000/rest/products/5/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":30,"response_words":32,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:09:26Z"}}
{"type":"http_record","data":{"uuid":"4308660e-6a61-4c58-adf7-3ea71e8834f9","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/6/reviews","url":"http://localhost:3000/rest/products/6/reviews","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":170,"response_words":35,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:10:54Z"}}
{"type":"http_record","data":{"uuid":"a0399c9c-3513-4a7e-9f61-2e1145505b8f","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/products/search?q=","url":"http://localhost:3000/rest/products/search?q","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":13644,"response_words":928,"has_response":true,"source":"spidering","created_at":"2026-05-27T15:55:38Z"}}
{"type":"http_record","data":{"uuid":"74c150c1-240c-432e-9ef7-6ee5625102ce","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/rest/user/whoami?fields=email","url":"http://localhost:3000/rest/user/whoami?fields=email","http_version":"HTTP/1.1","status_code":200,"response_content_type":"application/json; charset=utf-8","response_content_length":11,"response_words":32,"has_response":true,"source":"spidering","created_at":"2026-05-27T16:06:37Z"}}
{"type":"http_record","data":{"uuid":"858691bb-36ca-4f04-8ff9-5d0bf0f3905c","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"GET","path":"/socket.io/?EIO=4&transport=polling&t=Pvg9WWG","url":"http://localhost:3000/socket.io/?EIO=4&transport=polling&t=Pvg9WWG","http_version":"HTTP/1.1","status_code":200,"response_content_type":"text/plain; charset=UTF-8","response_content_length":96,"response_words":21,"has_response":true,"source":"spidering","created_at":"2026-05-27T15:55:38Z"}}
{"type":"http_record","data":{"uuid":"99fd82a8-ae63-400f-913f-768a2aab7ac9","scheme":"http","hostname":"localhost","port":3000,"ip":"::1","method":"POST","path":"/socket.io/?EIO=4&transport=polling&t=Pvg9WWv&sid=P7TDh2AkBCj3kPilAAJN","url":"http://localhost:3000/socket.io/?EIO=4&transport=polling&t=Pvg9WWv&sid=P7TDh2AkBCj3kPilAAJN","http_version":"HTTP/1.1","status_code":200,"response_content_type":"text/html","response_content_length":2,"response_words":20,"has_response":true,"source":"spidering","created_at":"2026-05-27T15:55:38Z"}}
{"type":"scan","data":{"uuid":"s0000000-0000-0000-0000-000000000002","status":"completed","target":"ginandjuice.shop","total_requests":30}}
{"type":"module","data":{"id":"spider","name":"Spider","module_type":"crawl"}}
Reference in New Issue View Git Blame Copy Permalink