Improve logging, tracking and docstrings. Add all OSint functions as RengineTasks

web/celery-entrypoint.sh

@@ -16,7 +16,7 @@ python3 manage.py migrate
 python3 manage.py collectstatic --no-input --clear
 
 # Load default engines
-python3 manage.py loaddata fixtures/default_scan_engines.yaml --app scanEngine.EngineType
+# python3 manage.py loaddata fixtures/default_scan_engines.yaml --app scanEngine.EngineType
 # Load default keywords
 python3 manage.py loaddata fixtures/default_keywords.yaml --app scanEngine.InterestingLookupModel
 # Load default external tools

web/reNgine/celery_custom_task.py

@@ -94,8 +94,9 @@ class RengineTask(Task):
 					return
 
 			# Create ScanActivity for this task and send start scan notifs
-			logger.warning(f'Task {self.task_name} is RUNNING')
-			self.create_scan_activity()
+			if self.track:
+				logger.warning(f'Task {self.task_name} is RUNNING')
+				self.create_scan_activity()
 
 		if RENGINE_CACHE_ENABLED:
 			# Check for result in cache and return it if it's a hit
@@ -103,7 +104,7 @@ class RengineTask(Task):
 			result = cache.get(record_key)
 			if result and result != b'null':
 				self.status = SUCCESS_TASK
-				if RENGINE_RECORD_ENABLED:
+				if RENGINE_RECORD_ENABLED and self.track:
 					logger.warning(f'Task {self.task_name} status is SUCCESS (CACHED)')
 					self.update_scan_activity()
 				return json.loads(result)
@@ -134,7 +135,7 @@ class RengineTask(Task):
 		finally:
 			self.write_results()
 
-			if RENGINE_RECORD_ENABLED:
+			if RENGINE_RECORD_ENABLED and self.track:
 				msg = f'Task {self.task_name} status is {self.status_str}'
 				msg += f' | Error: {self.error}' if self.error else ''
 				logger.warning(msg)

web/reNgine/settings.py

@@ -224,10 +224,10 @@ LOGGING = {
             'format': '%(message)s'
         },
         'brief': {
-            'format': '%(name)-25s | %(message)s'
+            'format': '%(name)-10s | %(message)s'
         },
         'task': {
-            '()': lambda : TaskFormatter('%(task_name)-25s | %(message)s')
+            '()': lambda : TaskFormatter('%(task_name)-34s | %(message)s')
         }
     },
     'loggers': {
@@ -240,6 +240,10 @@ LOGGING = {
             'handlers': ['null'],
             'propagate': False,
         },
+        'celery.task': {
+            'handlers': ['task'],
+            'propagate': False
+        },
         'celery.worker': {
             'handlers': ['null'],
             'propagate': False,
@@ -256,6 +260,6 @@ LOGGING = {
         'reNgine.tasks': {
             'handlers': ['task'],
             'propagate': False
-        },
+        }
     },
 }

web/test.py

@@ -0,0 +1,13 @@
+import os
+
+from celery.utils.log import get_task_logger
+
+logger = get_task_logger(__name__)
+
+if __name__ == '__main__':
+    with open('/tmp/secret.txt', 'w') as f:
+        f.write('testconfig')
+    os.environ['SECRET_FILE'] = '/tmp/secret.txt'
+    from reNgine import celery
+    from reNgine.tasks import subdomain_discovery
+    subdomain_discovery('jahmyst.synology.me', ctx={})