diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3359cfb8..29f8a3c8 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -58,6 +58,7 @@ services: command: celery -A reNgine beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler depends_on: - celery + - db environment: - DEBUG=1 - CELERY_BROKER=redis://redis:6379/0 diff --git a/docker-compose.yml b/docker-compose.yml index 6f40c5bd..141bf724 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,6 +68,7 @@ services: - POSTGRES_HOST=${POSTGRES_HOST} depends_on: - celery + - db volumes: - ./web:/usr/src/app - github_repos:/usr/src/github diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index 761738b5..95233045 100755 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -38,19 +38,19 @@ fi print_status "${INFO}Proceeding with uninstalling reNgine${RESET}" print_status "Stopping all containers related to reNgine..." -docker stop $(docker ps -a -q --filter name=rengine-) 2>/dev/null +docker stop $(docker ps -a -q --filter name=rengine) 2>/dev/null print_status "Removing all containers related to reNgine..." -docker rm $(docker ps -a -q --filter name=rengine-) 2>/dev/null +docker rm $(docker ps -a -q --filter name=rengine) 2>/dev/null print_status "Removing all volumes related to reNgine..." -docker volume rm $(docker volume ls -q --filter name=rengine-) 2>/dev/null +docker volume rm $(docker volume ls -q --filter name=rengine) 2>/dev/null print_status "Removing all networks related to reNgine..." -docker network rm $(docker network ls -q --filter name=rengine-) 2>/dev/null +docker network rm $(docker network ls -q --filter name=rengine) 2>/dev/null print_status "Removing all images related to reNgine..." -docker rmi $(docker images -q --filter reference=rengine-) 2>/dev/null +docker rmi $(docker images -q --filter reference=rengine) 2>/dev/null print_status "Performing final cleanup" docker system prune -f --volumes --filter "label=com.docker.compose.project=rengine" diff --git a/web/celery-entrypoint.sh b/web/celery-entrypoint.sh index 54e014cc..4a6d228d 100755 --- a/web/celery-entrypoint.sh +++ b/web/celery-entrypoint.sh @@ -1,7 +1,40 @@ #!/bin/bash -python3 manage.py makemigrations +# apply existing migrations python3 manage.py migrate + +# make migrations for specific apps +apps=( + "targetApp" + "scanEngine" + "startScan" + "dashboard" + "recon_note" +) + +create_migrations() { + local app=$1 + echo "Creating migrations for $app..." + python3 manage.py makemigrations $app + echo "Finished creating migrations for $app" + echo "----------------------------------------" +} + +echo "Starting migration creation process..." + +for app in "${apps[@]}" +do + create_migrations $app +done + +echo "Migration creation process completed." + +# apply migrations again +echo "Applying migrations..." +python3 manage.py migrate +echo "Migration process completed." + + python3 manage.py collectstatic --no-input --clear # Load default engines, keywords, and external tools @@ -172,7 +205,7 @@ generate_worker_command() { local app=${4:-"reNgine.tasks"} local directory=${5:-"/usr/src/app/reNgine/"} - local base_command="celery -A $app worker --pool=gevent --autoscale=$concurrency,1 --loglevel=$loglevel -Q $queue -n $worker_name" --optimization=fair + local base_command="celery -A $app worker --pool=gevent --optimization=fair --autoscale=$concurrency,1 --loglevel=$loglevel -Q $queue -n $worker_name" if [ "$DEBUG" == "1" ]; then echo "watchmedo auto-restart --recursive --pattern=\"*.py\" --directory=\"$directory\" -- $base_command &" diff --git a/web/dashboard/migrations/0001_initial.py b/web/dashboard/migrations/0001_initial.py index 542cb1f1..44e9ac9a 100644 --- a/web/dashboard/migrations/0001_initial.py +++ b/web/dashboard/migrations/0001_initial.py @@ -1,8 +1,6 @@ -# Generated by Django 3.2.23 on 2024-09-06 01:47 +# Generated by Django 3.2.23 on 2024-06-19 02:43 -from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): @@ -10,25 +8,9 @@ class Migration(migrations.Migration): initial = True dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ - migrations.CreateModel( - name='ChaosAPIKey', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('key', models.CharField(max_length=500)), - ], - ), - migrations.CreateModel( - name='HackerOneAPIKey', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('username', models.CharField(max_length=500)), - ('key', models.CharField(max_length=500)), - ], - ), migrations.CreateModel( name='NetlasAPIKey', fields=[ @@ -67,31 +49,4 @@ class Migration(migrations.Migration): ('query', models.CharField(max_length=1000)), ], ), - migrations.CreateModel( - name='UserPreferences', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('bug_bounty_mode', models.BooleanField(default=True)), - ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - ), - migrations.CreateModel( - name='InAppNotification', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('notification_type', models.CharField(choices=[('system', 'system'), ('project', 'project')], default='system', max_length=10)), - ('status', models.CharField(choices=[('success', 'Success'), ('info', 'Informational'), ('warning', 'Warning'), ('error', 'Error')], default='info', max_length=10)), - ('title', models.CharField(max_length=255)), - ('description', models.TextField()), - ('icon', models.CharField(max_length=50)), - ('is_read', models.BooleanField(default=False)), - ('created_at', models.DateTimeField(auto_now_add=True)), - ('redirect_link', models.URLField(blank=True, max_length=255, null=True)), - ('open_in_new_tab', models.BooleanField(default=False)), - ('project', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='dashboard.project')), - ], - options={ - 'ordering': ['-created_at'], - }, - ), ] diff --git a/web/dashboard/migrations/0002_chaosapikey_hackeroneapikey_inappnotification_userpreferences.py b/web/dashboard/migrations/0002_chaosapikey_hackeroneapikey_inappnotification_userpreferences.py new file mode 100644 index 00000000..9823c3b1 --- /dev/null +++ b/web/dashboard/migrations/0002_chaosapikey_hackeroneapikey_inappnotification_userpreferences.py @@ -0,0 +1,58 @@ +# Generated by Django 3.2.23 on 2024-09-11 01:46 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('dashboard', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ChaosAPIKey', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('key', models.CharField(max_length=500)), + ], + ), + migrations.CreateModel( + name='HackerOneAPIKey', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('username', models.CharField(max_length=500)), + ('key', models.CharField(max_length=500)), + ], + ), + migrations.CreateModel( + name='UserPreferences', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('bug_bounty_mode', models.BooleanField(default=True)), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='InAppNotification', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('notification_type', models.CharField(choices=[('system', 'system'), ('project', 'project')], default='system', max_length=10)), + ('status', models.CharField(choices=[('success', 'Success'), ('info', 'Informational'), ('warning', 'Warning'), ('error', 'Error')], default='info', max_length=10)), + ('title', models.CharField(max_length=255)), + ('description', models.TextField()), + ('icon', models.CharField(max_length=50)), + ('is_read', models.BooleanField(default=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('redirect_link', models.URLField(blank=True, max_length=255, null=True)), + ('open_in_new_tab', models.BooleanField(default=False)), + ('project', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='dashboard.project')), + ], + options={ + 'ordering': ['-created_at'], + }, + ), + ] diff --git a/web/dashboard/models.py b/web/dashboard/models.py index a1ed47a7..6628b16e 100644 --- a/web/dashboard/models.py +++ b/web/dashboard/models.py @@ -94,4 +94,4 @@ class UserPreferences(models.Model): bug_bounty_mode = models.BooleanField(default=True) def __str__(self): - return f"{self.user.username}'s preferences" + return f"{self.user.username}'s preferences" \ No newline at end of file diff --git a/web/recon_note/migrations/0001_initial.py b/web/recon_note/migrations/0001_initial.py index b4763f57..c9dff4ce 100644 --- a/web/recon_note/migrations/0001_initial.py +++ b/web/recon_note/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.23 on 2024-09-06 01:47 +# Generated by Django 3.2.23 on 2024-06-19 02:43 from django.db import migrations, models import django.db.models.deletion @@ -9,8 +9,8 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('dashboard', '0001_initial'), ('startScan', '0001_initial'), + ('dashboard', '0001_initial'), ] operations = [ diff --git a/web/scanEngine/migrations/0001_initial.py b/web/scanEngine/migrations/0001_initial.py index 2d58745f..aa0dcfc1 100644 --- a/web/scanEngine/migrations/0001_initial.py +++ b/web/scanEngine/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.23 on 2024-09-06 01:47 +# Generated by Django 3.2.23 on 2024-06-19 02:43 from django.db import migrations, models @@ -35,7 +35,6 @@ class Migration(migrations.Migration): ('id', models.AutoField(primary_key=True, serialize=False)), ('username', models.CharField(blank=True, max_length=100, null=True)), ('api_key', models.CharField(blank=True, max_length=200, null=True)), - ('send_report', models.BooleanField(blank=True, default=False, null=True)), ('send_critical', models.BooleanField(default=True)), ('send_high', models.BooleanField(default=True)), ('send_medium', models.BooleanField(default=False)), diff --git a/web/scanEngine/migrations/0002_hackerone_send_report.py b/web/scanEngine/migrations/0002_hackerone_send_report.py new file mode 100644 index 00000000..4023e222 --- /dev/null +++ b/web/scanEngine/migrations/0002_hackerone_send_report.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.23 on 2024-09-11 01:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanEngine', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='hackerone', + name='send_report', + field=models.BooleanField(blank=True, default=False, null=True), + ), + ] diff --git a/web/scanEngine/models.py b/web/scanEngine/models.py index f0beef50..8879f65f 100644 --- a/web/scanEngine/models.py +++ b/web/scanEngine/models.py @@ -140,4 +140,4 @@ class InstalledExternalTool(models.Model): subdomain_gathering_command = models.CharField(max_length=300, null=True, blank=True) def __str__(self): - return self.name + return self.name \ No newline at end of file diff --git a/web/startScan/migrations/0001_initial.py b/web/startScan/migrations/0001_initial.py index 303f2c6d..a2719120 100644 --- a/web/startScan/migrations/0001_initial.py +++ b/web/startScan/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.23 on 2024-09-06 01:47 +# Generated by Django 3.2.23 on 2024-06-19 02:43 from django.conf import settings import django.contrib.postgres.fields @@ -158,10 +158,6 @@ class Migration(migrations.Migration): ('stop_scan_date', models.DateTimeField(blank=True, null=True)), ('used_gf_patterns', models.CharField(blank=True, max_length=500, null=True)), ('error_message', models.CharField(blank=True, max_length=300, null=True)), - ('cfg_out_of_scope_subdomains', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None)), - ('cfg_starting_point_path', models.CharField(blank=True, max_length=200, null=True)), - ('cfg_excluded_paths', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None)), - ('cfg_imported_subdomains', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None)), ('aborted_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='aborted_scans', to=settings.AUTH_USER_MODEL)), ('buckets', models.ManyToManyField(blank=True, related_name='buckets', to='startScan.S3Bucket')), ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='targetApp.domain')), diff --git a/web/startScan/migrations/0002_auto_20240911_0145.py b/web/startScan/migrations/0002_auto_20240911_0145.py new file mode 100644 index 00000000..665ba541 --- /dev/null +++ b/web/startScan/migrations/0002_auto_20240911_0145.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.23 on 2024-09-11 01:45 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('startScan', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='scanhistory', + name='cfg_excluded_paths', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None), + ), + migrations.AddField( + model_name='scanhistory', + name='cfg_imported_subdomains', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None), + ), + migrations.AddField( + model_name='scanhistory', + name='cfg_out_of_scope_subdomains', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None), + ), + migrations.AddField( + model_name='scanhistory', + name='cfg_starting_point_path', + field=models.CharField(blank=True, max_length=200, null=True), + ), + ] diff --git a/web/startScan/models.py b/web/startScan/models.py index fcfbafa2..0f49c60a 100644 --- a/web/startScan/models.py +++ b/web/startScan/models.py @@ -678,4 +678,4 @@ class S3Bucket(models.Model): perm_all_users_write_acl = models.IntegerField(default=0) perm_all_users_full_control = models.IntegerField(default=0) num_objects = models.IntegerField(default=0) - size = models.IntegerField(default=0) + size = models.IntegerField(default=0) \ No newline at end of file diff --git a/web/targetApp/migrations/0001_initial.py b/web/targetApp/migrations/0001_initial.py index 0bfbc1a5..fb74ba39 100644 --- a/web/targetApp/migrations/0001_initial.py +++ b/web/targetApp/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.23 on 2024-09-06 01:47 +# Generated by Django 3.2.23 on 2024-06-19 02:43 from django.db import migrations, models import django.db.models.deletion