From dc9f88e299c05f16848356443e263518cba009b8 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Tue, 17 Jun 2025 20:52:16 +0500 Subject: [PATCH 1/5] fix to run result sbmission(with copy to predictions dir) --- compute_worker/compute_worker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index 85a2cbe06..e2edafc10 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -588,7 +588,7 @@ async def _run_program_directory(self, program_dir, kind): Function responsible for running program directory Args: - - program_dir : can be either ingestion program or program/submission + - program_dir : can be either ingestion program or program(submission or scoring) - kind : either `program` or `ingestion` """ # If the directory doesn't even exist, move on @@ -609,6 +609,9 @@ async def _run_program_directory(self, program_dir, kind): logger.info( "Program directory missing metadata, assuming it's going to be handled by ingestion" ) + # Copy program dir content to output directory because in case of only result submission, + # we need to copy the result submission files because the scoring program will use these as predictions + shutil.copytree(program_dir, self.output_dir) return else: raise SubmissionException("Program directory missing 'metadata.yaml/metadata'") From b00a4a7e52a8f9d57b7334a42918d42aa2e948a4 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 23 Jun 2025 19:40:22 +0500 Subject: [PATCH 2/5] chahub cleanup --- .../0028_submission_organization.py | 2 +- .../migrations/0059_auto_20250623_1341.py | 77 +++++++++++ src/apps/competitions/models.py | 122 +----------------- .../migrations/0011_auto_20250623_1341.py | 29 +++++ src/apps/datasets/models.py | 33 +---- src/apps/profiles/admin.py | 2 +- .../migrations/0003_auto_20191122_1942.py | 13 +- .../migrations/0017_auto_20250623_1341.py | 34 +++++ src/apps/profiles/models.py | 45 +------ src/apps/profiles/pipeline.py | 24 ---- src/apps/profiles/views.py | 10 -- .../migrations/0005_auto_20250623_1341.py | 45 +++++++ src/apps/tasks/models.py | 65 +--------- src/settings/base.py | 17 +-- src/settings/test.py | 4 - src/utils/oauth_backends.py | 31 ----- 16 files changed, 203 insertions(+), 350 deletions(-) create mode 100644 src/apps/competitions/migrations/0059_auto_20250623_1341.py create mode 100644 src/apps/datasets/migrations/0011_auto_20250623_1341.py create mode 100644 src/apps/profiles/migrations/0017_auto_20250623_1341.py delete mode 100644 src/apps/profiles/pipeline.py create mode 100644 src/apps/tasks/migrations/0005_auto_20250623_1341.py delete mode 100644 src/utils/oauth_backends.py diff --git a/src/apps/competitions/migrations/0028_submission_organization.py b/src/apps/competitions/migrations/0028_submission_organization.py index fe4e974fb..f617ef85d 100644 --- a/src/apps/competitions/migrations/0028_submission_organization.py +++ b/src/apps/competitions/migrations/0028_submission_organization.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('profiles', '0010_auto_20201230_0034'), + ('profiles', '0001_initial'), ('competitions', '0027_auto_20201219_1308'), ] diff --git a/src/apps/competitions/migrations/0059_auto_20250623_1341.py b/src/apps/competitions/migrations/0059_auto_20250623_1341.py new file mode 100644 index 000000000..235a0014c --- /dev/null +++ b/src/apps/competitions/migrations/0059_auto_20250623_1341.py @@ -0,0 +1,77 @@ +# Generated by Django 2.2.28 on 2025-06-23 13:41 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('competitions', '0058_phase_hide_prediction_output'), + ] + + operations = [ + migrations.RemoveField( + model_name='competition', + name='chahub_data_hash', + ), + migrations.RemoveField( + model_name='competition', + name='chahub_needs_retry', + ), + migrations.RemoveField( + model_name='competition', + name='chahub_timestamp', + ), + migrations.RemoveField( + model_name='competition', + name='deleted', + ), + migrations.RemoveField( + model_name='competitionparticipant', + name='chahub_data_hash', + ), + migrations.RemoveField( + model_name='competitionparticipant', + name='chahub_needs_retry', + ), + migrations.RemoveField( + model_name='competitionparticipant', + name='chahub_timestamp', + ), + migrations.RemoveField( + model_name='competitionparticipant', + name='deleted', + ), + migrations.RemoveField( + model_name='phase', + name='chahub_data_hash', + ), + migrations.RemoveField( + model_name='phase', + name='chahub_needs_retry', + ), + migrations.RemoveField( + model_name='phase', + name='chahub_timestamp', + ), + migrations.RemoveField( + model_name='phase', + name='deleted', + ), + migrations.RemoveField( + model_name='submission', + name='chahub_data_hash', + ), + migrations.RemoveField( + model_name='submission', + name='chahub_needs_retry', + ), + migrations.RemoveField( + model_name='submission', + name='chahub_timestamp', + ), + migrations.RemoveField( + model_name='submission', + name='deleted', + ), + ] diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index d8ce83dad..99d7476ec 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -5,7 +5,6 @@ import botocore.exceptions from django.conf import settings -from django.contrib.sites.models import Site from django.contrib.postgres.fields import JSONField from django.core.files.base import ContentFile from django.db import models @@ -15,7 +14,6 @@ from decimal import Decimal from celery_config import app, app_for_vhost -from chahub.models import ChaHubSaveMixin from leaderboards.models import SubmissionScore from profiles.models import User, Organization from utils.data import PathWrapper @@ -27,7 +25,7 @@ logger = logging.getLogger() -class Competition(ChaHubSaveMixin, models.Model): +class Competition(models.Model): COMPETITION = "competition" BENCHMARK = "benchmark" @@ -206,46 +204,6 @@ def update_phase_statuses(self): def get_absolute_url(self): return reverse('competitions:detail', kwargs={'pk': self.pk}) - @staticmethod - def get_chahub_endpoint(): - return "competitions/" - - def get_chahub_is_valid(self): - has_phases = self.phases.exists() - upload_finished = all([c.status == CompetitionCreationTaskStatus.FINISHED for c in - self.creation_statuses.all()]) if self.creation_statuses.exists() else True - return has_phases and upload_finished - - def get_whitelist(self): - return [ - 'remote_id', - 'participants', - 'phases', - 'published', - ] - - def get_chahub_data(self): - data = { - 'created_by': self.created_by.username, - 'creator_id': self.created_by.pk, - 'created_when': self.created_when.isoformat(), - 'title': self.title, - 'url': f'http://{Site.objects.get_current().domain}{self.get_absolute_url()}', - 'remote_id': self.pk, - 'published': self.published, - 'participants': [p.get_chahub_data() for p in self.participants.all()], - 'phases': [phase.get_chahub_data(send_competition_id=False) for phase in self.phases.all()], - } - start = getattr(self.phases.order_by('index').first(), 'start', None) - data['start'] = start.isoformat() if start is not None else None - end = getattr(self.phases.order_by('index').last(), 'end', None) - data['end'] = end.isoformat() if end is not None else None - if self.logo: - data['logo_url'] = self.logo.url - data['logo'] = self.logo.url - - return self.clean_private_data(data) - def make_logo_icon(self): if self.logo: # Read the content of the logo file @@ -318,7 +276,7 @@ def __str__(self): return f"pk: {self.pk} ({self.status})" -class Phase(ChaHubSaveMixin, models.Model): +class Phase(models.Model): PREVIOUS = "Previous" CURRENT = "Current" NEXT = "Next" @@ -388,30 +346,6 @@ def can_user_make_submissions(self, user): return False, 'Reached maximum allowed submissions for this phase' return True, None - @staticmethod - def get_chahub_endpoint(): - return 'phases/' - - def get_whitelist(self): - return ['remote_id', 'published', 'tasks', 'index', 'status', 'competition_remote_id'] - - def get_chahub_data(self, send_competition_id=True): - data = { - 'remote_id': self.pk, - 'published': self.published, - 'status': self.status, - 'index': self.index, - 'start': self.start.isoformat(), - 'end': self.end.isoformat() if self.end else None, - 'name': self.name, - 'description': self.description, - 'is_active': self.is_active, - 'tasks': [task.get_chahub_data() for task in self.tasks.all()] - } - if send_competition_id: - data['competition_remote_id'] = self.competition.pk - return self.clean_private_data(data) - @property def is_active(self): """ Returns true when this phase of the competition is on-going. """ @@ -488,7 +422,7 @@ def save(self, *args, **kwargs): return super().save(*args, **kwargs) -class Submission(ChaHubSaveMixin, models.Model): +class Submission(models.Model): NONE = "None" SUBMITTING = "Submitting" SUBMITTED = "Submitted" @@ -775,37 +709,8 @@ def on_leaderboard(self): on_leaderboard = bool(self.children.first().leaderboard) return on_leaderboard - @staticmethod - def get_chahub_endpoint(): - return "submissions/" - - def get_whitelist(self): - return [ - 'remote_id', - 'is_public', - 'competition', - 'phase_index', - 'data', - ] - - def get_chahub_data(self): - data = { - "remote_id": self.id, - "is_public": self.is_public, - "competition": self.phase.competition_id, - "phase_index": self.phase.index, - "owner": self.owner.id, - "participant_name": self.owner.username, - "submitted_at": self.created_when.isoformat(), - "data": self.data.get_chahub_data(), - } - return self.clean_private_data(data) - - def get_chahub_is_valid(self): - return self.status == self.FINISHED - -class CompetitionParticipant(ChaHubSaveMixin, models.Model): +class CompetitionParticipant(models.Model): UNKNOWN = 'unknown' DENIED = 'denied' APPROVED = 'approved' @@ -829,25 +734,6 @@ class Meta: def __str__(self): return f"({self.id}) - User: {self.user.username} in Competition: {self.competition.title}" - @staticmethod - def get_chahub_endpoint(): - return 'participants/' - - def get_whitelist(self): - return [ - 'remote_id', - 'competition_id' - ] - - def get_chahub_data(self): - data = { - 'remote_id': self.pk, - 'user': self.user.id, - 'status': self.status, - 'competition_id': self.competition_id - } - return self.clean_private_data(data) - def save(self, *args, **kwargs): # Determine if this is a new participant (no existing record in DB) is_new = self.pk is None diff --git a/src/apps/datasets/migrations/0011_auto_20250623_1341.py b/src/apps/datasets/migrations/0011_auto_20250623_1341.py new file mode 100644 index 000000000..ef8022e4f --- /dev/null +++ b/src/apps/datasets/migrations/0011_auto_20250623_1341.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.28 on 2025-06-23 13:41 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('datasets', '0010_auto_20250218_1100'), + ] + + operations = [ + migrations.RemoveField( + model_name='data', + name='chahub_data_hash', + ), + migrations.RemoveField( + model_name='data', + name='chahub_needs_retry', + ), + migrations.RemoveField( + model_name='data', + name='chahub_timestamp', + ), + migrations.RemoveField( + model_name='data', + name='deleted', + ), + ] diff --git a/src/apps/datasets/models.py b/src/apps/datasets/models.py index 67edb7343..6cb04908c 100644 --- a/src/apps/datasets/models.py +++ b/src/apps/datasets/models.py @@ -4,14 +4,12 @@ import botocore.exceptions from django.conf import settings -from django.contrib.sites.models import Site from django.db import models from django.db.models import Q from django.urls import reverse from django.utils.timezone import now from decimal import Decimal -from chahub.models import ChaHubSaveMixin from utils.data import PathWrapper from utils.storage import BundleStorage from competitions.models import Competition @@ -20,7 +18,7 @@ logger = logging.getLogger() -class Data(ChaHubSaveMixin, models.Model): +class Data(models.Model): """Data models are unqiue based on name + created_by. If no name is given, then there is no uniqueness to enforce""" # It's useful to have these defaults map to the YAML names for these, like `scoring_program` @@ -110,35 +108,6 @@ def in_use(self): def __str__(self): return f'{self.name}({self.id})' - @staticmethod - def get_chahub_endpoint(): - return "datasets/" - - def get_chahub_is_valid(self): - if not self.was_created_by_competition: - return self.upload_completed_successfully - else: - return True - - def get_whitelist(self): - return ['remote_id', 'is_public'] - - def get_chahub_data(self): - ssl = settings.SECURE_SSL_REDIRECT - site = Site.objects.get_current().domain - return self.clean_private_data({ - 'creator_id': self.created_by.id, - 'remote_id': self.pk, - 'created_by': str(self.created_by.username), - 'created_when': self.created_when.isoformat(), - 'name': self.name, - 'type': self.type, - 'description': self.description, - 'key': str(self.key), - 'is_public': self.is_public, - 'download_url': f'http{"s" if ssl else ""}://{site}{self.get_download_url()}' - }) - class DataGroup(models.Model): created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) diff --git a/src/apps/profiles/admin.py b/src/apps/profiles/admin.py index e22e8f9ea..cb2c5bee3 100644 --- a/src/apps/profiles/admin.py +++ b/src/apps/profiles/admin.py @@ -8,7 +8,7 @@ class UserAdmin(admin.ModelAdmin): change_form_template = "admin/auth/user/change_form.html" change_list_template = "admin/auth/user/change_list.html" search_fields = ['username', 'email'] - list_filter = ['is_staff', 'is_superuser', 'deleted', 'is_deleted', 'is_bot'] + list_filter = ['is_staff', 'is_superuser', 'is_deleted', 'is_bot'] list_display = ['username', 'email', 'is_staff', 'is_superuser'] diff --git a/src/apps/profiles/migrations/0003_auto_20191122_1942.py b/src/apps/profiles/migrations/0003_auto_20191122_1942.py index ec2474036..45518353a 100644 --- a/src/apps/profiles/migrations/0003_auto_20191122_1942.py +++ b/src/apps/profiles/migrations/0003_auto_20191122_1942.py @@ -1,7 +1,6 @@ # Generated by Django 2.1.11 on 2019-11-22 19:42 from django.db import migrations, models -import profiles.models class Migration(migrations.Migration): @@ -11,12 +10,12 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterModelManagers( - name='user', - managers=[ - ('objects', profiles.models.ChaHubUserManager()), - ], - ), + # migrations.AlterModelManagers( + # name='user', + # managers=[ + # ('objects', profiles.models.ChaHubUserManager()), + # ], + # ), migrations.AddField( model_name='user', name='deleted', diff --git a/src/apps/profiles/migrations/0017_auto_20250623_1341.py b/src/apps/profiles/migrations/0017_auto_20250623_1341.py new file mode 100644 index 000000000..e9601bfdb --- /dev/null +++ b/src/apps/profiles/migrations/0017_auto_20250623_1341.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.28 on 2025-06-23 13:41 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('profiles', '0016_deleteduser_user_id'), + ] + + operations = [ + migrations.AlterModelManagers( + name='user', + managers=[ + ], + ), + migrations.RemoveField( + model_name='user', + name='chahub_data_hash', + ), + migrations.RemoveField( + model_name='user', + name='chahub_needs_retry', + ), + migrations.RemoveField( + model_name='user', + name='chahub_timestamp', + ), + migrations.RemoveField( + model_name='user', + name='deleted', + ), + ] diff --git a/src/apps/profiles/models.py b/src/apps/profiles/models.py index 3a660440d..5caf7db2d 100644 --- a/src/apps/profiles/models.py +++ b/src/apps/profiles/models.py @@ -1,9 +1,8 @@ import uuid -from django.contrib.auth.models import PermissionsMixin, AbstractBaseUser, UserManager +from django.contrib.auth.models import PermissionsMixin, AbstractBaseUser from django.db import models from django.utils.timezone import now -from chahub.models import ChaHubSaveMixin from django.utils.text import slugify from utils.data import PathWrapper from django.urls import reverse @@ -25,14 +24,6 @@ ] -class ChaHubUserManager(UserManager): - def get_queryset(self): - return super().get_queryset().filter(deleted=False) - - def all_objects(self): - return super().get_queryset() - - class DeletedUser(models.Model): user_id = models.IntegerField(null=True, blank=True) # Store the same ID as in the User table username = models.CharField(max_length=255) @@ -43,7 +34,7 @@ def __str__(self): return f"{self.username} ({self.email})" -class User(ChaHubSaveMixin, AbstractBaseUser, PermissionsMixin): +class User(AbstractBaseUser, PermissionsMixin): # Social needs the below setting. Username is not really set to UID. USERNAME_FIELD = 'username' EMAIL_FIELD = 'email' @@ -100,9 +91,6 @@ class User(ChaHubSaveMixin, AbstractBaseUser, PermissionsMixin): # Robot submissions is_bot = models.BooleanField(default=False) - # Required for social auth and such to create users - objects = ChaHubUserManager() - # Soft deletion is_deleted = models.BooleanField(default=False) deleted_at = models.DateTimeField(null=True, blank=True) @@ -124,35 +112,6 @@ def __str__(self): def slug_url(self): return reverse('profiles:user_profile', args=[self.slug]) - @staticmethod - def get_chahub_endpoint(): - return "profiles/" - - def get_whitelist(self): - # all chahub data is ok to send - pass - - def clean_private_data(self, data): - # overriding this to filter out blacklist data from above, just to make _sure_ we don't send that info - return {k: v for k, v in data.items() if k not in PROFILE_DATA_BLACKLIST} - - def get_chahub_data(self): - data = { - 'email': self.email, - 'username': self.username, - 'remote_id': self.pk, - 'details': { - "is_active": self.is_active, - "last_login": self.last_login.isoformat() if self.last_login else None, - "date_joined": self.date_joined.isoformat() if self.date_joined else None, - } - } - return self.clean_private_data(data) - - def get_chahub_is_valid(self): - # By default, always push - return True - def get_used_storage_space(self, binary=False): """ Function to calculate storage used by a user diff --git a/src/apps/profiles/pipeline.py b/src/apps/profiles/pipeline.py deleted file mode 100644 index b14ea63ca..000000000 --- a/src/apps/profiles/pipeline.py +++ /dev/null @@ -1,24 +0,0 @@ -from profiles.models import GithubUserInfo - - -def user_details(user, **kwargs): - """Update user details using data from provider.""" - backend = kwargs.get('backend') - - if user: - if backend and backend.name == 'chahub': - if kwargs.get('details', {}).get('github_info'): - github_info = kwargs['details'].pop('github_info', None) - if github_info and github_info.get('uid'): - obj, created = GithubUserInfo.objects.update_or_create( - uid=github_info.pop('uid'), - defaults=github_info, - ) - user.github_info = obj - user.save() - if created: - print("New github user info created for user: {}".format(user.username)) - if not created: - print("We updated existing info for user: {}".format(user.username)) - else: - pass diff --git a/src/apps/profiles/views.py b/src/apps/profiles/views.py index 9fbc2daba..16bcd5d8c 100644 --- a/src/apps/profiles/views.py +++ b/src/apps/profiles/views.py @@ -33,8 +33,6 @@ class LoginView(auth_views.LoginView): def get_context_data(self, *args, **kwargs): context = super(LoginView, self).get_context_data(*args, **kwargs) - # "http://localhost:8888/profiles/signup?next=http://localhost/social/login/chahub" - context['chahub_signup_url'] = "{}/profiles/signup?next={}/social/login/chahub".format(settings.SOCIAL_AUTH_CHAHUB_BASE_URL, settings.SITE_DOMAIN) return context @@ -201,10 +199,6 @@ def sign_up(request): return redirect('accounts:login') context = {} - context['chahub_signup_url'] = "{}/profiles/signup?next={}/social/login/chahub".format( - settings.SOCIAL_AUTH_CHAHUB_BASE_URL, - settings.SITE_DOMAIN - ) if request.method == 'POST': form = SignUpForm(request.POST) if form.is_valid(): @@ -268,10 +262,6 @@ def log_in(request): next = request.GET.get('next', None) context = {} - context['chahub_signup_url'] = "{}/profiles/signup?next={}/social/login/chahub".format( - settings.SOCIAL_AUTH_CHAHUB_BASE_URL, - settings.SITE_DOMAIN - ) if request.method == 'POST': form = LoginForm(request.POST) diff --git a/src/apps/tasks/migrations/0005_auto_20250623_1341.py b/src/apps/tasks/migrations/0005_auto_20250623_1341.py new file mode 100644 index 000000000..a4ffc9cf2 --- /dev/null +++ b/src/apps/tasks/migrations/0005_auto_20250623_1341.py @@ -0,0 +1,45 @@ +# Generated by Django 2.2.28 on 2025-06-23 13:41 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('tasks', '0004_task_shared_with'), + ] + + operations = [ + migrations.RemoveField( + model_name='solution', + name='chahub_data_hash', + ), + migrations.RemoveField( + model_name='solution', + name='chahub_needs_retry', + ), + migrations.RemoveField( + model_name='solution', + name='chahub_timestamp', + ), + migrations.RemoveField( + model_name='solution', + name='deleted', + ), + migrations.RemoveField( + model_name='task', + name='chahub_data_hash', + ), + migrations.RemoveField( + model_name='task', + name='chahub_needs_retry', + ), + migrations.RemoveField( + model_name='task', + name='chahub_timestamp', + ), + migrations.RemoveField( + model_name='task', + name='deleted', + ), + ] diff --git a/src/apps/tasks/models.py b/src/apps/tasks/models.py index c93d2b60a..e54ed75e6 100644 --- a/src/apps/tasks/models.py +++ b/src/apps/tasks/models.py @@ -4,10 +4,8 @@ from django.db import models from django.utils.timezone import now -from chahub.models import ChaHubSaveMixin - -class Task(ChaHubSaveMixin, models.Model): +class Task(models.Model): name = models.CharField(max_length=256) description = models.TextField(null=True, blank=True) key = models.UUIDField(default=uuid.uuid4, blank=True, unique=True) @@ -31,43 +29,8 @@ def _validated(self): # TODO: Should only include submissions that are successful, not any! return self.solutions.filter(md5__in=self.phases.values_list('submissions__md5', flat=True)).exists() - @staticmethod - def get_chahub_endpoint(): - return 'tasks/' - - def get_whitelist(self): - return [ - 'remote_id', - 'is_public', - 'solutions', - 'ingestion_program', - 'input_data', - 'reference_data', - 'scoring_program', - ] - - def get_chahub_data(self, include_solutions=True): - data = { - 'remote_id': self.pk, - 'created_by': self.created_by.username, - 'creator_id': self.created_by.pk, - 'created_when': self.created_when.isoformat(), - 'name': self.name, - 'description': self.description, - 'key': str(self.key), - 'is_public': self.is_public, - 'ingestion_program': self.ingestion_program.get_chahub_data() if self.ingestion_program else None, - 'input_data': self.input_data.get_chahub_data() if self.input_data else None, - 'ingestion_only_during_scoring': self.ingestion_only_during_scoring, - 'reference_data': self.reference_data.get_chahub_data() if self.reference_data else None, - 'scoring_program': self.scoring_program.get_chahub_data() if self.scoring_program else None, - } - if include_solutions: - data['solutions'] = [solution.get_chahub_data(include_tasks=False) for solution in self.solutions.all()] - return self.clean_private_data(data) - -class Solution(ChaHubSaveMixin, models.Model): +class Solution(models.Model): name = models.CharField(max_length=256) description = models.TextField(null=True, blank=True) key = models.UUIDField(default=uuid.uuid4, blank=True, unique=True) @@ -79,27 +42,3 @@ class Solution(ChaHubSaveMixin, models.Model): def __str__(self): return f"Solution - {self.name} - ({self.id})" - - @staticmethod - def get_chahub_endpoint(): - return 'solutions/' - - def get_whitelist(self): - return [ - 'remote_id', - 'is_public', - 'data', - 'tasks', - ] - - def get_chahub_data(self, include_tasks=True): - data = { - 'remote_id': self.pk, - 'name': self.name, - 'description': self.description, - 'key': str(self.key), - 'data': self.data.get_chahub_data(), # Todo: Make sure data is public if solution is public - } - if include_tasks: - data['tasks'] = [task.get_chahub_data(include_solutions=False) for task in self.tasks.all()] - return self.clean_private_data(data) diff --git a/src/settings/base.py b/src/settings/base.py index 50d1616e2..0c55e4ddd 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -47,12 +47,11 @@ 'redis', ) OUR_APPS = ( - 'chahub', + 'profiles', 'analytics', 'competitions', 'datasets', 'pages', - 'profiles', 'leaderboards', 'tasks', 'commands', @@ -118,7 +117,6 @@ # ============================================================================= AUTHENTICATION_BACKENDS = ( 'social_core.backends.github.GithubOAuth2', - 'utils.oauth_backends.ChahubOAuth2', 'django.contrib.auth.backends.ModelBackend', 'django_su.backends.SuBackend', 'profiles.backends.EmailAuthenticationBackend', @@ -133,7 +131,6 @@ 'social_core.pipeline.social_auth.load_extra_data', # 'social_core.pipeline.user.user_details', 'social_core.pipeline.social_auth.associate_by_email', - 'profiles.pipeline.user_details', ) # Github @@ -141,11 +138,6 @@ SOCIAL_AUTH_GITHUB_SECRET = os.environ.get('SOCIAL_AUTH_GITHUB_SECRET') SOCIAL_AUTH_GITHUB_SCOPE = ['user'] -# Codalab Example settings -SOCIAL_AUTH_CHAHUB_BASE_URL = os.environ.get('SOCIAL_AUTH_CHAHUB_BASE_URL', 'asdfasdfasfd') -SOCIAL_AUTH_CHAHUB_KEY = os.environ.get('SOCIAL_AUTH_CHAHUB_KEY', 'asdfasdfasfd') -SOCIAL_AUTH_CHAHUB_SECRET = os.environ.get('SOCIAL_AUTH_CHAHUB_SECRET', 'asdfasdfasfdasdfasdf') - # Generic SOCIAL_AUTH_STRATEGY = 'social_django.strategy.DjangoStrategy' SOCIAL_AUTH_STORAGE = 'social_django.models.DjangoStorage' @@ -461,13 +453,6 @@ DEFAULT_FROM_EMAIL = os.environ.get('DEFAULT_FROM_EMAIL', 'Codabench ') SERVER_EMAIL = os.environ.get('SERVER_EMAIL', 'noreply@codabench.org') -# ============================================================================= -# Chahub -# ============================================================================= -CHAHUB_API_URL = os.environ.get('CHAHUB_API_URL') -CHAHUB_API_KEY = os.environ.get('CHAHUB_API_KEY') -CHAHUB_PRODUCER_ID = os.environ.get('CHAHUB_PRODUCER_ID') - # Django-Su (User impersonation) SU_LOGIN_CALLBACK = 'profiles.admin.su_login_callback' diff --git a/src/settings/test.py b/src/settings/test.py index 21c5ababc..ec5963f3d 100644 --- a/src/settings/test.py +++ b/src/settings/test.py @@ -24,7 +24,3 @@ SELENIUM_HOSTNAME = os.environ.get("SELENIUM_HOSTNAME", "localhost") IS_TESTING = True - -CHAHUB_API_URL = None -CHAHUB_API_KEY = None -CHAHUB_PRODUCER_ID = None diff --git a/src/utils/oauth_backends.py b/src/utils/oauth_backends.py deleted file mode 100644 index c4a766194..000000000 --- a/src/utils/oauth_backends.py +++ /dev/null @@ -1,31 +0,0 @@ -from django.conf import settings -from social_core.backends.oauth import BaseOAuth2 - - -BASE_URL = settings.SOCIAL_AUTH_CHAHUB_BASE_URL - - -class ChahubOAuth2(BaseOAuth2): - """Chahub OAuth authentication backend""" - name = 'chahub' - API_URL = '{}/api/v1/'.format(BASE_URL) - AUTHORIZATION_URL = '{}/oauth/authorize/'.format(BASE_URL) - ACCESS_TOKEN_URL = '{}/oauth/token/'.format(BASE_URL) - ACCESS_TOKEN_METHOD = 'POST' - ID_KEY = 'id' - - def get_user_id(self, details, response): - return details.get('id') - - def get_user_details(self, response): - access_token = response['access_token'] - my_profile_url = "{}my_profile/".format(self.API_URL) - data = self.get_json(my_profile_url, headers={'Authorization': 'Bearer {}'.format(access_token)}) - - return { - 'username': data.get('username'), - 'email': data.get('email'), - 'name': data.get('name', ''), - 'id': data.get('id'), - 'github_info': data.get('github_info', {}) - } From 469d3495795accebcf95697ac273864c8bc24cf5 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 23 Jun 2025 19:43:47 +0500 Subject: [PATCH 3/5] compute worker code reverted --- compute_worker/compute_worker.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index e2edafc10..85a2cbe06 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -588,7 +588,7 @@ async def _run_program_directory(self, program_dir, kind): Function responsible for running program directory Args: - - program_dir : can be either ingestion program or program(submission or scoring) + - program_dir : can be either ingestion program or program/submission - kind : either `program` or `ingestion` """ # If the directory doesn't even exist, move on @@ -609,9 +609,6 @@ async def _run_program_directory(self, program_dir, kind): logger.info( "Program directory missing metadata, assuming it's going to be handled by ingestion" ) - # Copy program dir content to output directory because in case of only result submission, - # we need to copy the result submission files because the scoring program will use these as predictions - shutil.copytree(program_dir, self.output_dir) return else: raise SubmissionException("Program directory missing 'metadata.yaml/metadata'") From bdb999bc760d6ea1afa7faa4e4a2997edf6a4550 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 23 Jun 2025 19:44:57 +0500 Subject: [PATCH 4/5] fixed migration --- .../competitions/migrations/0028_submission_organization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/competitions/migrations/0028_submission_organization.py b/src/apps/competitions/migrations/0028_submission_organization.py index f617ef85d..fe4e974fb 100644 --- a/src/apps/competitions/migrations/0028_submission_organization.py +++ b/src/apps/competitions/migrations/0028_submission_organization.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('profiles', '0001_initial'), + ('profiles', '0010_auto_20201230_0034'), ('competitions', '0027_auto_20201219_1308'), ] From bf16df5ad1c74d2fa07ecc47379d42e0a476e24e Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 23 Jun 2025 22:31:34 +0500 Subject: [PATCH 5/5] removed chahub app, updated UserManager --- src/apps/chahub/__init__.py | 0 src/apps/chahub/models.py | 143 --------------- src/apps/chahub/tasks.py | 165 ------------------ src/apps/chahub/tests/__init__.py | 0 src/apps/chahub/tests/test_chahub_mixin.py | 98 ----------- src/apps/chahub/tests/test_chahub_tasks.py | 37 ---- src/apps/chahub/tests/utils.py | 42 ----- src/apps/chahub/utils.py | 36 ---- .../migrations/0003_auto_20191122_1942.py | 13 +- .../migrations/0018_auto_20250623_1719.py | 20 +++ src/apps/profiles/models.py | 13 +- 11 files changed, 39 insertions(+), 528 deletions(-) delete mode 100644 src/apps/chahub/__init__.py delete mode 100644 src/apps/chahub/models.py delete mode 100644 src/apps/chahub/tasks.py delete mode 100644 src/apps/chahub/tests/__init__.py delete mode 100644 src/apps/chahub/tests/test_chahub_mixin.py delete mode 100644 src/apps/chahub/tests/test_chahub_tasks.py delete mode 100644 src/apps/chahub/tests/utils.py delete mode 100644 src/apps/chahub/utils.py create mode 100644 src/apps/profiles/migrations/0018_auto_20250623_1719.py diff --git a/src/apps/chahub/__init__.py b/src/apps/chahub/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/apps/chahub/models.py b/src/apps/chahub/models.py deleted file mode 100644 index 493b8a0a7..000000000 --- a/src/apps/chahub/models.py +++ /dev/null @@ -1,143 +0,0 @@ -import hashlib -import json -import logging - -from django.conf import settings -from django.db import models - -from chahub.tasks import send_to_chahub, delete_from_chahub - -logger = logging.getLogger(__name__) - - -class ChaHubModelManager(models.Manager): - def get_queryset(self): - return super().get_queryset().filter(deleted=False) - - def all_objects(self): - return super().get_queryset() - - -class ChaHubSaveMixin(models.Model): - """Helper mixin for saving model data to ChaHub. - - To use: - 1) Override `get_chahub_endpoint()` to return the endpoint on ChaHub API for this model - 2) Override `get_chahub_data()` to return a dictionary to send to ChaHub - 3) Override `get_whitelist()` to return a whitelist of fields to send to ChaHub if obj not public - 4) Be sure to call `self.clean_private_data()` inside `get_chahub_data` - 5) Override `get_chahub_is_valid()` to return True/False on whether or not the object is ready to send to ChaHub - 6) Data is sent on `save()` and `chahub_timestamp` timestamp is set - - To update remove the `chahub_timestamp` timestamp and call `save()`""" - # Timestamp set whenever a successful update happens - chahub_timestamp = models.DateTimeField(null=True, blank=True) - - # A hash of the last json information that was sent to avoid sending duplicate information - chahub_data_hash = models.TextField(null=True, blank=True) - - # If sending to chahub fails, we may need a retry. Signal that by setting this attribute to True - chahub_needs_retry = models.BooleanField(default=False) - - # Set to true if celery attempt at deletion does not get a 204 resp from chahub, so we can retry later - deleted = models.BooleanField(default=False) - - objects = ChaHubModelManager() - - class Meta: - abstract = True - - @property - def app_label(self): - return f'{self.__class__._meta.app_label}.{self.__class__.__name__}' - - def get_whitelist(self): - """Override this to set the return the whitelisted fields for private data - Example: - return ['remote_id', 'is_public'] - """ - raise NotImplementedError() - - # ------------------------------------------------------------------------- - # METHODS TO OVERRIDE WHEN USING THIS MIXIN! - # ------------------------------------------------------------------------- - @staticmethod - def get_chahub_endpoint(): - """Override this to return the endpoint URL for this resource - - Example: - # If the endpoint is chahub.org/api/v1/competitions/ then... - return "competitions/" - """ - raise NotImplementedError() - - def get_chahub_data(self): - """Override this to return a dictionary with data to send to chahub - - Example: - return {"name": self.name} - """ - raise NotImplementedError() - - def get_chahub_is_valid(self): - """Override this to validate the specific model before it's sent - - Example: - return comp.is_published - """ - # By default, always push - return True - - def clean_private_data(self, data): - """Override this to clean up any data that should not be sent to chahub if the object is not public""" - if hasattr(self, 'is_public'): - public = self.is_public - elif hasattr(self, 'published'): - public = self.published - else: - # assume data is good to push to chahub if there is no field saying otherwise - public = True - if not public: - for key in data.keys(): - if key not in self.get_whitelist(): - data[key] = None - return data - - # Regular methods - def save(self, send=True, *args, **kwargs): - # We do a save here to give us an ID for generating URLs and such - super().save(*args, **kwargs) - - # making sure get whitelist was implemented, making sure we don't send to chahub without cleaning our data - self.get_whitelist() - - if getattr(settings, 'IS_TESTING', False) and not getattr(settings, 'PYTEST_FORCE_CHAHUB', False): - # For tests let's just assume Chahub isn't available - # We can mock proper responses - return None - - # Make sure we're not sending these in tests - if settings.CHAHUB_API_URL and send: - is_valid = self.get_chahub_is_valid() - logger.info(f"ChaHub :: {self.__class__.__name__}({self.pk}) is_valid = {is_valid}") - - if is_valid: - data = [self.clean_private_data(self.get_chahub_data())] - - data_hash = hashlib.md5(json.dumps(data).encode('utf-8')).hexdigest() - # Send to chahub if we haven't yet, we have new data - if not self.chahub_timestamp or self.chahub_data_hash != data_hash: - send_to_chahub.apply_async((self.app_label, self.pk, data, data_hash)) - elif self.chahub_needs_retry: - # This is NOT valid but also marked as need retry, unmark need retry until this is valid again - logger.info('ChaHub :: This is invalid but marked for retry. Clearing retry until valid again.') - self.chahub_needs_retry = False - super().save() - - def delete(self, send=True, *args, **kwargs): - if settings.CHAHUB_API_URL and send: - self.deleted = True - self.save(send=False) - delete_from_chahub.apply_async((self.app_label, self.pk)) - else: - super().delete(*args, **kwargs) diff --git a/src/apps/chahub/tasks.py b/src/apps/chahub/tasks.py deleted file mode 100644 index a0049999a..000000000 --- a/src/apps/chahub/tasks.py +++ /dev/null @@ -1,165 +0,0 @@ -import json -import logging - -import requests -from django.utils import timezone - -from celery_config import app -from django.apps import apps -from django.conf import settings -from apps.chahub.utils import ChahubException - -logger = logging.getLogger(__name__) - - -def _send(endpoint, data): - url = f"{settings.CHAHUB_API_URL}{endpoint}" - headers = { - 'Content-type': 'application/json', - 'X-CHAHUB-API-KEY': settings.CHAHUB_API_KEY, - } - logger.info(f"ChaHub :: Sending to ChaHub ({url}) the following data: \n{data}") - return requests.post(url=url, data=json.dumps(data), headers=headers) - - -def get_obj(app_label, pk, include_deleted=False): - Model = apps.get_model(app_label) - - try: - if include_deleted: - obj = Model.objects.all_objects().get(pk=pk) - else: - obj = Model.objects.get(pk=pk) - except Model.DoesNotExist: - raise ChahubException(f"Could not find {app_label} with pk: {pk}") - return obj - - -@app.task(queue='site-worker') -def send_to_chahub(app_label, pk, data, data_hash): - """ - Does a post request to the specified API endpoint on chahub with the inputted data. - """ - if not settings.CHAHUB_API_URL: - raise ChahubException("CHAHUB_API_URL env var required to send to Chahub") - if not settings.CHAHUB_API_KEY: - raise ChahubException("No ChaHub API Key provided") - - obj = get_obj(app_label, pk) - - try: - resp = _send(obj.get_chahub_endpoint(), data) - except requests.exceptions.RequestException: - resp = None - - if resp and resp.status_code in (200, 201): - logger.info(f"ChaHub :: Received response {resp.status_code} {resp.content}") - obj.chahub_timestamp = timezone.now() - obj.chahub_data_hash = data_hash - obj.chahub_needs_retry = False - else: - status = getattr(resp, 'status_code', 'N/A') - body = getattr(resp, 'content', 'N/A') - logger.info(f"ChaHub :: Error sending to chahub, status={status}, body={body}") - obj.chahub_needs_retry = True - obj.save(send=False) - - -@app.task(queue='site-worker') -def delete_from_chahub(app_label, pk): - if not settings.CHAHUB_API_URL: - raise ChahubException("CHAHUB_API_URL env var required to send to Chahub") - if not settings.CHAHUB_API_KEY: - raise ChahubException("No ChaHub API Key provided") - - obj = get_obj(app_label, pk, include_deleted=True) - - url = f"{settings.CHAHUB_API_URL}{obj.get_chahub_endpoint()}{pk}/" - logger.info(f"ChaHub :: Sending to ChaHub ({url}) delete message") - - headers = {'X-CHAHUB-API-KEY': settings.CHAHUB_API_KEY} - - try: - resp = requests.delete(url=url, headers=headers) - except requests.exceptions.RequestException: - resp = None - - if resp and resp.status_code == 204: - logger.info(f"ChaHub :: Received response {resp.status_code} {resp.content}") - obj.delete(send=False) - else: - status = getattr(resp, 'status_code', 'N/A') - body = getattr(resp, 'content', 'N/A') - logger.info(f"ChaHub :: Error sending to chahub, status={status}, body={body}") - obj.chahub_needs_retry = True - obj.save(send=False) - - -def batch_send_to_chahub(model, limit=None, retry_only=False): - qs = model.objects.all() - if retry_only: - qs = qs.filter(chahub_needs_retry=True) - if limit is not None: - qs = qs[:limit] - - endpoint = model.get_chahub_endpoint() - data = [obj.get_chahub_data() for obj in qs if obj.get_chahub_is_valid()] - if not data: - logger.info(f'Nothing to send to Chahub at {endpoint}') - return - try: - logger.info(f"Sending all data to Chahub at {endpoint}") - resp = _send(endpoint=endpoint, data=data) - logger.info(f"Response Status Code: {resp.status_code}") - if resp.status_code != 201: - logger.warning(f'ChaHub Response Content: {resp.content}') - except ChahubException: - logger.info("There was a problem reaching Chahub. Retry again later") - - -def chahub_is_up(): - if not settings.CHAHUB_API_URL: - return False - - logger.info("Checking whether ChaHub is online before sending retries") - try: - response = requests.get(settings.CHAHUB_API_URL) - if response.ok: - logger.info("ChaHub is online") - return True - else: - logger.info("Bad Status from ChaHub") - return False - except requests.exceptions.RequestException: - # This base exception works for HTTP errors, Connection errors, etc. - logger.info("Request Exception trying to access ChaHub") - return False - - -def get_chahub_models(): - from chahub.models import ChaHubSaveMixin - return ChaHubSaveMixin.__subclasses__() - - -@app.task(queue='site-worker') -def do_chahub_retries(limit=None): - if not chahub_is_up(): - return - chahub_models = get_chahub_models() - logger.info(f'Retrying for ChaHub models: {chahub_models}') - for model in chahub_models: - batch_send_to_chahub(model, retry_only=True, limit=limit) - obj_to_be_deleted = model.objects.all_objects().filter(deleted=True) - if limit is not None: - obj_to_be_deleted = obj_to_be_deleted[:limit] - for obj in obj_to_be_deleted: - # TODO: call celery task here, instead of abstracting. - obj.delete() - - -@app.task(queue='site-worker') -def send_everything_to_chahub(limit=None): - if not chahub_is_up(): - return - for model in get_chahub_models(): - batch_send_to_chahub(model, limit=limit) diff --git a/src/apps/chahub/tests/__init__.py b/src/apps/chahub/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/apps/chahub/tests/test_chahub_mixin.py b/src/apps/chahub/tests/test_chahub_mixin.py deleted file mode 100644 index 7b408e75b..000000000 --- a/src/apps/chahub/tests/test_chahub_mixin.py +++ /dev/null @@ -1,98 +0,0 @@ -from chahub.tests.utils import ChaHubTestCase -from competitions.models import Submission -from factories import UserFactory, CompetitionFactory, DataFactory, SubmissionFactory, PhaseFactory, \ - CompetitionParticipantFactory -from profiles.models import User - - -class SubmissionMixinTests(ChaHubTestCase): - def setUp(self): - self.user = UserFactory() - self.comp = CompetitionFactory(published=True) - self.participant = CompetitionParticipantFactory(user=self.user, competition=self.comp) - self.phase = PhaseFactory(competition=self.comp) - self.data = DataFactory() - # Calling this after initial setup so we don't turn on FORCE_CHAHUB and try and send all our setup objects - super().setUp() - self.submission = SubmissionFactory.build( - owner=self.user, - phase=self.phase, - data=self.data, - participant=self.participant, - status='Finished', - is_public=True, - leaderboard=None - ) - - def test_submission_save_sends_to_chahub(self): - resp = self.mock_chahub_save(self.submission) - assert resp.called - - def test_submission_save_not_sending_duplicate_data(self): - resp1 = self.mock_chahub_save(self.submission) - assert resp1.called - self.submission = Submission.objects.get(id=self.submission.id) - resp2 = self.mock_chahub_save(self.submission) - assert not resp2.called - - def test_submission_save_sends_updated_data(self): - resp1 = self.mock_chahub_save(self.submission) - assert resp1.called - self.phase.index += 1 - resp2 = self.mock_chahub_save(self.submission) - assert resp2.called - - # def test_invalid_submission_not_sent(self): - # self.submission.status = "Running" - # self.submission.is_public = False - # resp1 = self.mock_chahub_save(self.submission) - # assert not resp1.called - # self.submission = Submission.objects.get(id=self.submission.id) - # self.submission.status = "Finished" - # resp2 = self.mock_chahub_save(self.submission) - # assert resp2.called - - # def test_retrying_invalid_submission_wont_retry_again(self): - # self.submission.status = "Running" - # self.submission.chahub_needs_retry = True - # resp = self.mock_chahub_save(self.submission) - # assert not resp.called - # assert not Submission.objects.get(id=self.submission.id).chahub_needs_retry - - def test_valid_submission_marked_for_retry_sent_and_needs_retry_unset(self): - # Mark submission for retry - self.submission.chahub_needs_retry = True - resp = self.mock_chahub_save(self.submission) - assert resp.called - assert not Submission.objects.get(id=self.submission.id).chahub_needs_retry - - -class ProfileMixinTests(ChaHubTestCase): - def setUp(self): - self.user = UserFactory.build(username='admin') # create a user but don't save until later in the mock - super().setUp() - - def test_profile_save_not_sending_on_blacklisted_data_update(self): - resp1 = self.mock_chahub_save(self.user) - assert resp1.called - self.user = User.objects.get(id=self.user.id) - self.user.password = 'this_is_different' # Not using user.set_password() to control when the save happens - resp2 = self.mock_chahub_save(self.user) - assert not resp2.called - - -class CompetitionMixinTests(ChaHubTestCase): - def setUp(self): - self.comp = CompetitionFactory(published=False) - PhaseFactory(competition=self.comp) - super().setUp() - - def test_unpublished_comp_doesnt_send_private_data(self): - resp = self.mock_chahub_save(self.comp) - # Gross traversal through call args to get the data passed to _send - assert resp.called - data = resp.call_args[0][1][0] - whitelist = self.comp.get_whitelist() - for key, value in data.items(): - if key not in whitelist: - assert value is None diff --git a/src/apps/chahub/tests/test_chahub_tasks.py b/src/apps/chahub/tests/test_chahub_tasks.py deleted file mode 100644 index cd8e6836e..000000000 --- a/src/apps/chahub/tests/test_chahub_tasks.py +++ /dev/null @@ -1,37 +0,0 @@ -from chahub.tests.utils import ChaHubTestCase -from factories import UserFactory, CompetitionFactory, DataFactory, SubmissionFactory, PhaseFactory, \ - CompetitionParticipantFactory - - -class ChaHubDoRetriesTests(ChaHubTestCase): - def setUp(self): - for _ in range(5): - user = UserFactory(chahub_needs_retry=True) - comp = CompetitionFactory(chahub_needs_retry=True, published=True) - participant = CompetitionParticipantFactory(competition=comp, user=user, status='approved') - phase = PhaseFactory(competition=comp) - DataFactory(chahub_needs_retry=True, is_public=True, upload_completed_successfully=True) - SubmissionFactory( - chahub_needs_retry=True, - status="Finished", - phase=phase, - is_public=True, - participant=participant - ) - super().setUp() - - def test_do_retries_picks_up_all_expected_items(self): - resp = self.mock_retries() - # Should call once each for Users, Comps, Datasets, Submissions - assert resp.call_count == 4 - for call in resp.call_args_list: - # Should get passed a batch of data that is 5 long - assert len(call[1]['data']) == 5 - - def test_do_retries_limit_will_limit_number_of_retries(self): - resp = self.mock_retries(limit=2) - # Should call once each for Users, Comps, Datasets, Submissions - assert resp.call_count == 4 - for call in resp.call_args_list: - # Should get passed a batch of data that is 2 long, matching the limit - assert len(call[1]['data']) == 2 diff --git a/src/apps/chahub/tests/utils.py b/src/apps/chahub/tests/utils.py deleted file mode 100644 index 0a8501303..000000000 --- a/src/apps/chahub/tests/utils.py +++ /dev/null @@ -1,42 +0,0 @@ -from unittest import mock - -from django.conf import settings -from django.http.response import HttpResponseBase -from django.test import TestCase - -from chahub.tasks import do_chahub_retries - - -class ChaHubTestResponse(HttpResponseBase): - @property - def ok(self): - return self.status_code < 400 - - -class ChaHubTestCase(TestCase): - def setUp(self): - settings.PYTEST_FORCE_CHAHUB = True - # set the url to localhost for tests - settings.CHAHUB_API_URL = 'http://localhost/' - settings.CHAHUB_API_KEY = 'asdf' - - def tearDown(self): - settings.PYTEST_FORCE_CHAHUB = False - settings.CHAHUB_API_URL = None - - def mock_chahub_save(self, obj): - with mock.patch('chahub.tasks._send') as chahub_mock: - chahub_mock.return_value = ChaHubTestResponse(status=201) - chahub_mock.return_value.content = '' - obj.save() - return chahub_mock - - def mock_retries(self, limit=None): - with mock.patch('apps.chahub.tasks.requests.get') as chahub_get_mock: - # This checks that ChaHub is up, mock this so the task doesn't bail - chahub_get_mock.return_value = ChaHubTestResponse(status=200) - with mock.patch('chahub.tasks._send') as send_to_chahub_mock: - send_to_chahub_mock.return_value = ChaHubTestResponse(status=201) - send_to_chahub_mock.return_value.content = '' - do_chahub_retries(limit=limit) - return send_to_chahub_mock diff --git a/src/apps/chahub/utils.py b/src/apps/chahub/utils.py deleted file mode 100644 index 6aa859c96..000000000 --- a/src/apps/chahub/utils.py +++ /dev/null @@ -1,36 +0,0 @@ -from django.conf import settings - -import logging -import requests -import json - -logger = logging.getLogger(__name__) - - -class ChahubException(Exception): - pass - - -def send_to_chahub(endpoint, data): - """ - Does a post request to the specified API endpoint on chahub with the inputted data. - :param endpoint: String designating which API endpoint; IE: 'producers/' - :param data: Dictionary containing data we are sending away to the endpoint. - :return: - """ - if not endpoint: - raise ChahubException("No ChaHub API endpoint given") - if not settings.CHAHUB_API_URL: - raise ChahubException("CHAHUB_API_URL env var required to send to Chahub") - - url = f"{settings.CHAHUB_API_URL}{endpoint}" - - logger.info(f"ChaHub :: Sending to ChaHub ({url}) the following data: \n{data}") - try: - headers = { - 'Content-type': 'application/json', - 'X-CHAHUB-API-KEY': settings.CHAHUB_API_KEY, - } - return requests.post(url=url, data=json.dumps(data), headers=headers) - except requests.ConnectionError: - raise ChahubException('Connection Error with ChaHub') diff --git a/src/apps/profiles/migrations/0003_auto_20191122_1942.py b/src/apps/profiles/migrations/0003_auto_20191122_1942.py index 45518353a..2bfd8a549 100644 --- a/src/apps/profiles/migrations/0003_auto_20191122_1942.py +++ b/src/apps/profiles/migrations/0003_auto_20191122_1942.py @@ -1,6 +1,7 @@ # Generated by Django 2.1.11 on 2019-11-22 19:42 from django.db import migrations, models +import profiles.models class Migration(migrations.Migration): @@ -10,12 +11,12 @@ class Migration(migrations.Migration): ] operations = [ - # migrations.AlterModelManagers( - # name='user', - # managers=[ - # ('objects', profiles.models.ChaHubUserManager()), - # ], - # ), + migrations.AlterModelManagers( + name='user', + managers=[ + ('objects', profiles.models.CodabenchUserManager()), + ], + ), migrations.AddField( model_name='user', name='deleted', diff --git a/src/apps/profiles/migrations/0018_auto_20250623_1719.py b/src/apps/profiles/migrations/0018_auto_20250623_1719.py new file mode 100644 index 000000000..25d07fb3b --- /dev/null +++ b/src/apps/profiles/migrations/0018_auto_20250623_1719.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.28 on 2025-06-23 17:19 + +from django.db import migrations +import profiles.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('profiles', '0017_auto_20250623_1341'), + ] + + operations = [ + migrations.AlterModelManagers( + name='user', + managers=[ + ('objects', profiles.models.CodabenchUserManager()), + ], + ), + ] diff --git a/src/apps/profiles/models.py b/src/apps/profiles/models.py index 5caf7db2d..4b5697551 100644 --- a/src/apps/profiles/models.py +++ b/src/apps/profiles/models.py @@ -1,6 +1,6 @@ import uuid -from django.contrib.auth.models import PermissionsMixin, AbstractBaseUser +from django.contrib.auth.models import PermissionsMixin, AbstractBaseUser, UserManager from django.db import models from django.utils.timezone import now from django.utils.text import slugify @@ -24,6 +24,14 @@ ] +class CodabenchUserManager(UserManager): + def get_queryset(self): + return super().get_queryset().filter() + + def all_objects(self): + return super().get_queryset() + + class DeletedUser(models.Model): user_id = models.IntegerField(null=True, blank=True) # Store the same ID as in the User table username = models.CharField(max_length=255) @@ -91,6 +99,9 @@ class User(AbstractBaseUser, PermissionsMixin): # Robot submissions is_bot = models.BooleanField(default=False) + # Required for social auth and such to create users + objects = CodabenchUserManager() + # Soft deletion is_deleted = models.BooleanField(default=False) deleted_at = models.DateTimeField(null=True, blank=True)