From d5c933608e7c0f15251937f74f26e6017c772e5d Mon Sep 17 00:00:00 2001 From: Lukas Winkler Date: Wed, 22 Jul 2020 20:47:13 +0200 Subject: [PATCH] add simple framework for runing data consistency checks --- acros/management/commands/datacheck.py | 12 +++++++ acros/templates/acros/datacheck.html | 19 ++++++++++++ acros/urls.py | 3 +- acros/utils/checks/__init__.py | 5 +++ acros/utils/checks/basecheck.py | 5 +++ acros/utils/checks/check_registry.py | 22 +++++++++++++ acros/utils/checks/checks/acronym.py | 23 ++++++++++++++ acros/utils/checks/checks/image.py | 13 ++++++++ acros/utils/checks/messages.py | 43 ++++++++++++++++++++++++++ acros/views.py | 11 +++++++ 10 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 acros/management/commands/datacheck.py create mode 100644 acros/templates/acros/datacheck.html create mode 100644 acros/utils/checks/__init__.py create mode 100644 acros/utils/checks/basecheck.py create mode 100644 acros/utils/checks/check_registry.py create mode 100644 acros/utils/checks/checks/acronym.py create mode 100644 acros/utils/checks/checks/image.py create mode 100644 acros/utils/checks/messages.py diff --git a/acros/management/commands/datacheck.py b/acros/management/commands/datacheck.py new file mode 100644 index 0000000..cc3d07d --- /dev/null +++ b/acros/management/commands/datacheck.py @@ -0,0 +1,12 @@ +from django.core.management.base import BaseCommand + +from acros.utils.checks import registry + + +class Command(BaseCommand): + help = 'check integrity of data' + + def handle(self, *args, **kwargs): + errors = registry.run_checks() + for error in errors: + print(error) diff --git a/acros/templates/acros/datacheck.html b/acros/templates/acros/datacheck.html new file mode 100644 index 0000000..016f390 --- /dev/null +++ b/acros/templates/acros/datacheck.html @@ -0,0 +1,19 @@ +{% extends 'base.html' %} +{% load static %} +{% block heading %} +

Data Checks

+{% endblock %} + +{% block content %} + + {% for error in errors %} + + {% endfor %} + +{% endblock %} diff --git a/acros/urls.py b/acros/urls.py index e4468c9..fe81fb2 100644 --- a/acros/urls.py +++ b/acros/urls.py @@ -34,7 +34,8 @@ urlpatterns = [ path('tags', views.TagListView.as_view(), name='tags'), path('tag', RedirectView.as_view(pattern_name="tags")), path('tag/', views.TagAcroView.as_view(), name='tag'), - path('integrations', views.IntegrationsView.as_view(), name="integrations") + path('integrations', views.IntegrationsView.as_view(), name="integrations"), + path('datachecks', views.DataCheckView.as_view(), name="datachecks") ] if settings.DEBUG: diff --git a/acros/utils/checks/__init__.py b/acros/utils/checks/__init__.py new file mode 100644 index 0000000..948b345 --- /dev/null +++ b/acros/utils/checks/__init__.py @@ -0,0 +1,5 @@ +from .basecheck import BaseCheck +from .check_registry import CheckRegistry, registry +from .messages import DEBUG, INFO, WARNING, ERROR, CRITICAL, CheckMessage, CheckWarning, CheckInfo +from .checks.acronym import * +from .checks.image import * diff --git a/acros/utils/checks/basecheck.py b/acros/utils/checks/basecheck.py new file mode 100644 index 0000000..45d051e --- /dev/null +++ b/acros/utils/checks/basecheck.py @@ -0,0 +1,5 @@ +class BaseCheck: + def run(self): + return [] + + diff --git a/acros/utils/checks/check_registry.py b/acros/utils/checks/check_registry.py new file mode 100644 index 0000000..ef7c3f4 --- /dev/null +++ b/acros/utils/checks/check_registry.py @@ -0,0 +1,22 @@ +from typing import Set + +from . import BaseCheck + + +class CheckRegistry: + def __init__(self): + self.checks: Set[BaseCheck] = set() + + def register(self, check: BaseCheck): + self.checks.add(check) + + def run_checks(self): + errors = [] + for Check in self.checks: + inst = Check() + new_errors = list(inst.run()) + errors.extend(new_errors) + return errors + + +registry = CheckRegistry() diff --git a/acros/utils/checks/checks/acronym.py b/acros/utils/checks/checks/acronym.py new file mode 100644 index 0000000..3f8b3bf --- /dev/null +++ b/acros/utils/checks/checks/acronym.py @@ -0,0 +1,23 @@ +from acros.models import Acronym +from acros.utils.checks import BaseCheck, CheckWarning, registry, CheckInfo + + +class LetterCheck(BaseCheck): + def run(self): + for acronym in Acronym.objects.all(): + if acronym.acro_letters is None: + yield CheckInfo( + "missing acronym letters", + obj=acronym + ) + continue + let_len = len(acronym.acro_letters) + acr_len = len(acronym.name) + if let_len != acr_len: + yield CheckWarning( + f"number of letters selected ({let_len}) not equal to letters in acronym ({acr_len})", + obj=acronym + ) + + +registry.register(LetterCheck) diff --git a/acros/utils/checks/checks/image.py b/acros/utils/checks/checks/image.py new file mode 100644 index 0000000..37690bc --- /dev/null +++ b/acros/utils/checks/checks/image.py @@ -0,0 +1,13 @@ +from acros.models import WikipediaImage +from acros.utils.checks import BaseCheck, CheckWarning, registry + + +class AuthorAttributionCheck(BaseCheck): + def run(self): + for image in WikipediaImage.objects.all(): + if image.attribution_required and not image.artist: + yield CheckWarning("Image needs attribution, but is missing an artist", obj=image) + + +registry.register(AuthorAttributionCheck) + diff --git a/acros/utils/checks/messages.py b/acros/utils/checks/messages.py new file mode 100644 index 0000000..0d528fd --- /dev/null +++ b/acros/utils/checks/messages.py @@ -0,0 +1,43 @@ +from django.urls import reverse + +from acros.models import Acronym + +DEBUG = 10 +INFO = "info" +WARNING = "warning" +ERROR = 40 +CRITICAL = 50 + + +class CheckMessage: + def __init__(self, level: str, msg: str, obj=None): + self.level = level + self.msg = msg + self.obj = obj + + def __str__(self): + obj = str(self.obj) + print(self.obj._meta.label) + return f"{obj}: {self.msg}" + + @property + def edit_url(self): + if isinstance(self.obj, Acronym): + return reverse("edit", args=[self.obj.slug]) + return None + + @property + def admin_edit_url(self): + if isinstance(self.obj, Acronym): + return reverse("admin:acros_acronym_change", args=[self.obj.id]) + return None + + +class CheckWarning(CheckMessage): + def __init__(self, msg: str, obj=None): + super(CheckWarning, self).__init__(WARNING, msg, obj) + + +class CheckInfo(CheckMessage): + def __init__(self, msg: str, obj=None): + super(CheckInfo, self).__init__(INFO, msg, obj) diff --git a/acros/views.py b/acros/views.py index 54934cb..902f58f 100644 --- a/acros/views.py +++ b/acros/views.py @@ -11,6 +11,7 @@ from acros.forms import EditForm, AddForm, WikipediaForm, PaperForm, WeblinkForm from acros.models import Acronym, Tag, AcroOfTheDay, WikipediaLink, PaperReference, Weblink from acros.serializers import AcronymSerializer, AcronymListSerializer, TagSerializer from acros.utils.assets import get_css +from acros.utils.checks import registry handler404 = 'acros.views.PageNotFoundView' @@ -122,6 +123,16 @@ class TagAcroView(generic.ListView): return Acronym.objects.filter(tags__slug__exact=self.kwargs['slug']) +class DataCheckView(generic.TemplateView, LoginRequiredMixin): + template_name = "acros/datacheck.html" + + def get_context_data(self, **kwargs): + data = super().get_context_data(**kwargs) + errors = registry.run_checks() + data['errors'] = errors + return data + + #### API Views #### class AcronymViewSet(viewsets.ReadOnlyModelViewSet):