From 25b1d060475835580a0ccdba87ef8f25159a8b06 Mon Sep 17 00:00:00 2001 From: Parth Shandilya <24358501+ParthS007@users.noreply.github.com> Date: Mon, 20 May 2024 11:25:48 +0200 Subject: [PATCH] add-ons: Add AddonActivityLog model and logging --- docs/admin/addons.rst | 9 ++ docs/admin/config.rst | 12 ++- docs/changes.rst | 2 + .../migrations/0003_addonactivitylog.py | 71 ++++++++++++++++ weblate/addons/models.py | 51 +++++++++++- weblate/addons/tasks.py | 18 ++++ weblate/addons/tests.py | 33 +++++++- weblate/addons/views.py | 59 ++++++++----- weblate/templates/addons/addon_list.html | 3 + weblate/templates/addons/addon_logs.html | 83 +++++++++++++++++++ weblate/urls.py | 5 ++ 11 files changed, 322 insertions(+), 24 deletions(-) create mode 100644 weblate/addons/migrations/0003_addonactivitylog.py create mode 100644 weblate/templates/addons/addon_logs.html diff --git a/docs/admin/addons.rst b/docs/admin/addons.rst index f72d2f24d95f..20c3d868e704 100644 --- a/docs/admin/addons.rst +++ b/docs/admin/addons.rst @@ -1034,3 +1034,12 @@ Use the commit script to automatically change a translation before it is committ to the repository. It is passed as a single parameter consisting of the filename of a current translation. + + +Add-on activity logging +----------------------- + +Add-on activity log keeps track of the add-on execution and can be used to +keep track of add-on activity. + +The logs can be pruned after a certain time interval by configuring the :setting:`ADDON_ACTIVITY_LOG_EXPIRY`. diff --git a/docs/admin/config.rst b/docs/admin/config.rst index d3a000b842c7..b97ace4c32be 100644 --- a/docs/admin/config.rst +++ b/docs/admin/config.rst @@ -2002,7 +2002,17 @@ example: .. seealso:: :ref:`addons`, - :setting:`DEFAULT_ADDONS` + :setting:`DEFAULT_ADDONS`, + :setting:`ADDON_ACTIVITY_LOG_EXPIRY` + +.. setting:: ADDON_ACTIVITY_LOG_EXPIRY + +ADDON_ACTIVITY_LOG_EXPIRY +------------------------- + +.. versionadded:: 5.6 + +Configures how long activity logs for add-ons are kept. Defaults to 180 days. .. setting:: WEBLATE_EXPORTERS diff --git a/docs/changes.rst b/docs/changes.rst index 7381695bb607..78388028c5d2 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -5,6 +5,8 @@ Not yet released. **New features** +* :ref:`addons` Added add-on activity log model for tracking add-on activity. + **Improvements** * :ref:`subscriptions` now include strings which need updating. diff --git a/weblate/addons/migrations/0003_addonactivitylog.py b/weblate/addons/migrations/0003_addonactivitylog.py new file mode 100644 index 000000000000..0385a63ccbc6 --- /dev/null +++ b/weblate/addons/migrations/0003_addonactivitylog.py @@ -0,0 +1,71 @@ +# Copyright © Michal Čihař +# +# SPDX-License-Identifier: GPL-3.0-or-later + +# Generated by Django 4.2.5 on 2024-02-28 10:14 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("trans", "0012_alter_announcement_notify"), + ("addons", "0002_remove_addon_project_scope_addon_project_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="AddonActivityLog", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "event", + models.IntegerField( + choices=[ + (1, "repository post-push"), + (2, "repository post-update"), + (3, "repository pre-commit"), + (4, "repository post-commit"), + (5, "repository post-add"), + (6, "unit post-create"), + (7, "storage post-load"), + (8, "unit post-save"), + (9, "repository pre-update"), + (10, "repository pre-push"), + (11, "daily"), + (12, "component update"), + ] + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("details", models.JSONField(default=dict)), + ( + "addon", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="addons.addon" + ), + ), + ( + "component", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="trans.component", + ), + ), + ], + options={ + "verbose_name": "add-on activity log", + "verbose_name_plural": "add-on activity logs", + "ordering": ["-created"], + }, + ), + ] diff --git a/weblate/addons/models.py b/weblate/addons/models.py index 80a89913315b..be3f14b78a9b 100644 --- a/weblate/addons/models.py +++ b/weblate/addons/models.py @@ -11,7 +11,7 @@ from appconf import AppConf from django.db import Error as DjangoDatabaseError from django.db import models, transaction -from django.db.models import Q +from django.db.models import Q, QuerySet from django.db.models.signals import post_save from django.dispatch import receiver from django.urls import reverse @@ -203,6 +203,10 @@ def log_debug(self, message: str, *args): else: self.logger.debug(message, *args) + def get_addon_activity_logs(self) -> QuerySet[AddonActivityLog]: + """Return activity logs for add-on.""" + return self.addonactivitylog_set.order_by("-created") + class Event(models.Model): addon = models.ForeignKey(Addon, on_delete=models.deletion.CASCADE, db_index=False) @@ -252,6 +256,9 @@ class AddonsConf(AppConf): LOCALIZE_CDN_URL = None LOCALIZE_CDN_PATH = None + # How long to keep add-on activity log entries + ADDON_ACTIVITY_LOG_EXPIRY = 180 + class Meta: prefix = "" @@ -264,6 +271,17 @@ def execute_addon_event( method: str | Callable, args: tuple | None = None, ): + # Log logging result and error flag for add-on activity log + log_result = None + error_occurred = False + + # Events to exclude from logging + exclude_from_logging = { + AddonEvent.EVENT_UNIT_PRE_CREATE, + AddonEvent.EVENT_UNIT_POST_SAVE, + AddonEvent.EVENT_STORE_POST_LOAD, + } + with transaction.atomic(): scope.log_debug("running %s add-on: %s", event.label, addon.name) # Skip unsupported components silently @@ -282,14 +300,16 @@ def execute_addon_event( op=f"addon.{event.name}", description=addon.name ): if isinstance(method, str): - getattr(addon.addon, method)(*args) + log_result = getattr(addon.addon, method)(*args) else: # Callback is used in tasks - method(addon, component) + log_result = method(addon, component) except DjangoDatabaseError: raise except Exception as error: # Log failure + error_occurred = True + log_result = str(error) scope.log_error("failed %s add-on: %s: %s", event.label, addon.name, error) report_error(cause=f"add-on {addon.name} failed", project=component.project) # Uninstall no longer compatible add-ons @@ -297,6 +317,15 @@ def execute_addon_event( addon.disable() else: scope.log_debug("completed %s add-on: %s", event.label, addon.name) + finally: + # Check if add-on is still installed and log activity + if event not in exclude_from_logging and addon.pk is not None: + AddonActivityLog.objects.create( + addon=addon, + component=component, + event=event, + details={"result": log_result, "error": error_occurred}, + ) def handle_addon_event( @@ -455,3 +484,19 @@ def store_post_load_handler(sender, translation, store, **kwargs) -> None: (translation, store), translation=translation, ) + + +class AddonActivityLog(models.Model): + addon = models.ForeignKey(Addon, on_delete=models.deletion.CASCADE) + component = models.ForeignKey(Component, on_delete=models.deletion.CASCADE) + event = models.IntegerField(choices=AddonEvent.choices) + created = models.DateTimeField(auto_now_add=True) + details = models.JSONField(default=dict) + + class Meta: + verbose_name = "add-on activity log" + verbose_name_plural = "add-on activity logs" + ordering = ["-created"] + + def __str__(self): + return f"{self.addon}: {self.get_event_display()} at {self.created}" diff --git a/weblate/addons/tasks.py b/weblate/addons/tasks.py index 87be95c64120..c86349b6b8d4 100644 --- a/weblate/addons/tasks.py +++ b/weblate/addons/tasks.py @@ -5,11 +5,14 @@ from __future__ import annotations import os +from datetime import timedelta from celery.schedules import crontab +from django.conf import settings from django.db.models import F, Q from django.http import HttpRequest from django.utils import timezone +from django.utils.timezone import now from lxml import html from weblate.addons.events import AddonEvent @@ -125,6 +128,16 @@ def daily_callback(addon, component) -> None: ) +@app.task(trail=False) +def cleanup_addon_activity_log() -> None: + """Cleanup old add-on activity log entries.""" + from weblate.addons.models import AddonActivityLog + + AddonActivityLog.objects.filter( + created__lt=now() - timedelta(days=settings.ADDON_ACTIVITY_LOG_EXPIRY) + ).delete() + + @app.task( trail=False, autoretry_for=(WeblateLockTimeoutError,), @@ -139,3 +152,8 @@ def postconfigure_addon(addon_id: int, addon=None) -> None: @app.on_after_finalize.connect def setup_periodic_tasks(sender, **kwargs) -> None: sender.add_periodic_task(crontab(minute=45), daily_addons.s(), name="daily-addons") + sender.add_periodic_task( + crontab(hour=0, minute=40), # Not to run on minute 0 to spread the load + cleanup_addon_activity_log.s(), + name="cleanup-addon-activity-log", + ) diff --git a/weblate/addons/tests.py b/weblate/addons/tests.py index 59c500bfac7a..df4c25c67405 100644 --- a/weblate/addons/tests.py +++ b/weblate/addons/tests.py @@ -10,6 +10,7 @@ from django.core.management import call_command from django.core.management.base import CommandError +from django.test import TestCase from django.test.utils import override_settings from django.urls import reverse from django.utils import timezone @@ -48,7 +49,7 @@ from weblate.addons.properties import PropertiesSortAddon from weblate.addons.removal import RemoveComments, RemoveSuggestions from weblate.addons.resx import ResxUpdateAddon -from weblate.addons.tasks import daily_addons +from weblate.addons.tasks import cleanup_addon_activity_log, daily_addons from weblate.addons.xml import XMLCustomizeAddon from weblate.addons.yaml import YAMLCustomizeAddon from weblate.lang.models import Language @@ -689,6 +690,31 @@ def test_list(self) -> None: response = self.client.get(reverse("addons", kwargs=self.kw_component)) self.assertContains(response, "Generate MO files") + def test_addon_logs(self) -> None: + response = self.client.post( + reverse("addons", kwargs=self.kw_component), + {"name": "weblate.gettext.authors"}, + follow=True, + ) + addon = self.component.addon_set.all()[0] + response = self.client.get(reverse("addon-logs", kwargs={"pk": addon.pk})) + + self.assertEqual(response.status_code, 200) + self.assertTemplateUsed(response, "addons/addon_logs.html") + self.assertEqual(response.context["instance"], addon) + + def test_addon_logs_without_authentication(self) -> None: + response = self.client.post( + reverse("addons", kwargs=self.kw_component), + {"name": "weblate.gettext.authors"}, + follow=True, + ) + addon = self.component.addon_set.all()[0] + + self.client.logout() + response = self.client.get(reverse("addon-logs", kwargs={"pk": addon.pk})) + self.assertEqual(response.status_code, 403) + def test_add_simple(self) -> None: response = self.client.post( reverse("addons", kwargs=self.kw_component), @@ -1482,3 +1508,8 @@ def test_json(self): self.get_translation().commit_pending("test", None) self.assertNotEqual(rev, self.component.repository.last_revision) + + +class TasksTest(TestCase): + def test_cleanup_addon_activity_log(self) -> None: + cleanup_addon_activity_log() diff --git a/weblate/addons/views.py b/weblate/addons/views.py index 04b25441fb4d..eb413cd8a6e2 100644 --- a/weblate/addons/views.py +++ b/weblate/addons/views.py @@ -6,12 +6,12 @@ from django.shortcuts import redirect from django.urls import reverse from django.utils.translation import gettext -from django.views.generic import ListView, UpdateView +from django.views.generic import DetailView, ListView, UpdateView from weblate.addons.models import ADDONS, Addon from weblate.trans.models import Change, Component, Project from weblate.utils import messages -from weblate.utils.views import PathViewMixin +from weblate.utils.views import PathViewMixin, get_paginator class AddonList(PathViewMixin, ListView): @@ -155,8 +155,27 @@ def post(self, request, **kwargs): ) -class AddonDetail(UpdateView): +class BaseAddonView(DetailView): model = Addon + + def get_object(self): + obj = super().get_object() + if obj.component and not self.request.user.has_perm( + "component.edit", obj.component + ): + raise PermissionDenied("Can not edit component") + if obj.project and not self.request.user.has_perm("project.edit", obj.project): + raise PermissionDenied("Can not edit project") + if ( + obj.project is None + and obj.component is None + and not self.request.user.has_perm("management.addons") + ): + raise PermissionDenied("Can not manage add-ons") + return obj + + +class AddonDetail(BaseAddonView, UpdateView): template_name_suffix = "_detail" def get_form(self, form_class=None): @@ -180,22 +199,6 @@ def get_success_url(self): return reverse("manage-addons") return reverse("addons", kwargs={"path": target.get_url_path()}) - def get_object(self): - obj = super().get_object() - if obj.component and not self.request.user.has_perm( - "component.edit", obj.component - ): - raise PermissionDenied("Can not edit component") - if obj.project and not self.request.user.has_perm("project.edit", obj.project): - raise PermissionDenied("Can not edit project") - if ( - obj.project is None - and obj.component is None - and not self.request.user.has_perm("management.addons") - ): - raise PermissionDenied("Can not manage add-ons") - return obj - def post(self, request, *args, **kwargs): obj = self.get_object() obj.acting_user = request.user @@ -206,3 +209,21 @@ def post(self, request, *args, **kwargs): return redirect(reverse("manage-addons")) return redirect(reverse("addons", kwargs={"path": target.get_url_path()})) return super().post(request, *args, **kwargs) + + +class AddonLogs(BaseAddonView): + template_name_suffix = "_logs" + + def get_context_data(self, **kwargs): + result = super().get_context_data(**kwargs) + if self.object.project: + result["object"] = self.object.project + else: + result["object"] = self.object.component + result["instance"] = self.object + result["addon"] = self.object.addon + result["addon_activity_log"] = get_paginator( + self.request, + self.object.get_addon_activity_logs(), + ) + return result diff --git a/weblate/templates/addons/addon_list.html b/weblate/templates/addons/addon_list.html index 0e4977860511..1f16b4a2273f 100644 --- a/weblate/templates/addons/addon_list.html +++ b/weblate/templates/addons/addon_list.html @@ -43,6 +43,9 @@
{% csrf_token %}
+ + {% trans "View Logs" %} + {% if addon.addon.has_settings %} {% trans "Configure" %} diff --git a/weblate/templates/addons/addon_logs.html b/weblate/templates/addons/addon_logs.html new file mode 100644 index 000000000000..824eb352a853 --- /dev/null +++ b/weblate/templates/addons/addon_logs.html @@ -0,0 +1,83 @@ +{% extends "base.html" %} +{% load i18n %} +{% load translations %} +{% load crispy_forms_tags %} +{% load permissions %} + +{% block breadcrumbs %} +{% if object %} + {% path_object_breadcrumbs object %} +
  • {% trans "Add-ons" %}
    • +{% else %} +
  • {% trans "Add-ons" %}
    • +{% endif %} +
  • +{% if instance %} +{{ addon.verbose }} +{% else %} +{{ addon.verbose }} +{% endif %} +
    • +{% endblock %} + +{% block content %} + +
    +
    +

    {% trans "Add-on Activity Logs" %}

    +
    + + + + + + + + + + + {% for log in addon_activity_log %} + + + + + + + {% empty %} + + + + {% endfor %} + +
    {% trans "Status" %}{% trans "Event" %}{% trans "Details" %}{% trans "Timestamp" %}
    + {% if log.details.error_occured %} + + ⚠ + {% else %} + + ✔ + {% endif %} + + {{ log.get_event_display }} + {{ log.details.result }}{{ log.created|date:"Y-m-d H:i" }}
    {% trans "No add-on activity logs available." %}
    + + + {% if addon_activity_log.has_other_pages %} +
    +
    + +
    +
    + {% endif %} +
    + + {% endblock %} diff --git a/weblate/urls.py b/weblate/urls.py index 2fb776af2359..71b9bdb5c539 100644 --- a/weblate/urls.py +++ b/weblate/urls.py @@ -196,6 +196,11 @@ weblate.addons.views.AddonDetail.as_view(), name="addon-detail", ), + path( + "addon//logs/", + weblate.addons.views.AddonLogs.as_view(), + name="addon-logs", + ), path( "access//", weblate.trans.views.acl.manage_access,