add-ons: Add AddonActivityLog model and logging

docs/admin/addons.rst

@@ -1034,3 +1034,12 @@ Use the commit script to automatically change a translation before it is committ
 to the repository.
 
 It is passed as a single parameter consisting of the filename of a current translation.
+
+
+Add-on activity logging
+-----------------------
+
+Add-on activity log keeps track of the add-on execution and can be used to
+keep track of add-on activity.
+
+The logs can be pruned after a certain time interval by configuring the :setting:`ADDON_ACTIVITY_LOG_EXPIRY`.

docs/admin/config.rst

@@ -2002,7 +2002,17 @@ example:
 .. seealso::
 
  :ref:`addons`,
- :setting:`DEFAULT_ADDONS`
+ :setting:`DEFAULT_ADDONS`,
+ :setting:`ADDON_ACTIVITY_LOG_EXPIRY`
+
+.. setting:: ADDON_ACTIVITY_LOG_EXPIRY
+
+ADDON_ACTIVITY_LOG_EXPIRY
+-------------------------
+
+.. versionadded:: 5.6
+
+Configures how long activity logs for add-ons are kept. Defaults to 180 days.
 
 .. setting:: WEBLATE_EXPORTERS
 

docs/changes.rst

@@ -5,6 +5,8 @@ Not yet released.
 
 **New features**
 
+* :ref:`addons` Added add-on activity log model for tracking add-on activity.
+
 **Improvements**
 
 * :ref:`subscriptions` now include strings which need updating.

weblate/addons/migrations/0003_addonactivitylog.py

@@ -0,0 +1,71 @@
+# Copyright © Michal Čihař <[email protected]>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# Generated by Django 4.2.5 on 2024-02-28 10:14
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("trans", "0012_alter_announcement_notify"),
+ ("addons", "0002_remove_addon_project_scope_addon_project_and_more"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="AddonActivityLog",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "event",
+ models.IntegerField(
+ choices=[
+ (1, "repository post-push"),
+ (2, "repository post-update"),
+ (3, "repository pre-commit"),
+ (4, "repository post-commit"),
+ (5, "repository post-add"),
+ (6, "unit post-create"),
+ (7, "storage post-load"),
+ (8, "unit post-save"),
+ (9, "repository pre-update"),
+ (10, "repository pre-push"),
+ (11, "daily"),
+ (12, "component update"),
+ ]
+ ),
+ ),
+ ("created", models.DateTimeField(auto_now_add=True)),
+ ("details", models.JSONField(default=dict)),
+ (
+ "addon",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE, to="addons.addon"
+ ),
+ ),
+ (
+ "component",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ to="trans.component",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "add-on activity log",
+ "verbose_name_plural": "add-on activity logs",
+ "ordering": ["-created"],
+ },
+ ),
+ ]

weblate/addons/models.py

@@ -11,7 +11,7 @@
 from appconf import AppConf
 from django.db import Error as DjangoDatabaseError
 from django.db import models, transaction
-from django.db.models import Q
+from django.db.models import Q, QuerySet
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 from django.urls import reverse
@@ -203,6 +203,10 @@ def log_debug(self, message: str, *args):
  else:
  self.logger.debug(message, *args)
 
+ def get_addon_activity_logs(self) -> QuerySet[AddonActivityLog]:
+ """Return activity logs for add-on."""
+ return self.addonactivitylog_set.order_by("-created")
+
 
 class Event(models.Model):
  addon = models.ForeignKey(Addon, on_delete=models.deletion.CASCADE, db_index=False)
@@ -252,6 +256,9 @@ class AddonsConf(AppConf):
  LOCALIZE_CDN_URL = None
  LOCALIZE_CDN_PATH = None
 
+ # How long to keep add-on activity log entries
+ ADDON_ACTIVITY_LOG_EXPIRY = 180
+
  class Meta:
  prefix = ""
 
@@ -264,6 +271,17 @@ def execute_addon_event(
  method: str | Callable,
  args: tuple | None = None,
 ):
+ # Log logging result and error flag for add-on activity log
+ log_result = None
+ error_occurred = False
+
+ # Events to exclude from logging
+ exclude_from_logging = {
+ AddonEvent.EVENT_UNIT_PRE_CREATE,
+ AddonEvent.EVENT_UNIT_POST_SAVE,
+ AddonEvent.EVENT_STORE_POST_LOAD,
+ }
+
  with transaction.atomic():
  scope.log_debug("running %s add-on: %s", event.label, addon.name)
  # Skip unsupported components silently
@@ -282,21 +300,32 @@ def execute_addon_event(
  op=f"addon.{event.name}", description=addon.name
  ):
  if isinstance(method, str):
- getattr(addon.addon, method)(*args)
+ log_result = getattr(addon.addon, method)(*args)
  else:
  # Callback is used in tasks
- method(addon, component)
+ log_result = method(addon, component)
  except DjangoDatabaseError:
  raise
  except Exception as error:
  # Log failure
+ error_occurred = True
+ log_result = str(error)
  scope.log_error("failed %s add-on: %s: %s", event.label, addon.name, error)
  report_error(cause=f"add-on {addon.name} failed", project=component.project)
  # Uninstall no longer compatible add-ons
  if not addon.addon.can_install(component, None):
  addon.disable()
  else:
  scope.log_debug("completed %s add-on: %s", event.label, addon.name)
+ finally:
+ # Check if add-on is still installed and log activity
+ if event not in exclude_from_logging and addon.pk is not None:
+ AddonActivityLog.objects.create(
+ addon=addon,
+ component=component,
+ event=event,
+ details={"result": log_result, "error": error_occurred},
+ )
 
 
 def handle_addon_event(
@@ -455,3 +484,19 @@ def store_post_load_handler(sender, translation, store, **kwargs) -> None:
  (translation, store),
  translation=translation,
  )
+
+
+class AddonActivityLog(models.Model):
+ addon = models.ForeignKey(Addon, on_delete=models.deletion.CASCADE)
+ component = models.ForeignKey(Component, on_delete=models.deletion.CASCADE)
+ event = models.IntegerField(choices=AddonEvent.choices)
+ created = models.DateTimeField(auto_now_add=True)
+ details = models.JSONField(default=dict)
+
+ class Meta:
+ verbose_name = "add-on activity log"
+ verbose_name_plural = "add-on activity logs"
+ ordering = ["-created"]
+
+ def __str__(self):
+ return f"{self.addon}: {self.get_event_display()} at {self.created}"

weblate/addons/tasks.py

@@ -5,11 +5,14 @@
 from __future__ import annotations
 
 import os
+from datetime import timedelta
 
 from celery.schedules import crontab
+from django.conf import settings
 from django.db.models import F, Q
 from django.http import HttpRequest
 from django.utils import timezone
+from django.utils.timezone import now
 from lxml import html
 
 from weblate.addons.events import AddonEvent
@@ -125,6 +128,16 @@ def daily_callback(addon, component) -> None:
  )
 
 
+@app.task(trail=False)
+def cleanup_addon_activity_log() -> None:
+ """Cleanup old add-on activity log entries."""
+ from weblate.addons.models import AddonActivityLog
+
+ AddonActivityLog.objects.filter(
+ created__lt=now() - timedelta(days=settings.ADDON_ACTIVITY_LOG_EXPIRY)
+ ).delete()
+
+
 @app.task(
  trail=False,
  autoretry_for=(WeblateLockTimeoutError,),
@@ -139,3 +152,8 @@ def postconfigure_addon(addon_id: int, addon=None) -> None:
 @app.on_after_finalize.connect
 def setup_periodic_tasks(sender, **kwargs) -> None:
  sender.add_periodic_task(crontab(minute=45), daily_addons.s(), name="daily-addons")
+ sender.add_periodic_task(
+ crontab(hour=0, minute=40), # Not to run on minute 0 to spread the load
+ cleanup_addon_activity_log.s(),
+ name="cleanup-addon-activity-log",
+ )

weblate/addons/tests.py

@@ -10,6 +10,7 @@
 
 from django.core.management import call_command
 from django.core.management.base import CommandError
+from django.test import TestCase
 from django.test.utils import override_settings
 from django.urls import reverse
 from django.utils import timezone
@@ -48,7 +49,7 @@
 from weblate.addons.properties import PropertiesSortAddon
 from weblate.addons.removal import RemoveComments, RemoveSuggestions
 from weblate.addons.resx import ResxUpdateAddon
-from weblate.addons.tasks import daily_addons
+from weblate.addons.tasks import cleanup_addon_activity_log, daily_addons
 from weblate.addons.xml import XMLCustomizeAddon
 from weblate.addons.yaml import YAMLCustomizeAddon
 from weblate.lang.models import Language
@@ -689,6 +690,19 @@ def test_list(self) -> None:
  response = self.client.get(reverse("addons", kwargs=self.kw_component))
  self.assertContains(response, "Generate MO files")
 
+ def test_addon_logs(self) -> None:
+ response = self.client.post(
+ reverse("addons", kwargs=self.kw_component),
+ {"name": "weblate.gettext.authors"},
+ follow=True,
+ )
+ addon = self.component.addon_set.all()[0]
+ response = self.client.get(reverse("addon-logs", kwargs={"pk": addon.pk}))
+
+ self.assertEqual(response.status_code, 200)
+ self.assertTemplateUsed(response, "addons/addon_logs.html")
+ self.assertEqual(response.context["instance"], addon)
+
  def test_add_simple(self) -> None:
  response = self.client.post(
  reverse("addons", kwargs=self.kw_component),
@@ -1482,3 +1496,8 @@ def test_json(self):
  self.get_translation().commit_pending("test", None)
 
  self.assertNotEqual(rev, self.component.repository.last_revision)
+
+
+class TasksTest(TestCase):
+ def test_cleanup_addon_activity_log(self) -> None:
+ cleanup_addon_activity_log()

weblate/addons/views.py

@@ -6,12 +6,12 @@
 from django.shortcuts import redirect
 from django.urls import reverse
 from django.utils.translation import gettext
-from django.views.generic import ListView, UpdateView
+from django.views.generic import DetailView, ListView, UpdateView
 
 from weblate.addons.models import ADDONS, Addon
 from weblate.trans.models import Change, Component, Project
 from weblate.utils import messages
-from weblate.utils.views import PathViewMixin
+from weblate.utils.views import PathViewMixin, get_paginator
 
 
 class AddonList(PathViewMixin, ListView):
@@ -206,3 +206,22 @@ def post(self, request, *args, **kwargs):
  return redirect(reverse("manage-addons"))
  return redirect(reverse("addons", kwargs={"path": target.get_url_path()}))
  return super().post(request, *args, **kwargs)
+
+
+class AddonLogs(DetailView):
+ model = Addon
+ template_name_suffix = "_logs"
+
+ def get_context_data(self, **kwargs):
+ result = super().get_context_data(**kwargs)
+ if self.object.project:
+ result["object"] = self.object.project
+ else:
+ result["object"] = self.object.component
+ result["instance"] = self.object
+ result["addon"] = self.object.addon
+ result["addon_activity_log"] = get_paginator(
+ self.request,
+ self.object.get_addon_activity_logs(),
+ )
+ return result

weblate/templates/addons/addon_list.html

@@ -43,6 +43,9 @@
  <td class="bottom-button">
  <form method="POST" action="{{ addon.get_absolute_url }}">{% csrf_token %}<input type="hidden" name="delete" value="{{ addon.name }}" /><button type="submit" class="btn btn-danger">{% trans "Uninstall" %}</button></form>
  </td>
+ <td class="bottom-button">
+ <a class="btn btn-primary" href="{% url 'addon-logs' addon.id %}">{% trans "View Logs" %}</a>
+ </td>
  <td class="bottom-button">
  {% if addon.addon.has_settings %}
  <a class="btn btn-primary" href="{{ addon.get_absolute_url }}">{% trans "Configure" %}</a>