TryGhost · ronaldlangeveld · May 2, 2024 · May 2, 2024 · May 6, 2024 · May 6, 2024
diff --git a/.../data/migrations/versions/5.83/2024-05-02-06-01-13-add-latest-event-timestamp-to-email.js b/.../data/migrations/versions/5.83/2024-05-02-06-01-13-add-latest-event-timestamp-to-email.js
@@ -0,0 +1,7 @@
+// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253
+
+const {createAddColumnMigration} = require('../../utils');
+
+module.exports = createAddColumnMigration('emails', 'latest_event_timestamp', {
+ type: 'dateTime', nullable: true
+});
diff --git a/ghost/core/core/server/data/schema/schema.js b/ghost/core/core/server/data/schema/schema.js
@@ -839,7 +839,8 @@ module.exports = {
  created_at: {type: 'dateTime', nullable: false},
  created_by: {type: 'string', maxlength: 24, nullable: false},
  updated_at: {type: 'dateTime', nullable: true},
- updated_by: {type: 'string', maxlength: 24, nullable: true}
+ updated_by: {type: 'string', maxlength: 24, nullable: true},
+ latest_event_timestamp: {type: 'dateTime', nullable: true}
  },
  email_batches: {
  id: {type: 'string', maxlength: 24, nullable: false, primary: true},

diff --git a/ghost/core/core/server/models/email.js b/ghost/core/core/server/models/email.js
@@ -15,7 +15,8 @@ const Email = ghostBookshelf.Model.extend({
  delivered_count: 0,
  opened_count: 0,
  failed_count: 0,
- source_type: 'html'
+ source_type: 'html',
+ latest_event_timestamp: null
  };
  },
 

diff --git a/ghost/core/core/server/services/email-analytics/lib/queries.js b/ghost/core/core/server/services/email-analytics/lib/queries.js
@@ -1,4 +1,3 @@
-const _ = require('lodash');
 const debug = require('@tryghost/debug')('services:email-analytics');
 const db = require('../../../data/db');
 
@@ -14,30 +13,36 @@ module.exports = {
  async getLastSeenEventTimestamp() {
  const startDate = new Date();
 
- // three separate queries is much faster than using max/greatest (with coalesce to handle nulls) across columns
- let {maxDeliveredAt} = await db.knex('email_recipients').select(db.knex.raw('MAX(delivered_at) as maxDeliveredAt')).first() || {};
- let {maxOpenedAt} = await db.knex('email_recipients').select(db.knex.raw('MAX(opened_at) as maxOpenedAt')).first() || {};
- let {maxFailedAt} = await db.knex('email_recipients').select(db.knex.raw('MAX(failed_at) as maxFailedAt')).first() || {};
-
- if (maxDeliveredAt && !(maxDeliveredAt instanceof Date)) {
- // SQLite returns a string instead of a Date
- maxDeliveredAt = new Date(maxDeliveredAt);
- }
-
- if (maxOpenedAt && !(maxOpenedAt instanceof Date)) {
- // SQLite returns a string instead of a Date
- maxOpenedAt = new Date(maxOpenedAt);
+ // First, try to fetch the max latest_event_timestamp
+ let {maxTimestamp} = await db.knex('emails').select(db.knex.raw('MAX(latest_event_timestamp) as maxTimestamp')).first() || {};
+
+ if (maxTimestamp) {
+ if (!(maxTimestamp instanceof Date)) {
+ maxTimestamp = new Date(maxTimestamp);
+ }
+ } else {
+ let {maxDeliveredAt} = await db.knex('email_recipients').select(db.knex.raw('MAX(delivered_at) as maxDeliveredAt')).first() || {};
+ let {maxOpenedAt} = await db.knex('email_recipients').select(db.knex.raw('MAX(opened_at) as maxOpenedAt')).first() || {};
+ let {maxFailedAt} = await db.knex('email_recipients').select(db.knex.raw('MAX(failed_at) as maxFailedAt')).first() || {};
+
+ if (maxDeliveredAt && !(maxDeliveredAt instanceof Date)) {
+ // SQLite returns a string instead of a Date
+ maxDeliveredAt = new Date(maxDeliveredAt);
+ }
+
+ if (maxOpenedAt && !(maxOpenedAt instanceof Date)) {
+ // SQLite returns a string instead of a Date
+ maxOpenedAt = new Date(maxOpenedAt);
+ }
+
+ if (maxFailedAt && !(maxFailedAt instanceof Date)) {
+ // SQLite returns a string instead of a Date
+ maxFailedAt = new Date(maxFailedAt);
+ }
  }
 
- if (maxFailedAt && !(maxFailedAt instanceof Date)) {
- // SQLite returns a string instead of a Date
- maxFailedAt = new Date(maxFailedAt);
- }
-
- const lastSeenEventTimestamp = _.max([maxDeliveredAt, maxOpenedAt, maxFailedAt]);
  debug(`getLastSeenEventTimestamp: finished in ${Date.now() - startDate}ms`);
-
- return lastSeenEventTimestamp;
+ return maxTimestamp;
  },
 
  async aggregateEmailStats(emailId) {

diff --git a/ghost/core/test/e2e-api/admin/__snapshots__/activity-feed.test.js.snap b/ghost/core/test/e2e-api/admin/__snapshots__/activity-feed.test.js.snap
@@ -22699,7 +22699,7 @@ exports[`Activity Feed API Can filter events by post id 2: [headers] 1`] = `
 Object {
  "access-control-allow-origin": "http://127.0.0.1:2369",
  "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
- "content-length": "17559",
+ "content-length": "17739",
  "content-type": "application/json; charset=utf-8",
  "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
  "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
@@ -23852,7 +23852,7 @@ exports[`Activity Feed API Returns email delivered events in activity feed 2: [h
 Object {
  "access-control-allow-origin": "http://127.0.0.1:2369",
  "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
- "content-length": "1051",
+ "content-length": "1081",
  "content-type": "application/json; charset=utf-8",
  "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
  "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
@@ -23886,7 +23886,7 @@ exports[`Activity Feed API Returns email opened events in activity feed 2: [head
 Object {
  "access-control-allow-origin": "http://127.0.0.1:2369",
  "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
- "content-length": "1048",
+ "content-length": "1078",
  "content-type": "application/json; charset=utf-8",
  "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
  "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
@@ -23944,7 +23944,7 @@ exports[`Activity Feed API Returns email sent events in activity feed 2: [header
 Object {
  "access-control-allow-origin": "http://127.0.0.1:2369",
  "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
- "content-length": "3860",
+ "content-length": "3980",
  "content-type": "application/json; charset=utf-8",
  "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
  "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,

diff --git a/ghost/core/test/e2e-api/admin/__snapshots__/emails.test.js.snap b/ghost/core/test/e2e-api/admin/__snapshots__/emails.test.js.snap
@@ -499,6 +499,7 @@ Object {
  "from": null,
  "html": "<p>Look! I'm an email</p>",
  "id": StringMatching /\\[a-f0-9\\]\\{24\\}/,
+ "latest_event_timestamp": null,
  "newsletter_id": null,
  "opened_count": 1,
  "plaintext": "Waba-daba-dab-da",
@@ -526,6 +527,7 @@ Object {
  "from": null,
  "html": "<p>What's that? Another email!</p>",
  "id": StringMatching /\\[a-f0-9\\]\\{24\\}/,
+ "latest_event_timestamp": null,
  "newsletter_id": null,
  "opened_count": 0,
  "plaintext": "yes this is an email",
@@ -560,7 +562,7 @@ exports[`Emails API Can browse emails 2: [headers] 1`] = `
 Object {
  "access-control-allow-origin": "http://127.0.0.1:2369",
  "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
- "content-length": "1405",
+ "content-length": "1465",
  "content-type": "application/json; charset=utf-8",
  "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
  "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
@@ -698,6 +700,7 @@ Object {
  "from": null,
  "html": "<p>Look! I'm an email</p>",
  "id": StringMatching /\\[a-f0-9\\]\\{24\\}/,
+ "latest_event_timestamp": null,
  "newsletter_id": null,
  "opened_count": 1,
  "plaintext": "Waba-daba-dab-da",
@@ -722,7 +725,7 @@ exports[`Emails API Can read an email 2: [headers] 1`] = `
 Object {
  "access-control-allow-origin": "http://127.0.0.1:2369",
  "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
- "content-length": "642",
+ "content-length": "672",
  "content-type": "application/json; charset=utf-8",
  "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
  "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
@@ -745,6 +748,7 @@ Object {
  "from": null,
  "html": "<p>What's that? Another email!</p>",
  "id": StringMatching /\\[a-f0-9\\]\\{24\\}/,
+ "latest_event_timestamp": null,
  "newsletter_id": null,
  "opened_count": 0,
  "plaintext": "yes this is an email",
@@ -769,7 +773,7 @@ exports[`Emails API Can retry a failed email 2: [headers] 1`] = `
 Object {
  "access-control-allow-origin": "http://127.0.0.1:2369",
  "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
- "content-length": "688",
+ "content-length": "718",
  "content-type": "application/json; charset=utf-8",
  "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
  "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
@@ -792,6 +796,7 @@ Object {
  "from": "[email protected]",
  "html": "<p style=\\"margin: 0 0 1.5em 0; line-height: 1.6em;\\">Hey Jamie, Hey Jamie,</p><a href=\\"http://127.0.0.1:2369/unsubscribe/?uuid=example-uuid&newsletter=preview\\">Unsubscribe</a>",
  "id": StringMatching /\\[a-f0-9\\]\\{24\\}/,
+ "latest_event_timestamp": null,
  "newsletter_id": StringMatching /\\[a-f0-9\\]\\{24\\}/,
  "opened_count": 0,
  "plaintext": "Hey Jamie, Hey Jamie
@@ -817,7 +822,7 @@ exports[`Emails API Does default replacements on the HTML body of an old email 2
 Object {
  "access-control-allow-origin": "http://127.0.0.1:2369",
  "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
- "content-length": "923",
+ "content-length": "953",
  "content-type": "application/json; charset=utf-8",
  "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
  "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,

diff --git a/ghost/core/test/e2e-api/admin/__snapshots__/members.test.js.snap b/ghost/core/test/e2e-api/admin/__snapshots__/members.test.js.snap
@@ -3217,6 +3217,68 @@ Object {
 }
 `;
 
+exports[`Members API Can filter by conversion attribution 1: [body] 1`] = `
+Object {
+ "members": Array [
+ Object {
+ "avatar_image": null,
+ "comped": false,
+ "created_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000Z/,
+ "email": "[email protected]",
+ "email_count": 0,
+ "email_open_rate": null,
+ "email_opened_count": 0,
+ "email_suppression": Object {
+ "info": null,
+ "suppressed": false,
+ },
+ "geolocation": null,
+ "id": StringMatching /\\[a-f0-9\\]\\{24\\}/,
+ "labels": Any<Array>,
+ "last_seen_at": null,
+ "name": "Mr Egg",
+ "newsletters": Array [
+ Object {
+ "description": null,
+ "id": StringMatching /\\[a-f0-9\\]\\{24\\}/,
+ "name": "Daily newsletter",
+ "status": "active",
+ },
+ ],
+ "note": null,
+ "status": "free",
+ "subscribed": true,
+ "subscriptions": Any<Array>,
+ "updated_at": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}T\\\\d\\{2\\}:\\\\d\\{2\\}:\\\\d\\{2\\}\\\\\\.000Z/,
+ "uuid": StringMatching /\\[a-f0-9\\]\\{8\\}-\\[a-f0-9\\]\\{4\\}-\\[a-f0-9\\]\\{4\\}-\\[a-f0-9\\]\\{4\\}-\\[a-f0-9\\]\\{12\\}/,
+ },
+ ],
+ "meta": Object {
+ "pagination": Object {
+ "limit": 15,
+ "next": null,
+ "page": 1,
+ "pages": 1,
+ "prev": null,
+ "total": 1,
+ },
+ },
+}
+`;
+
+exports[`Members API Can filter by conversion attribution 2: [headers] 1`] = `
+Object {
+ "access-control-allow-origin": "http://127.0.0.1:2369",
+ "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
+ "content-length": "830",
+ "content-type": "application/json; charset=utf-8",
+ "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
+ "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
+ "vary": "Accept-Version, Origin, Accept-Encoding",
+ "x-powered-by": "Express",
+}
+`;
+
 exports[`Members API Can filter by paid status 1: [body] 1`] = `
 Object {
  "members": Array [

diff --git a/ghost/core/test/integration/services/email-service/email-event-storage.test.js b/ghost/core/test/integration/services/email-service/email-event-storage.test.js
@@ -1135,4 +1135,107 @@ describe('EmailEventStorage', function () {
  const result = await emailAnalytics.fetchLatest();
  assert.equal(result, 0);
  });
+
+ it('populates latest_event_timestamp in email when handleDelivered fires', async function () {
+ const emailBatch = fixtureManager.get('email_batches', 0);
+ const emailId = emailBatch.email_id;
+
+ const emailRecipient = fixtureManager.get('email_recipients', 0);
+ assert(emailRecipient.batch_id === emailBatch.id);
+ const providerId = emailBatch.provider_id;
+ const timestamp = new Date(2000, 0, 1);
+
+ events = [{
+ event: 'delivered',
+ recipient: emailRecipient.member_email,
+ 'user-variables': {
+ 'email-id': emailId
+ },
+ message: {
+ headers: {
+ 'message-id': providerId
+ }
+ },
+ // unix timestamp
+ timestamp: Math.round(timestamp.getTime() / 1000)
+ }];
+
+ const result = await emailAnalytics.fetchLatest();
+ assert.equal(result, 1);
+
+ await DomainEvents.allSettled();
+
+ const updatedEmail = await models.Email.findOne({
+ id: emailId
+ }, {require: true});
+
+ assert.equal(updatedEmail.get('latest_event_timestamp').toUTCString(), timestamp.toUTCString());
+ });
+
+ it('populates latest_event_timestamp in email when handleOpened fires', async function () {
+ const emailBatch = fixtureManager.get('email_batches', 0);
+ const emailId = emailBatch.email_id;
+
+ const emailRecipient = fixtureManager.get('email_recipients', 0);
+ assert(emailRecipient.batch_id === emailBatch.id);
+ const providerId = emailBatch.provider_id;
+ const timestamp = new Date(2000, 0, 1);
+
+ events = [{
+ event: 'opened',
+ recipient: emailRecipient.member_email,
+ 'user-variables': {
+ 'email-id': emailId
+ },
+ message: {
+ headers: {
+ 'message-id': providerId
+ }
+ },
+ // unix timestamp
+ timestamp: Math.round(timestamp.getTime() / 1000)
+ }];
+
+ const result = await emailAnalytics.fetchLatest();
+ assert.equal(result, 1);
+
+ await DomainEvents.allSettled();
+
+ const updatedEmail = await models.Email.findOne({
+ id: emailId
+ }, {require: true});
+
+ assert.equal(updatedEmail.get('latest_event_timestamp').toUTCString(), timestamp.toUTCString());
+ });
+
+ it('populates latest_event_timestamp in email when handlePermanentFailed fires', async function () {
+ const emailBatch = fixtureManager.get('email_batches', 0);
+ const emailId = emailBatch.email_id;
+
+ const emailRecipient = fixtureManager.get('email_recipients', 0);
+ assert(emailRecipient.batch_id === emailBatch.id);
+ const timestamp = new Date(2000, 0, 1);
+
+ events = [{
+ event: 'failed',
+ severity: 'permanent',
+ recipient: emailRecipient.member_email,
+ 'user-variables': {
+ 'email-id': emailId
+ },
+ // unix timestamp
+ timestamp: Math.round(timestamp.getTime() / 1000)
+ }];
+
+ const result = await emailAnalytics.fetchLatest();
+ assert.equal(result, 1);
+
+ await DomainEvents.allSettled();
+
+ const updatedEmail = await models.Email.findOne({
+ id: emailId
+ }, {require: true});
+
+ assert.equal(updatedEmail.get('latest_event_timestamp').toUTCString(), timestamp.toUTCString());
+ });
 });
diff --git a/ghost/core/test/unit/server/data/schema/integrity.test.js b/ghost/core/test/unit/server/data/schema/integrity.test.js
@@ -35,7 +35,7 @@ const validateRouteSettings = require('../../../../../core/server/services/route
  */
 describe('DB version integrity', function () {
  // Only these variables should need updating
- const currentSchemaHash = 'ccf3893bc3f8930f0d1188e646abda6d';
+ const currentSchemaHash = '7495ca1ca1127247a2638a19c0c59b8e';
  const currentFixturesHash = 'a489d615989eab1023d4b8af0ecee7fd';
  const currentSettingsHash = '5c957ceb48c4878767d7d3db484c592d';
  const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';