Improve telemetry with deeper insights into system usage

.changeset/fluffy-zebras-buy.md

@@ -0,0 +1,5 @@
+---
+'@directus/api': patch
+---
+
+Improved telemetry with deeper insights into system usage

api/src/database/helpers/schema/dialects/cockroachdb.ts

@@ -1,6 +1,9 @@
 import type { KNEX_TYPES } from '@directus/constants';
 import type { Options } from '../types.js';
 import { SchemaHelper } from '../types.js';
+import { useEnv } from '@directus/env';
+
+const env = useEnv();
 
 export class SchemaHelperCockroachDb extends SchemaHelper {
  override async changeToType(
@@ -23,4 +26,16 @@ export class SchemaHelperCockroachDb extends SchemaHelper {
  return existingName + suffix;
  }
  }
+
+ override async getDatabaseSize(): Promise<number> {
+ try {
+ const result = await this.knex
+ .select(this.knex.raw('round(SUM(range_size_mb) * 1024 * 1024, 0) AS size'))
+ .from(this.knex.raw('[SHOW RANGES FROM database ??]', [env['DB_DATABASE']]));
+
+ return result[0]?.['size'] ? Number(result[0]?.['size']) : 0;
+ } catch {
+ return 0;
+ }
+ }
 }

api/src/database/helpers/schema/dialects/mssql.ts

@@ -20,4 +20,14 @@ export class SchemaHelperMSSQL extends SchemaHelper {
  override formatUUID(uuid: string): string {
  return uuid.toUpperCase();
  }
+
+ override async getDatabaseSize(): Promise<number> {
+ try {
+ const result = await this.knex.raw('SELECT SUM(size) * 8192 AS size FROM sys.database_files;');
+
+ return result[0]?.['size'] ? Number(result[0]?.['size']) : 0;
+ } catch {
+ return 0;
+ }
+ }
 }

api/src/database/helpers/schema/dialects/mysql.ts

@@ -1,7 +1,10 @@
+import { useEnv } from '@directus/env';
 import type { Knex } from 'knex';
 import { getDatabaseVersion } from '../../../index.js';
 import { SchemaHelper } from '../types.js';
 
+const env = useEnv();
+
 export class SchemaHelperMySQL extends SchemaHelper {
  override applyMultiRelationalSort(
  knex: Knex,
@@ -28,4 +31,22 @@ export class SchemaHelperMySQL extends SchemaHelper {
 
  return super.applyMultiRelationalSort(knex, dbQuery, table, primaryKey, orderByString, orderByFields);
  }
+
+ override async getDatabaseSize(): Promise<number> {
+ try {
+ const result = (await this.knex
+ .sum('size AS size')
+ .from(
+ this.knex
+ .select(this.knex.raw('data_length + index_length AS size'))
+ .from('information_schema.TABLES')
+ .where('table_schema', '=', String(env['DB_DATABASE']))
+ .as('size'),
+ )) as Record<string, any>[];
+
+ return result[0]?.['size'] ? Number(result[0]?.['size']) : 0;
+ } catch {
+ return 0;
+ }
+ }
 }

api/src/database/helpers/schema/dialects/oracle.ts

@@ -40,4 +40,14 @@ export class SchemaHelperOracle extends SchemaHelper {
 
  return field.type;
  }
+
+ override async getDatabaseSize(): Promise<number> {
+ try {
+ const result = await this.knex.raw('select SUM(bytes) from dba_segments');
+
+ return result[0]?.['SUM(BYTES)'] ? Number(result[0]?.['SUM(BYTES)']) : 0;
+ } catch {
+ return 0;
+ }
+ }
 }

api/src/database/helpers/schema/dialects/postgres.ts

@@ -0,0 +1,16 @@
+import { useEnv } from '@directus/env';
+import { SchemaHelper } from '../types.js';
+
+const env = useEnv();
+
+export class SchemaHelperPostgres extends SchemaHelper {
+ override async getDatabaseSize(): Promise<number> {
+ try {
+ const result = await this.knex.select(this.knex.raw(`pg_database_size(?) as size;`, [env['DB_DATABASE']]));
+
+ return result[0]?.['size'] ? Number(result[0]?.['size']) : 0;
+ } catch {
+ return 0;
+ }
+ }
+}

api/src/database/helpers/schema/dialects/sqlite.ts

@@ -14,4 +14,16 @@ export class SchemaHelperSQLite extends SchemaHelper {
  override async postColumnChange(): Promise<void> {
  await this.knex.raw('PRAGMA foreign_keys = ON');
  }
+
+ override async getDatabaseSize(): Promise<number> {
+ try {
+ const result = await this.knex.raw(
+ 'SELECT page_count * page_size as "size" FROM pragma_page_count(), pragma_page_size();',
+ );
+
+ return result[0]?.['size'] ? Number(result[0]?.['size']) : 0;
+ } catch {
+ return 0;
+ }
+ }
 }

api/src/database/helpers/schema/index.ts

@@ -1,7 +1,7 @@
-export { SchemaHelperDefault as postgres } from './dialects/default.js';
 export { SchemaHelperCockroachDb as cockroachdb } from './dialects/cockroachdb.js';
 export { SchemaHelperDefault as redshift } from './dialects/default.js';
+export { SchemaHelperMSSQL as mssql } from './dialects/mssql.js';
+export { SchemaHelperMySQL as mysql } from './dialects/mysql.js';
 export { SchemaHelperOracle as oracle } from './dialects/oracle.js';
+export { SchemaHelperPostgres as postgres } from './dialects/postgres.js';
 export { SchemaHelperSQLite as sqlite } from './dialects/sqlite.js';
-export { SchemaHelperMySQL as mysql } from './dialects/mysql.js';
-export { SchemaHelperMSSQL as mssql } from './dialects/mssql.js';

api/src/database/helpers/schema/types.ts

@@ -139,4 +139,8 @@ export abstract class SchemaHelper extends DatabaseHelper {
  formatUUID(uuid: string): string {
  return uuid; // no-op by default
  }
+
+ async getDatabaseSize(): Promise<number> {
+ return 0;
+ }
 }

api/src/telemetry/lib/get-report.test.ts

@@ -3,13 +3,22 @@ import { version } from 'directus/version';
 import { type Knex } from 'knex';
 import { afterEach, beforeEach, expect, test, vi } from 'vitest';
 import { getDatabase, getDatabaseClient } from '../../database/index.js';
+import { getFieldCount, type FieldCount } from '../utils/get-field-count.js';
 import { getItemCount } from '../utils/get-item-count.js';
 import { getUserCount, type UserCount } from '../utils/get-user-count.js';
 import { getUserItemCount, type UserItemCount } from '../utils/get-user-item-count.js';
 import { getReport } from './get-report.js';
 
 vi.mock('../../database/index.js');
 
+vi.mock('../../database/helpers/index.js', () => ({
+ getHelpers: vi.fn().mockImplementation(() => ({
+ schema: {
+ getDatabaseSize: vi.fn().mockReturnValue(0),
+ },
+ })),
+}));
+
 // This is required because logger uses global env which is imported before the tests run. Can be
 // reduce to just mock the file when logger is also using useLogger everywhere @TODO
 vi.mock('@directus/env', () => ({ useEnv: vi.fn().mockReturnValue({}) }));
@@ -18,11 +27,13 @@ vi.mock('../utils/get-item-count.js');
 vi.mock('../utils/get-storage.js');
 vi.mock('../utils/get-user-item-count.js');
 vi.mock('../utils/get-user-count.js');
+vi.mock('../utils/get-field-count.js');
 
 let mockEnv: Record<string, unknown>;
 let mockDb: Knex;
 let mockUserCounts: UserCount;
 let mockUserItemCounts: UserItemCount;
+let mockFieldCounts: FieldCount;
 
 beforeEach(() => {
  mockEnv = {
@@ -35,12 +46,15 @@ beforeEach(() => {
 
  mockUserItemCounts = { collections: 25, items: 15000 };
 
+ mockFieldCounts = { max: 28, total: 88 };
+
  vi.mocked(useEnv).mockReturnValue(mockEnv);
  vi.mocked(getDatabase).mockReturnValue(mockDb);
 
  vi.mocked(getItemCount).mockResolvedValue({});
  vi.mocked(getUserCount).mockResolvedValue(mockUserCounts);
  vi.mocked(getUserItemCount).mockResolvedValue(mockUserItemCounts);
+ vi.mocked(getFieldCount).mockResolvedValue(mockFieldCounts);
 });
 
 afterEach(() => {
@@ -72,12 +86,12 @@ test('Runs and returns basic counts', async () => {
  const report = await getReport();
 
  expect(getItemCount).toHaveBeenCalledWith(mockDb, [
- 'directus_dashboards',
- 'directus_extensions',
- 'directus_files',
- 'directus_flows',
- 'directus_roles',
- 'directus_shares',
+ { collection: 'directus_dashboards' },
+ { collection: 'directus_extensions', where: ['enabled', '=', true] },
+ { collection: 'directus_files' },
+ { collection: 'directus_flows', where: ['status', '=', 'active'] },
+ { collection: 'directus_roles' },
+ { collection: 'directus_shares' },
  ]);
 
  expect(report.dashboards).toBe(mockItemCount.directus_dashboards);
@@ -106,3 +120,12 @@ test('Runs and returns user item counts', async () => {
  expect(report.collections).toBe(mockUserItemCounts.collections);
  expect(report.items).toBe(mockUserItemCounts.items);
 });
+
+test('Runs and returns field counts', async () => {
+ const report = await getReport();
+
+ expect(getFieldCount).toHaveBeenCalledWith(mockDb);
+
+ expect(report.collections).toBe(mockUserItemCounts.collections);
+ expect(report.items).toBe(mockUserItemCounts.items);
+});

api/src/telemetry/lib/get-report.ts

@@ -2,17 +2,25 @@ import { useEnv } from '@directus/env';
 import { version } from 'directus/version';
 import { getDatabase, getDatabaseClient } from '../../database/index.js';
 import type { TelemetryReport } from '../types/report.js';
+import { getFieldCount } from '../utils/get-field-count.js';
 import { getItemCount } from '../utils/get-item-count.js';
 import { getUserCount } from '../utils/get-user-count.js';
 import { getUserItemCount } from '../utils/get-user-item-count.js';
+import { getHelpers } from '../../database/helpers/index.js';
 
-const basicCountCollections = [
- 'directus_dashboards',
- 'directus_extensions',
- 'directus_files',
- 'directus_flows',
- 'directus_roles',
- 'directus_shares',
+const basicCountTasks = [
+ { collection: 'directus_dashboards' },
+ {
+ collection: 'directus_extensions',
+ where: ['enabled', '=', true],
+ },
+ { collection: 'directus_files' },
+ {
+ collection: 'directus_flows',
+ where: ['status', '=', 'active'],
+ },
+ { collection: 'directus_roles' },
+ { collection: 'directus_shares' },
 ] as const;
 
 /**
@@ -21,11 +29,14 @@ const basicCountCollections = [
 export const getReport = async (): Promise<TelemetryReport> => {
  const db = getDatabase();
  const env = useEnv();
+ const helpers = getHelpers(db);
 
- const [basicCounts, userCounts, userItemCount] = await Promise.all([
- getItemCount(db, basicCountCollections),
+ const [basicCounts, userCounts, userItemCount, fieldsCounts, databaseSize] = await Promise.all([
+ getItemCount(db, basicCountTasks),
  getUserCount(db),
  getUserItemCount(db),
+ getFieldCount(db),
+ helpers.schema.getDatabaseSize(),
  ]);
 
  return {
@@ -46,5 +57,10 @@ export const getReport = async (): Promise<TelemetryReport> => {
 
  collections: userItemCount.collections,
  items: userItemCount.items,
+
+ fields_max: fieldsCounts.max,
+ fields_total: fieldsCounts.total,
+
+ database_size: databaseSize,
  };
 };

api/src/telemetry/types/report.ts

@@ -68,4 +68,19 @@ export interface TelemetryReport {
  * Number of shares in the system
  */
  shares: number;
+
+ /**
+ * Maximum number of fields in a collection
+ */
+ fields_max: number;
+
+ /**
+ * Number of fields in the system
+ */
+ fields_total: number;
+
+ /**
+ * Size of the database
+ */
+ database_size: number;
 }

api/src/telemetry/utils/get-field-count.ts

@@ -0,0 +1,30 @@
+import { type Knex } from 'knex';
+
+export interface FieldCount {
+ max: number;
+ total: number;
+}
+
+export const getFieldCount = async (db: Knex): Promise<FieldCount> => {
+ const counts: FieldCount = {
+ max: 0,
+ total: 0,
+ };
+
+ const result = <{ max: number | string; total: number | string }[]>(
+ await db
+ .max({ max: 'field_count' })
+ .sum({ total: 'field_count' })
+ .from(db.select('collection').count('* as field_count').from('directus_fields').groupBy('collection').as('inner'))
+ );
+
+ if (result[0]?.max) {
+ counts.max = Number(result[0].max);
+ }
+
+ if (result[0]?.total) {
+ counts.total = Number(result[0].total);
+ }
+
+ return counts;
+};