Merge pull request #2050 from caprover/automated-cleanup

API skeleton added for automated cleanup

package.json

@@ -19,6 +19,7 @@
  "body-parser": "^1.19.0",
  "configstore": "^5.0.1",
  "cookie-parser": "~1.4.5",
+ "cron": "^3.1.7",
  "debug": "~4.3.2",
  "dockerode": "^3.3.0",
  "ejs": "^3.1.6",

src/datastore/DataStore.ts

@@ -3,6 +3,10 @@
  */
 import Configstore = require('configstore')
 import fs = require('fs-extra')
+import {
+ AutomatedCleanupConfigsCleaner,
+ IAutomatedCleanupConfigs,
+} from '../models/AutomatedCleanupConfigs'
 import CaptainConstants from '../utils/CaptainConstants'
 import CaptainEncryptor from '../utils/Encryptor'
 import AppsDataStore from './AppsDataStore'
@@ -22,6 +26,7 @@ const NGINX_BASE_CONFIG = 'nginxBaseConfig'
 const NGINX_CAPTAIN_CONFIG = 'nginxCaptainConfig'
 const CUSTOM_ONE_CLICK_APP_URLS = 'oneClickAppUrls'
 const FEATURE_FLAGS = 'featureFlags'
+const AUTOMATED_CLEANUP = 'automatedCleanup'
 
 const DEFAULT_CAPTAIN_ROOT_DOMAIN = 'captain.localhost'
 
@@ -107,6 +112,30 @@ class DataStore {
  })
  }
 
+ setDiskCleanupConfigs(configs: IAutomatedCleanupConfigs) {
+ const self = this
+ return Promise.resolve().then(function () {
+ return self.data.set(
+ AUTOMATED_CLEANUP,
+ AutomatedCleanupConfigsCleaner.cleanup(configs)
+ )
+ })
+ }
+
+ getDiskCleanupConfigs(): Promise<IAutomatedCleanupConfigs> {
+ const self = this
+ return Promise.resolve().then(function () {
+ return (
+ self.data.get(AUTOMATED_CLEANUP) ||
+ AutomatedCleanupConfigsCleaner.cleanup({
+ mostRecentLimit: 0,
+ cronSchedule: '',
+ timezone: '',
+ })
+ )
+ })
+ }
+
  /*
  "smtp": {
  "to": "",

src/models/AutomatedCleanupConfigs.ts

@@ -0,0 +1,18 @@
+export interface IAutomatedCleanupConfigs {
+ mostRecentLimit: number
+ cronSchedule: string
+ timezone: string
+}
+
+export class AutomatedCleanupConfigsCleaner {
+ static cleanup(instance: IAutomatedCleanupConfigs) {
+ return {
+ mostRecentLimit:
+ Number(instance.mostRecentLimit) > 0
+ ? Number(instance.mostRecentLimit)
+ : 1,
+ cronSchedule: `${instance.cronSchedule || ''}`.trim(),
+ timezone: `${instance.timezone || ''}`.trim() || 'UTC',
+ }
+ }
+}

src/routes/user/apps/appdefinition/AppDefinitionRouter.ts

@@ -3,6 +3,7 @@ import ApiStatusCodes from '../../../../api/ApiStatusCodes'
 import BaseApi from '../../../../api/BaseApi'
 import InjectionExtractor from '../../../../injection/InjectionExtractor'
 import { CaptainError } from '../../../../models/OtherTypes'
+import CaptainManager from '../../../../user/system/CaptainManager'
 import CaptainConstants from '../../../../utils/CaptainConstants'
 import Logger from '../../../../utils/Logger'
 import Utils from '../../../../utils/Utils'
@@ -18,13 +19,12 @@ const DEFAULT_APP_CAPTAIN_DEFINITION = JSON.stringify({
 
 // unused images
 router.get('/unusedImages', function (req, res, next) {
- const serviceManager =
- InjectionExtractor.extractUserFromInjected(res).user.serviceManager
-
  Promise.resolve()
  .then(function () {
  const mostRecentLimit = Number(req.query.mostRecentLimit || '0')
- return serviceManager.getUnusedImages(mostRecentLimit)
+ return CaptainManager.get()
+ .getDiskCleanupManager()
+ .getUnusedImages(mostRecentLimit)
  })
  .then(function (unusedImages) {
  const baseApi = new BaseApi(
@@ -41,13 +41,13 @@ router.get('/unusedImages', function (req, res, next) {
 
 // delete images
 router.post('/deleteImages', function (req, res, next) {
- const serviceManager =
- InjectionExtractor.extractUserFromInjected(res).user.serviceManager
  const imageIds = req.body.imageIds || []
 
  Promise.resolve()
  .then(function () {
- return serviceManager.deleteImages(imageIds)
+ return CaptainManager.get()
+ .getDiskCleanupManager()
+ .deleteImages(imageIds)
  })
  .then(function () {
  const baseApi = new BaseApi(

src/routes/user/system/SystemRouter.ts

@@ -5,6 +5,7 @@ import BaseApi from '../../../api/BaseApi'
 import DockerApi from '../../../docker/DockerApi'
 import DockerUtils from '../../../docker/DockerUtils'
 import InjectionExtractor from '../../../injection/InjectionExtractor'
+import { AutomatedCleanupConfigsCleaner } from '../../../models/AutomatedCleanupConfigs'
 import CaptainManager from '../../../user/system/CaptainManager'
 import VersionManager from '../../../user/system/VersionManager'
 import CaptainConstants from '../../../utils/CaptainConstants'
@@ -203,6 +204,44 @@ router.post('/versionInfo/', function (req, res, next) {
  .catch(ApiStatusCodes.createCatcher(res))
 })
 
+router.get('/diskcleanup/', function (req, res, next) {
+ return Promise.resolve()
+ .then(function () {
+ return CaptainManager.get().getDiskCleanupManager().getConfigs()
+ })
+ .then(function (data) {
+ const baseApi = new BaseApi(
+ ApiStatusCodes.STATUS_OK,
+ 'Disk cleanup configs retrieved'
+ )
+ baseApi.data = data
+ res.send(baseApi)
+ })
+ .catch(ApiStatusCodes.createCatcher(res))
+})
+
+router.post('/diskcleanup/', function (req, res, next) {
+ const configs = AutomatedCleanupConfigsCleaner.cleanup({
+ mostRecentLimit: req.body.mostRecentLimit,
+ cronSchedule: req.body.cronSchedule,
+ timezone: req.body.timezone,
+ })
+ return Promise.resolve()
+ .then(function () {
+ return CaptainManager.get()
+ .getDiskCleanupManager()
+ .setConfig(configs)
+ })
+ .then(function () {
+ const baseApi = new BaseApi(
+ ApiStatusCodes.STATUS_OK,
+ 'Disk cleanup configs updated'
+ )
+ res.send(baseApi)
+ })
+ .catch(ApiStatusCodes.createCatcher(res))
+})
+
 router.get('/netdata/', function (req, res, next) {
  const dataStore =
  InjectionExtractor.extractUserFromInjected(res).user.dataStore

src/user/ServiceManager.ts

@@ -1,4 +1,3 @@
-import { ImageInfo } from 'dockerode'
 import ApiStatusCodes from '../api/ApiStatusCodes'
 import DataStore from '../datastore/DataStore'
 import DockerApi, { IDockerUpdateOrders } from '../docker/DockerApi'
@@ -549,85 +548,6 @@ class ServiceManager {
  })
  }
 
- getUnusedImages(mostRecentLimit: number) {
- Logger.d(
- `Getting unused images, excluding most recent ones: ${mostRecentLimit}`
- )
-
- const dockerApi = this.dockerApi
- const dataStore = this.dataStore
- let allImages: ImageInfo[]
-
- return Promise.resolve()
- .then(function () {
- return dockerApi.getImages()
- })
- .then(function (images) {
- allImages = images
-
- return dataStore.getAppsDataStore().getAppDefinitions()
- })
- .then(function (apps) {
- const unusedImages = []
-
- if (mostRecentLimit < 0) {
- throw ApiStatusCodes.createError(
- ApiStatusCodes.ILLEGAL_PARAMETER,
- 'Most Recent Limit cannot be negative'
- )
- }
-
- for (let i = 0; i < allImages.length; i++) {
- const currentImage = allImages[i]
- let imageInUse = false
-
- const repoTags = currentImage.RepoTags || []
-
- Object.keys(apps).forEach(function (appName) {
- const app = apps[appName]
- for (let k = 0; k < mostRecentLimit + 1; k++) {
- const versionToCheck =
- Number(app.deployedVersion) - k
-
- if (versionToCheck < 0) continue
-
- let deployedImage = ''
- app.versions.forEach((v) => {
- if (v.version === versionToCheck) {
- deployedImage = v.deployedImageName || ''
- }
- })
-
- if (!deployedImage) continue
-
- if (repoTags.indexOf(deployedImage) >= 0) {
- imageInUse = true
- }
- }
- })
-
- if (!imageInUse) {
- unusedImages.push({
- id: currentImage.Id,
- tags: repoTags,
- })
- }
- }
-
- return unusedImages
- })
- }
-
- deleteImages(imageIds: string[]) {
- Logger.d('Deleting images...')
-
- const dockerApi = this.dockerApi
-
- return Promise.resolve().then(function () {
- return dockerApi.deleteImages(imageIds)
- })
- }
-
  createPreDeployFunctionIfExist(
  app: IAppDef
  ): PreDeployFunction | undefined {

src/user/system/CaptainManager.ts

@@ -19,6 +19,7 @@ import {
 import ProManager from '../pro/ProManager'
 import BackupManager from './BackupManager'
 import CertbotManager from './CertbotManager'
+import DiskCleanupManager from './DiskCleanupManager'
 import DomainResolveChecker from './DomainResolveChecker'
 import LoadBalancerManager from './LoadBalancerManager'
 import SelfHostedDockerRegistry from './SelfHostedDockerRegistry'
@@ -41,6 +42,7 @@ class CaptainManager {
  private certbotManager: CertbotManager
  private loadBalancerManager: LoadBalancerManager
  private domainResolveChecker: DomainResolveChecker
+ private diskCleanupManager: DiskCleanupManager
  private dockerRegistry: SelfHostedDockerRegistry
  private backupManager: BackupManager
  private myNodeId: string | undefined
@@ -68,6 +70,10 @@ class CaptainManager {
  this.loadBalancerManager,
  this.certbotManager
  )
+ this.diskCleanupManager = new DiskCleanupManager(
+ this.dataStore,
+ dockerApi
+ )
  this.myNodeId = undefined
  this.inited = false
  this.waitUntilRestarted = false
@@ -251,6 +257,9 @@ class CaptainManager {
  }
  )
  })
+ .then(function () {
+ return self.diskCleanupManager.init()
+ })
  .then(function () {
  self.inited = true
 
@@ -456,6 +465,10 @@ class CaptainManager {
  return this.certbotManager
  }
 
+ getDiskCleanupManager() {
+ return this.diskCleanupManager
+ }
+
  isInitialized() {
  return (
  this.inited &&