master: Add an option to janitor to delete logs for specific builders.

common/code_spelling_ignore_words.txt

@@ -1467,6 +1467,7 @@ thursday
 tid
 timedelta
 timestamp
+timestamps
 timezone
 timezones
 tld

master/buildbot/configurators/janitor.py

@@ -17,6 +17,7 @@
 
 from twisted.internet import defer
 
+from buildbot import config as bbconfig
 from buildbot.config import BuilderConfig
 from buildbot.configurators import ConfiguratorBase
 from buildbot.process.buildstep import BuildStep
@@ -39,43 +40,97 @@ class LogChunksJanitor(BuildStep):
  name = 'LogChunksJanitor'
  renderables = ["logHorizon"]
 
- def __init__(self, logHorizon):
+ def __init__(self, logHorizon=None, horizon_per_builder=None):
  super().__init__()
  self.logHorizon = logHorizon
+ self.horizon_per_builder = horizon_per_builder
 
  @defer.inlineCallbacks
  def run(self):
- older_than_timestamp = datetime2epoch(now() - self.logHorizon)
- deleted = yield self.master.db.logs.deleteOldLogChunks(older_than_timestamp)
- self.descriptionDone = ["deleted", str(deleted), "logchunks"]
+ if self.logHorizon is not None:
+ older_than_timestamp = datetime2epoch(now() - self.logHorizon)
+ deleted = yield self.master.db.logs.deleteOldLogChunks(
+ older_than_timestamp=older_than_timestamp)
+ self.descriptionDone = ["deleted", str(deleted), "logchunks"]
+
+ if self.horizon_per_builder is not None:
+ deleted = yield self.master.db.logs.deleteOldLogChunks(
+ horizon_per_builder=self.horizon_per_builder)
+ self.descriptionDone = ["deleted", str(deleted), "logchunks"]
+
  return SUCCESS
 
 
 class BuildDataJanitor(BuildStep):
  name = 'BuildDataJanitor'
  renderables = ["build_data_horizon"]
 
- def __init__(self, build_data_horizon):
+ def __init__(self, build_data_horizon=None, horizon_per_builder=None):
  super().__init__()
  self.build_data_horizon = build_data_horizon
+ self.horizon_per_builder = horizon_per_builder
 
  @defer.inlineCallbacks
  def run(self):
- older_than_timestamp = datetime2epoch(now() - self.build_data_horizon)
- deleted = yield self.master.db.build_data.deleteOldBuildData(older_than_timestamp)
+ if self.build_data_horizon is not None:
+ older_than_timestamp = datetime2epoch(now() - self.build_data_horizon)
+ deleted = yield self.master.db.build_data.deleteOldBuildData(
+ older_than_timestamp=older_than_timestamp)
+
+ if self.horizon_per_builder is not None:
+ deleted = yield self.master.db.build_data.deleteOldBuildData(
+ horizon_per_builder=self.horizon_per_builder)
+
  self.descriptionDone = ["deleted", str(deleted), "build data key-value pairs"]
  return SUCCESS
 
 
+class BuildsJanitor(BuildStep):
+ name = 'BuildsJanitor'
+
+ def __init__(self, horizon_per_builder=None):
+ super().__init__()
+ self.horizon_per_builder = horizon_per_builder
+
+ @defer.inlineCallbacks
+ def run(self):
+ if self.horizon_per_builder is not None:
+ deleted = yield self.master.db.builds.deleteOldBuilds(
+ horizon_per_builder=self.horizon_per_builder)
+
+ self.descriptionDone = ["deleted", str(deleted), "builds"]
+ return SUCCESS
+
+
+class BuildersJanitor(BuildStep):
+ name = 'BuildJanitor'
+
+ @defer.inlineCallbacks
+ def run(self):
+ deleted = yield self.master.db.builders.deleteOldBuilders()
+ self.descriptionDone = ["deleted", str(deleted), "builders"]
+ return SUCCESS
+
+
 class JanitorConfigurator(ConfiguratorBase):
  """ Janitor is a configurator which create a Janitor Builder with all needed Janitor steps"""
 
- def __init__(self, logHorizon=None, hour=0, build_data_horizon=None, **kwargs):
+ def __init__(self,
+ logHorizon=None,
+ hour=0,
+ build_data_horizon=None,
+ horizon_per_builder=None,
+ **kwargs):
  super().__init__()
  self.logHorizon = logHorizon
  self.build_data_horizon = build_data_horizon
+ self.horizon_per_builder = horizon_per_builder
  self.hour = hour
  self.kwargs = kwargs
+ if ((self.logHorizon is not None or self.build_data_horizon is not None)
+ and self.horizon_per_builder is not None):
+ bbconfig.error("JanitorConfigurator: horizon_per_builder only " +
+ "possible without logHorizon and build_data_horizon set.")
 
  def configure(self, config_dict):
  steps = []
@@ -84,6 +139,12 @@ def configure(self, config_dict):
  if self.build_data_horizon is not None:
  steps.append(BuildDataJanitor(build_data_horizon=self.build_data_horizon))
 
+ if self.horizon_per_builder is not None:
+ steps.append(LogChunksJanitor(horizon_per_builder=self.horizon_per_builder))
+ steps.append(BuildDataJanitor(horizon_per_builder=self.horizon_per_builder))
+ steps.append(BuildsJanitor(horizon_per_builder=self.horizon_per_builder))
+ steps.append(BuildersJanitor())
+
  if not steps:
  return
 

master/buildbot/db/build_data.py

@@ -13,12 +13,15 @@
 #
 # Copyright Buildbot Team Members
 
+from datetime import datetime
+
 import sqlalchemy as sa
 
 from twisted.internet import defer
 
 from buildbot.db import NULL
 from buildbot.db import base
+from buildbot.util import datetime2epoch
 
 
 class BuildDataDict(dict):
@@ -123,9 +126,10 @@ def thd(conn):
  return res
 
  @defer.inlineCallbacks
- def deleteOldBuildData(self, older_than_timestamp):
+ def deleteOldBuildData(self, older_than_timestamp=None, horizon_per_builder=None):
  build_data = self.db.model.build_data
  builds = self.db.model.builds
+ builders = self.db.model.builders
 
  def count_build_datum(conn):
  res = conn.execute(sa.select([sa.func.count(build_data.c.id)]))
@@ -136,22 +140,50 @@ def count_build_datum(conn):
  def thd(conn):
  count_before = count_build_datum(conn)
 
- if self.db._engine.dialect.name == 'sqlite':
- # sqlite does not support delete with a join, so for this case we use a subquery,
- # which is much slower
-
- q = sa.select([builds.c.id])
- q = q.where((builds.c.complete_at >= older_than_timestamp) |
- (builds.c.complete_at == NULL))
-
- q = build_data.delete().where(build_data.c.buildid.notin_(q))
+ if horizon_per_builder is not None:
+ for builderName in horizon_per_builder:
+ if "buildDataHorizon" in horizon_per_builder[builderName]:
+ older_than_timestamp_ = datetime2epoch(datetime.now() -
+ horizon_per_builder[builderName]["buildDataHorizon"])
+ if self.db._engine.dialect.name == 'sqlite':
+ # sqlite does not support delete with a join,
+ # so for this case we use a subquery,
+ # which is much slower
+
+ q = sa.select([builds.c.id])
+ q = q.where(sa.and_((builds.c.complete_at < older_than_timestamp_),
+ builds.c.builderid == builders.c.id,
+ builders.c.name.like(builderName)))
+
+ q = build_data.delete().where(build_data.c.buildid.in_(q))
+
+ else:
+ q = build_data.delete()
+ q = q.where(sa.and_(builds.c.id == build_data.c.buildid,
+ builds.c.builderid == builders.c.id,
+ builds.c.complete_at < older_than_timestamp_,
+ builders.c.name.like(builderName)))
+
+ res = conn.execute(q)
+ res.close()
  else:
- q = build_data.delete()
- q = q.where(builds.c.id == build_data.c.buildid)
- q = q.where((builds.c.complete_at >= older_than_timestamp) |
- (builds.c.complete_at == NULL))
- res = conn.execute(q)
- res.close()
+ if self.db._engine.dialect.name == 'sqlite':
+ # sqlite does not support delete with a join,
+ # so for this case we use a subquery,
+ # which is much slower
+
+ q = sa.select([builds.c.id])
+ q = q.where((builds.c.complete_at >= older_than_timestamp) |
+ (builds.c.complete_at == NULL))
+
+ q = build_data.delete().where(build_data.c.buildid.notin_(q))
+ else:
+ q = build_data.delete()
+ q = q.where(builds.c.id == build_data.c.buildid)
+ q = q.where((builds.c.complete_at >= older_than_timestamp) |
+ (builds.c.complete_at == NULL))
+ res = conn.execute(q)
+ res.close()
 
  count_after = count_build_datum(conn)
  return count_before - count_after

master/buildbot/db/builders.py

@@ -149,3 +149,32 @@ def thd(conn):
  last['masterids'].append(row['masterid'])
  return rv
  return self.db.pool.do(thd)
+
+ # returns a Deferred that returns a value
+ def deleteOldBuilders(self):
+ builders = self.db.model.builders
+ builds = self.db.model.builds
+ builder_masters = self.db.model.builder_masters
+
+ def countBuilders(conn):
+ res = conn.execute(sa.select([sa.func.count(builders.c.id)]))
+ count = res.fetchone()[0]
+ res.close()
+ return count
+
+ def thdDeleteOldBuilders(conn):
+ count_before = countBuilders(conn)
+
+ q1 = sa.select([builder_masters.c.builderid])
+ q2 = sa.select([builds.c.builderid])
+ q = q1.union(q2)
+ q = builders.delete().where(builders.c.id.notin_(q))
+
+ res = conn.execute(q)
+ res.close()
+
+ count_after = countBuilders(conn)
+
+ return count_before - count_after
+
+ return self.db.pool.do(thdDeleteOldBuilders)

master/buildbot/db/builds.py

@@ -23,6 +23,8 @@
 from buildbot.db import NULL
 from buildbot.db import base
 from buildbot.util import epoch2datetime
+from buildbot.util import datetime2epoch
+from datetime import datetime
 
 
 class BuildsConnectorComponent(base.DBConnectorComponent):
@@ -257,3 +259,47 @@ def _builddictFromRow(self, row):
  complete_at=epoch2datetime(row.complete_at),
  state_string=row.state_string,
  results=row.results)
+
+ # returns a Deferred that returns a value
+ def deleteOldBuilds(self, horizon_per_builder=None):
+ builds = self.db.model.builds
+ builders = self.db.model.builders
+
+ def countBuilds(conn):
+ res = conn.execute(sa.select([sa.func.count(builds.c.id)]))
+ count = res.fetchone()[0]
+ res.close()
+ return count
+
+ def thdDeleteOldBuilds(conn):
+ count_before = countBuilds(conn)
+
+ for builderName in horizon_per_builder:
+ if "buildHorizon" in horizon_per_builder[builderName]:
+ older_than_timestamp = datetime2epoch(datetime.now() -
+ horizon_per_builder[builderName]["buildHorizon"])
+ if self.db._engine.dialect.name == 'sqlite':
+ # sqlite does not support delete with a join,
+ # so for this case we use a subquery,
+ # which is much slower
+
+ q = sa.select([builds.c.id])
+ q = q.where(sa.and_((builds.c.complete_at < older_than_timestamp),
+ builds.c.builderid == builders.c.id,
+ builders.c.name.like(builderName)))
+ q = builds.delete().where(builds.c.id.in_(q))
+
+ else:
+ q = builds.delete()
+ q = q.where(sa.and_(builds.c.builderid == builders.c.id,
+ builds.c.complete_at < older_than_timestamp,
+ builders.c.name.like(builderName)))
+
+ res = conn.execute(q)
+ res.close()
+
+ count_after = countBuilds(conn)
+
+ return count_before - count_after
+
+ return self.db.pool.do(thdDeleteOldBuilds)