debounce new buildrequests events.

The brd has an api that can schedule several builders for requests,
but we don't use it unless at startup.

This change uses the debouncer so that we wait until there is no
new buildrequest event for 1 s before starting them all

master/buildbot/process/botmaster.py

@@ -24,6 +24,7 @@
 from buildbot.process.results import CANCELLED
 from buildbot.process.results import RETRY
 from buildbot.process.workerforbuilder import States
+from buildbot.util import debounce
 from buildbot.util import service
 
 
@@ -100,6 +101,7 @@ def __init__(self):
  # a distributor for incoming build requests; see below
  self.brd = BuildRequestDistributor(self)
  self.brd.setServiceParent(self)
+ self._pending_builderids = set()
 
  @defer.inlineCallbacks
  def cleanShutdown(self, quickMode=False, stopReactor=True):
@@ -198,6 +200,26 @@ def getBuildernames(self):
  def getBuilders(self):
  return list(self.builders.values())
 
+ def _buildrequest_added(self, key, msg):
+ self._pending_builderids.add(msg['builderid'])
+ self._flush_pending_builders()
+
+ # flush pending builders needs to be debounced, as per design the
+ # buildrequests events will arrive in burst.
+ # We debounce them to let the brd manage them as a whole
+ # without having to debounce the brd itself
+ @debounce.method(wait=.1, until_idle=True)
+ def _flush_pending_builders(self):
+ if not self._pending_builderids:
+ return
+ buildernames = []
+ for builderid in self._pending_builderids:
+ buildername = self._builderid_to_buildername.get(builderid)
+ if buildername:
+ buildernames.append(buildername)
+ self._pending_builderids.clear()
+ self.brd.maybeStartBuildsOn(buildernames)
+
  @defer.inlineCallbacks
  def getBuilderById(self, builderid):
  for builder in self.getBuilders():
@@ -207,20 +229,13 @@ def getBuilderById(self, builderid):
 
  @defer.inlineCallbacks
  def startService(self):
- @defer.inlineCallbacks
- def buildRequestAdded(key, msg):
- builderid = msg['builderid']
- builder = yield self.getBuilderById(builderid)
- if builder is not None:
- self.maybeStartBuildsForBuilder(builder.name)
-
- # consume both 'new' and 'unclaimed' build requests
+ # consume both 'new' and 'unclaimed' build requests events
  startConsuming = self.master.mq.startConsuming
  self.buildrequest_consumer_new = yield startConsuming(
- buildRequestAdded,
+ self._buildrequest_added,
  ('buildrequests', None, "new"))
  self.buildrequest_consumer_unclaimed = yield startConsuming(
- buildRequestAdded,
+ self._buildrequest_added,
  ('buildrequests', None, 'unclaimed'))
  yield super().startService()
 
@@ -281,6 +296,9 @@ def reconfigServiceBuilders(self, new_config):
  yield builder.setServiceParent(self)
 
  self.builderNames = list(self.builders)
+ self._builderid_to_buildername = {}
+ for builder in self.builders.values():
+ self._builderid_to_buildername[(yield builder.getBuilderId())] = builder.name
 
  yield self.master.data.updates.updateBuilderList(
  self.master.masterid,
@@ -298,6 +316,8 @@ def stopService(self):
  if self.buildrequest_consumer_unclaimed:
  self.buildrequest_consumer_unclaimed.stopConsuming()
  self.buildrequest_consumer_unclaimed = None
+ self._pending_builderids.clear()
+ self._flush_pending_builders.stop()
  return super().stopService()
 
  def maybeStartBuildsForBuilder(self, buildername):

master/buildbot/test/integration/test_worker.py

@@ -111,6 +111,7 @@ def test_latent_max_builds(self):
  'project': ''},
  ],
  )
+ self.master.reactor.advance(1)
 
  # The worker fails to substantiate.
  controller.start_instance(True)
@@ -165,6 +166,7 @@ def test_local_worker_max_builds(self):
  'project': ''},
  ],
  )
+ self.master.reactor.advance(1)
 
  self.assertEqual(len(started_builds), 1)
 

master/buildbot/test/util/integration.py

@@ -151,6 +151,8 @@ def create_build_request(self, builder_ids, properties=None):
  ],
  properties=properties,
  )
+ # run debounced calls
+ self.master.reactor.advance(1)
  return ret
 
  @defer.inlineCallbacks