Rest: Filter buildsets by sourcestamp attributes

Allow the buildsets REST endpoint to filter by sourcestamp attributes.
We use the 'contains' relation to filter by a set of sourcestamp attributes:
buildsets?sourcestamps__contains={"ssid":26}

The list of buildsets is then only listing buildsets that have a
sourcetsamp with that ssid. If the sourcestamps__contains filter is used
multiple times it follows the "or" semantics like the other filters. If
a single filter lists multiple attributes these follow and semantics,
within the filter.

sourcestamps are encoded as JSON in the REST query.

The filters are implemented on SQL-level. Necessary joins are only
performed if this filter is actually used.

master/buildbot/data/sourcestamps.py

@@ -14,11 +14,14 @@
 # Copyright Buildbot Team Members
 
 
+import json
+
 from twisted.internet import defer
 
 from buildbot.data import base
 from buildbot.data import patches
 from buildbot.data import types
+from buildbot.util import bytes2unicode
 
 
 def _db2data(ss):
@@ -89,4 +92,12 @@ class EntityType(types.Entity):
  codebase = types.String()
  patch = types.NoneOk(patches.Patch.entityType)
  created_at = types.DateTime()
+
+ def valueFromString(self, arg):
+ d = json.loads(bytes2unicode(arg))
+ d = {k:self.fields[k].valueFromString(v) for k,v in d.items()}
+ if len(d.keys()) == 0:
+ raise TypeError # don't allow {}
+ return d
+
  entityType = EntityType(name, 'Sourcestamp')

master/buildbot/db/buildsets.py

@@ -17,6 +17,9 @@
 """
 
 import json
+from functools import reduce
+from operator import or_
+from operator import and_
 
 import sqlalchemy as sa
 
@@ -165,15 +168,34 @@ def thd(conn):
  @defer.inlineCallbacks
  def getBuildsets(self, complete=None, resultSpec=None):
  def thd(conn):
+ if resultSpec is not None:
+ sourceStampFilter = resultSpec.popFilter('sourcestamps', 'contains')
+ else:
+ sourceStampFilter = None
+
  bs_tbl = self.db.model.buildsets
- q = bs_tbl.select()
+ ss_tbl = self.db.model.sourcestamps
+
+ j = bs_tbl
+ if sourceStampFilter is not None:
+ j = bs_tbl
+ j = j.join(self.db.model.buildset_sourcestamps)
+ j = j.join(ss_tbl)
+ q = sa.select(columns=[bs_tbl], from_obj=[j], distinct=True)
+
  if complete is not None:
  if complete:
  q = q.where(bs_tbl.c.complete != 0)
  else:
  q = q.where((bs_tbl.c.complete == 0) |
  (bs_tbl.c.complete == NULL))
+
  if resultSpec is not None:
+ if sourceStampFilter:
+ def ssid_as_id(key):
+ return "id" if key == "ssid" else key
+ q = q.where(reduce(or_, [reduce(and_, [ss_tbl.c[ssid_as_id(key)] == value for key, value in filter.items()])
+ for filter in sourceStampFilter]))
  return resultSpec.thd_execute(conn, q, lambda x: self._thd_row2dict(conn, x))
  res = conn.execute(q)
  return [self._thd_row2dict(conn, row) for row in res.fetchall()]