v0.37.x:state/kvindexer: port 0.34 query fix

.changelog/unreleased/bug-fixes/77-kvindexer-fix-evattribute-indexing.md

@@ -0,0 +1 @@
+- [state/kvindexer] \#77 Fixed the default behaviour of the kvindexer to index and query attributes by events in which they occur. In 0.34.25 this was mitigated by a separated RPC flag. (@jmalicevic)

abci/example/kvstore/kvstore.go

@@ -69,13 +69,20 @@ type Application struct {
  state State
  RetainBlocks int64 // blocks to retain after commit (via ResponseCommit.RetainHeight)
  txToRemove map[string]struct{}
+ // If true, the app will generate block events in BeginBlock. Used to test the event indexer
+ // Should be false by default to avoid generating too much data.
+ genBlockEvents bool
 }
 
 func NewApplication() *Application {
  state := loadState(dbm.NewMemDB())
  return &Application{state: state}
 }
 
+func (app *Application) SetGenBlockEvents() {
+ app.genBlockEvents = true
+}
+
 func (app *Application) Info(req types.RequestInfo) (resInfo types.ResponseInfo) {
  return types.ResponseInfo{
  Data: fmt.Sprintf("{\"size\":%v}", app.state.Size),
@@ -116,6 +123,15 @@ func (app *Application) DeliverTx(req types.RequestDeliverTx) types.ResponseDeli
  {Key: "noindex_key", Value: "index is working", Index: false},
  },
  },
+ {
+ Type: "app",
+ Attributes: []types.EventAttribute{
+ {Key: "creator", Value: "Cosmoshi", Index: true},
+ {Key: "key", Value: value, Index: true},
+ {Key: "index_key", Value: "index is working", Index: true},
+ {Key: "noindex_key", Value: "index is working", Index: false},
+ },
+ },
  }
 
  return types.ResponseDeliverTx{Code: code.CodeTypeOK, Events: events}
@@ -189,7 +205,70 @@ func (app *Application) Query(reqQuery types.RequestQuery) (resQuery types.Respo
 
 func (app *Application) BeginBlock(req types.RequestBeginBlock) types.ResponseBeginBlock {
  app.txToRemove = map[string]struct{}{}
- return types.ResponseBeginBlock{}
+ response := types.ResponseBeginBlock{}
+
+ if !app.genBlockEvents {
+ return response
+ }
+
+ if app.state.Height%2 == 0 {
+ response = types.ResponseBeginBlock{
+ Events: []types.Event{
+ {
+ Type: "begin_event",
+ Attributes: []types.EventAttribute{
+ {
+ Key: "foo",
+ Value: "100",
+ Index: true,
+ },
+ {
+ Key: "bar",
+ Value: "200",
+ Index: true,
+ },
+ },
+ },
+ {
+ Type: "begin_event",
+ Attributes: []types.EventAttribute{
+ {
+ Key: "foo",
+ Value: "200",
+ Index: true,
+ },
+ {
+ Key: "bar",
+ Value: "300",
+ Index: true,
+ },
+ },
+ },
+ },
+ }
+ } else {
+ response = types.ResponseBeginBlock{
+ Events: []types.Event{
+ {
+ Type: "begin_event",
+ Attributes: []types.EventAttribute{
+ {
+ Key: "foo",
+ Value: "400",
+ Index: true,
+ },
+ {
+ Key: "bar",
+ Value: "300",
+ Index: true,
+ },
+ },
+ },
+ },
+ }
+ }
+
+ return response
 }
 
 func (app *Application) ProcessProposal(

abci/example/kvstore/persistent_kvstore.go

@@ -54,6 +54,10 @@ func NewPersistentKVStoreApplication(dbDir string) *PersistentKVStoreApplication
  }
 }
 
+func (app *PersistentKVStoreApplication) SetGenBlockEvents() {
+ app.app.genBlockEvents = true
+}
+
 func (app *PersistentKVStoreApplication) SetLogger(l log.Logger) {
  app.logger = l
 }

docs/app-dev/indexing-transactions.md

@@ -4,7 +4,7 @@ order: 6
 
 # Indexing Transactions
 
-Tendermint allows you to index transactions and blocks and later query or
+CometBFT allows you to index transactions and blocks and later query or
 subscribe to their results. Transactions are indexed by `TxResult.Events` and
 blocks are indexed by `Response(Begin|End)Block.Events`. However, transactions
 are also indexed by a primary key which includes the transaction hash and maps
@@ -15,7 +15,7 @@ the block itself is never stored.
 Each event contains a type and a list of attributes, which are key-value pairs
 denoting something about what happened during the method's execution. For more
 details on `Events`, see the
-[ABCI](https://github.com/tendermint/tendermint/blob/v0.37.x/spec/abci/abci.md#events)
+[ABCI](https://github.com/cometbft/cometbft/blob/v0.37.x/spec/abci/abci.md#events)
 documentation.
 
 An `Event` has a composite key associated with it. A `compositeKey` is
@@ -31,9 +31,12 @@ For example:
 
 would be equal to the composite key of `jack.account.number`.
 
-By default, Tendermint will index all transactions by their respective hashes
+By default, CometBFT will index all transactions by their respective hashes
 and height and blocks by their height.
 
+CometBFT allows for different events within the same height to have 
+equal attributes.
+
 ## Configuration
 
 Operators can configure indexing via the `[tx_index]` section. The `indexer`
@@ -62,23 +65,77 @@ be turned off regardless of other values provided.
 #### KV
 
 The `kv` indexer type is an embedded key-value store supported by the main
-underlying Tendermint database. Using the `kv` indexer type allows you to query
-for block and transaction events directly against Tendermint's RPC. However, the
+underlying CometBFT database. Using the `kv` indexer type allows you to query
+for block and transaction events directly against CometBFT's RPC. However, the
 query syntax is limited and so this indexer type might be deprecated or removed
 entirely in the future.
 
+**Implementation and data layout**
+
+The kv indexer stores each attribute of an event individually, by creating a composite key 
+with
+- event type,
+- attribute key,
+- attribute value,
+- event generator (e.g. `EndBlock` and `BeginBlock`)
+- the height, and
+- event counter.
+ For example the following events:
+
+```
+Type: "transfer",
+ Attributes: []abci.EventAttribute{
+ {Key: "sender", Value: "Bob", Index: true},
+ {Key: "recipient", Value: "Alice", Index: true},
+ {Key: "balance", Value: "100", Index: true},
+ {Key: "note", Value: "nothing", Index: true},
+ },
+
+```
+
+```
+Type: "transfer",
+ Attributes: []abci.EventAttribute{
+ {Key: "sender", Value: "Tom", Index: true},
+ {Key: "recipient", Value: "Alice", Index: true},
+ {Key: "balance", Value: "200", Index: true},
+ {Key: "note", Value: "nothing", Index: true},
+ },
+```
+
+will be represented as follows in the store, assuming these events result from the EndBlock call for height 1: 
+
+```
+Key value
+---- event1 ------
+transferSenderBobEndBlock11 1
+transferRecipientAliceEndBlock11 1
+transferBalance100EndBlock11 1
+transferNodeNothingEndblock11 1
+---- event2 ------
+transferSenderTomEndBlock12 1
+transferRecepientAliceEndBlock12 1
+transferBalance200EndBlock12 1
+transferNodeNothingEndblock12 1
+
+```
+The event number is a local variable kept by the indexer and incremented when a new event is processed. 
+It is an `int64` variable and has no other semantics besides being used to associate attributes belonging to the same events within a height. 
+This variable is not atomically incremented as event indexing is deterministic. **Should this ever change**, the event id generation
+will be broken. 
+
 #### PostgreSQL
 
 The `psql` indexer type allows an operator to enable block and transaction event
 indexing by proxying it to an external PostgreSQL instance allowing for the events
 to be stored in relational models. Since the events are stored in a RDBMS, operators
 can leverage SQL to perform a series of rich and complex queries that are not
 supported by the `kv` indexer type. Since operators can leverage SQL directly,
-searching is not enabled for the `psql` indexer type via Tendermint's RPC -- any
+searching is not enabled for the `psql` indexer type via CometBFT's RPC -- any
 such query will fail.
 
 Note, the SQL schema is stored in `state/indexer/sink/psql/schema.sql` and operators
-must explicitly create the relations prior to starting Tendermint and enabling
+must explicitly create the relations prior to starting CometBFT and enabling
 the `psql` indexer type.
 
 Example:
@@ -89,7 +146,7 @@ $ psql ... -f state/indexer/sink/psql/schema.sql
 
 ## Default Indexes
 
-The Tendermint tx and block event indexer indexes a few select reserved events
+The CometBFT tx and block event indexer indexes a few select reserved events
 by default.
 
 ### Transactions
@@ -107,7 +164,7 @@ The following indexes are indexed by default:
 
 ## Adding Events
 
-Applications are free to define which events to index. Tendermint does not
+Applications are free to define which events to index. CometBFT does not
 expose functionality to define which events to index and which to ignore. In
 your application's `DeliverTx` method, add the `Events` field with pairs of
 UTF-8 encoded strings (e.g. "transfer.sender": "Bob", "transfer.recipient":
@@ -122,10 +179,10 @@ func (app *KVStoreApplication) DeliverTx(req types.RequestDeliverTx) types.Resul
  {
  Type: "transfer",
  Attributes: []abci.EventAttribute{
- {Key: []byte("sender"), Value: []byte("Bob"), Index: true},
- {Key: []byte("recipient"), Value: []byte("Alice"), Index: true},
- {Key: []byte("balance"), Value: []byte("100"), Index: true},
- {Key: []byte("note"), Value: []byte("nothing"), Index: true},
+ {Key: "sender ", Value: "Bob ", Index: true},
+ {Key: "recipient ", Value: "Alice ", Index: true},
+ {Key: "balance ", Value: "100 ", Index: true},
+ {Key: "note ", Value: "nothing ", Index: true},
  },
  },
  }
@@ -146,7 +203,7 @@ You can query for a paginated set of transaction by their events by calling the
 curl "localhost:26657/tx_search?query=\"message.sender='cosmos1...'\"&prove=true"
 ```
 
-Check out [API docs](https://docs.tendermint.com/v0.37/rpc/#/Info/tx_search)
+Check out [API docs](https://docs.cometbft.com/v0.37/rpc/#/Info/tx_search)
 for more information on query syntax and other options.
 
 ## Subscribing to Transactions
@@ -165,10 +222,10 @@ a query to `/subscribe` RPC endpoint.
 }
 ```
 
-Check out [API docs](https://docs.tendermint.com/v0.37/rpc/#subscribe) for more information
+Check out [API docs](https://docs.cometbft.com/v0.37/rpc/#subscribe) for more information
 on query syntax and other options.
 
-## Querying Blocks Events
+## Querying Block Events
 
 You can query for a paginated set of blocks by their events by calling the
 `/block_search` RPC endpoint:
@@ -177,5 +234,12 @@ You can query for a paginated set of blocks by their events by calling the
 curl "localhost:26657/block_search?query=\"block.height > 10 AND val_set.num_changed > 0\""
 ```
 
-Check out [API docs](https://docs.tendermint.com/v0.37/rpc/#/Info/block_search)
-for more information on query syntax and other options.
+**Backwards compatibility**
+
+Storing the event sequence was introduced in CometBFT 0.34.26. Before that, up until Tendermint Core 0.34.26, 
+the event sequence was not stored in the kvstore and events were stored only by height. That means that queries 
+returned blocks and transactions whose event attributes match within the height but can match across different 
+events on that height. 
+This behavior was fixed with CometBFT 0.34.26+. However, if the data was indexed with earlier versions of
+Tendermint Core and not re-indexed, that data will be queried as if all the attributes within a height
+occurred within the same event.

rpc/client/main_test.go

@@ -19,6 +19,8 @@ func TestMain(m *testing.M) {
  }
 
  app := kvstore.NewPersistentKVStoreApplication(dir)
+ // If testing block event generation
+ // app.SetGenBlockEvents() needs to be called here
  node = rpctest.StartTendermint(app)
 
  code := m.Run()

rpc/client/rpc_test.go

@@ -516,6 +516,27 @@ func TestTxSearchWithTimeout(t *testing.T) {
  require.Greater(t, len(result.Txs), 0, "expected a lot of transactions")
 }
 
+// This test does nothing if we do not call app.SetGenBlockEvents() within main_test.go
+// It will nevertheless pass as there are no events being generated.
+func TestBlockSearch(t *testing.T) {
+ c := getHTTPClient()
+
+ // first we broadcast a few txs
+ for i := 0; i < 10; i++ {
+ _, _, tx := MakeTxKV()
+
+ _, err := c.BroadcastTxCommit(context.Background(), tx)
+ require.NoError(t, err)
+ }
+ require.NoError(t, client.WaitForHeight(c, 5, nil))
+ // This cannot test match_events as it calls the client BlockSearch function directly
+ // It is the RPC request handler that processes the match_event
+ result, err := c.BlockSearch(context.Background(), "begin_event.foo = 100 AND begin_event.bar = 300", nil, nil, "asc")
+ require.NoError(t, err)
+ blockCount := len(result.Blocks)
+ require.Equal(t, blockCount, 0)
+
+}
 func TestTxSearch(t *testing.T) {
  c := getHTTPClient()
 
@@ -536,8 +557,7 @@ func TestTxSearch(t *testing.T) {
  find := result.Txs[len(result.Txs)-1]
  anotherTxHash := types.Tx("a different tx").Hash()
 
- for i, c := range GetClients() {
- t.Logf("client %d", i)
+ for _, c := range GetClients() {
 
  // now we query for the tx.
  result, err := c.TxSearch(context.Background(), fmt.Sprintf("tx.hash='%v'", find.Hash), true, nil, nil, "asc")
@@ -616,16 +636,17 @@ func TestTxSearch(t *testing.T) {
  pages = int(math.Ceil(float64(txCount) / float64(perPage)))
  )
 
+ totalTx := 0
  for page := 1; page <= pages; page++ {
  page := page
- result, err := c.TxSearch(context.Background(), "tx.height >= 1", false, &page, &perPage, "asc")
+ result, err := c.TxSearch(context.Background(), "tx.height >= 1", true, &page, &perPage, "asc")
  require.NoError(t, err)
  if page < pages {
  require.Len(t, result.Txs, perPage)
  } else {
  require.LessOrEqual(t, len(result.Txs), perPage)
  }
- require.Equal(t, txCount, result.TotalCount)
+ totalTx = totalTx + len(result.Txs)
  for _, tx := range result.Txs {
  require.False(t, seen[tx.Height],
  "Found duplicate height %v in page %v", tx.Height, page)
@@ -635,6 +656,7 @@ func TestTxSearch(t *testing.T) {
  maxHeight = tx.Height
  }
  }
+ require.Equal(t, txCount, totalTx)
  require.Len(t, seen, txCount)
  }
 }