checksum handling unification

build/transpile.js

@@ -269,6 +269,9 @@ class Transpiler {
  [ /\.checkProxySettings\s/g, '.check_proxy_settings'],
  [ /\.getExchangePropAllCase\s/g, '.get_exchange_prop_all_case'],
  [ /\.setExchangePropAllCase\s/g, '.set_exchange_prop_all_case'],
+ [ /\.orderBookSequenceErrorThrow\s/g, '.order_book_sequence_error_throw'],
+ [ /\.orderBookSequenceErrorReject\s/g, '.order_book_sequence_error_reject'],
+ [ /\.orderBookSequenceErrorMessage\s/g, '.order_book_sequence_error_message'],
  [ /\.getProperty\s/g, '.get_property'],
  [ /\.safeProp\s/g, '.safe_prop'],
  [ /\.safeProp2\s/g, '.safe_prop_2'],

python/ccxt/base/exchange.py

@@ -17,6 +17,7 @@
 from ccxt.base.errors import ExchangeNotAvailable
 from ccxt.base.errors import InvalidAddress
 from ccxt.base.errors import InvalidOrder
+from ccxt.base.errors import InvalidNonce
 from ccxt.base.errors import ArgumentsRequired
 from ccxt.base.errors import BadSymbol
 from ccxt.base.errors import NullResponse

ts/src/base/Exchange.ts

@@ -128,6 +128,7 @@ import {
  , RequestTimeout
  , NetworkError
  , ProxyError
+ , InvalidNonce
  , ExchangeNotAvailable
  , ArgumentsRequired
  , RateLimitExceeded,
@@ -2486,6 +2487,32 @@ export default class Exchange {
  this.createNetworksByIdObject ();
  }
 
+ orderBookSequenceErrorMessage (symbol:Str, storedNonceOrChecksum, incomingNonceOrChecksum) {
+ let msg = this.id + ' incoming orderbook data checksum/nonce validation failed. In such cases you are advised to have a reconnection logic to get in sync of correct orderbook, or temporarily you can mute the error by setting exhcange.options["validateOrderBookSequences"] = false';
+ const safeSymbol = (symbol === undefined) ? '' : symbol;
+ const storedString = (storedNonceOrChecksum === undefined) ? '' : storedNonceOrChecksum.toString ();
+ const incomingString = (incomingNonceOrChecksum === undefined) ? '' : incomingNonceOrChecksum.toString ();
+ msg += ' symbol:' + safeSymbol + ', stored: ' + storedString + ', incoming: ' + incomingString;
+ return msg;
+ }
+
+ orderBookSequenceErrorThrow (symbol:Str, storedNonceOrChecksum, incomingNonceOrChecksum) {
+ const validate = this.safeBool2 (this.options, 'validateOrderBookSequences', 'checksum', true);
+ if (validate) {
+ const msg = this.orderBookSequenceErrorMessage (symbol, storedNonceOrChecksum, incomingNonceOrChecksum);
+ throw new InvalidNonce ('' + msg.toString ()); // transpiler trick
+ }
+ }
+
+ orderBookSequenceErrorReject (client, messageHash, symbol:Str, storedNonceOrChecksum, incomingNonceOrChecksum) {
+ const validate = this.safeBool2 (this.options, 'validateOrderBookSequences', 'checksum', true);
+ if (validate) {
+ const msg = this.orderBookSequenceErrorMessage (symbol, storedNonceOrChecksum, incomingNonceOrChecksum);
+ const err = new InvalidNonce ('' + msg.toString ());
+ client.reject (err, messageHash);
+ }
+ }
+
  createNetworksByIdObject () {
  // automatically generate network-id-to-code mappings
  const networkIdsToCodesGenerated = this.invertFlatStringDictionary (this.safeValue (this.options, 'networks', {})); // invert defined networks dictionary

ts/src/pro/binance.ts

@@ -3,7 +3,7 @@
 
 import binanceRest from '../binance.js';
 import { Precise } from '../base/Precise.js';
-import { InvalidNonce, ArgumentsRequired, BadRequest, NotSupported } from '../base/errors.js';
+import { ArgumentsRequired, BadRequest, NotSupported } from '../base/errors.js';
 import { ArrayCache, ArrayCacheByTimestamp, ArrayCacheBySymbolById, ArrayCacheBySymbolBySide } from '../base/ws/Cache.js';
 import type { Int, OrderSide, OrderType, Str, Strings, Trade, OrderBook, Order, Ticker, Tickers, OHLCV, Position, Balances, Num } from '../base/types.js';
 import { sha256 } from '../static_dependencies/noble-hashes/sha256.js';
@@ -429,11 +429,8 @@ export default class binance extends binanceRest {
  client.resolve (orderbook, messageHash);
  }
  } else {
- const checksum = this.safeBool (this.options, 'checksum', true);
- if (checksum) {
- // todo: client.reject from handleOrderBookMessage properly
- throw new InvalidNonce (this.id + ' handleOrderBook received an out-of-order nonce');
- }
+ // todo: client.reject from handleOrderBookMessage properly
+ this.orderBookSequenceErrorThrow (symbol, orderbook['nonce'], U - 1);
  }
  }
  } else {
@@ -448,11 +445,8 @@ export default class binance extends binanceRest {
  client.resolve (orderbook, messageHash);
  }
  } else {
- const checksum = this.safeBool (this.options, 'checksum', true);
- if (checksum) {
- // todo: client.reject from handleOrderBookMessage properly
- throw new InvalidNonce (this.id + ' handleOrderBook received an out-of-order nonce');
- }
+ // todo: client.reject from handleOrderBookMessage properly
+ this.orderBookSequenceErrorThrow (symbol, orderbook['nonce'], pu);
  }
  }
  }

ts/src/pro/bitfinex2.ts

@@ -3,7 +3,7 @@
 
 import bitfinex2Rest from '../bitfinex2.js';
 import { Precise } from '../base/Precise.js';
-import { ExchangeError, AuthenticationError, InvalidNonce } from '../base/errors.js';
+import { ExchangeError, AuthenticationError } from '../base/errors.js';
 import { ArrayCache, ArrayCacheBySymbolById, ArrayCacheByTimestamp } from '../base/ws/Cache.js';
 import { sha384 } from '../static_dependencies/noble-hashes/sha512.js';
 import type { Int, Str, OrderBook, Order, Trade, Ticker, OHLCV, Balances } from '../base/types.js';
@@ -39,7 +39,7 @@ export default class bitfinex2 extends bitfinex2Rest {
  'freq': 'F0',
  },
  'ordersLimit': 1000,
- 'checksum': true,
+ 'validateOrderBookSequences': true,
  },
  });
  }
@@ -692,8 +692,7 @@ export default class bitfinex2 extends bitfinex2Rest {
  const localChecksum = this.crc32 (payload, true);
  const responseChecksum = this.safeInteger (message, 2);
  if (responseChecksum !== localChecksum) {
- const error = new InvalidNonce (this.id + ' invalid checksum');
- client.reject (error, messageHash);
+ this.orderBookSequenceErrorReject (client, messageHash, symbol, localChecksum, responseChecksum);
  }
  }
 

ts/src/pro/bitget.ts

@@ -1,7 +1,7 @@
 // ---------------------------------------------------------------------------
 
 import bitgetRest from '../bitget.js';
-import { AuthenticationError, BadRequest, ArgumentsRequired, NotSupported, InvalidNonce, ExchangeError, RateLimitExceeded } from '../base/errors.js';
+import { AuthenticationError, BadRequest, ArgumentsRequired, NotSupported, ExchangeError, RateLimitExceeded } from '../base/errors.js';
 import { Precise } from '../base/Precise.js';
 import { ArrayCache, ArrayCacheBySymbolById, ArrayCacheBySymbolBySide, ArrayCacheByTimestamp } from '../base/ws/Cache.js';
 import { sha256 } from '../static_dependencies/noble-hashes/sha256.js';
@@ -570,8 +570,7 @@ export default class bitget extends bitgetRest {
  const calculatedChecksum = this.crc32 (payload, true);
  const responseChecksum = this.safeInteger (rawOrderBook, 'checksum');
  if (calculatedChecksum !== responseChecksum) {
- const error = new InvalidNonce (this.id + ' invalid checksum');
- client.reject (error, messageHash);
+ this.orderBookSequenceErrorReject (client, messageHash, symbol, calculatedChecksum, responseChecksum);
  }
  }
  } else {

ts/src/pro/cryptocom.ts

@@ -2,7 +2,7 @@
 // ---------------------------------------------------------------------------
 
 import cryptocomRest from '../cryptocom.js';
-import { AuthenticationError, InvalidNonce, NetworkError } from '../base/errors.js';
+import { AuthenticationError, NetworkError } from '../base/errors.js';
 import { ArrayCache, ArrayCacheByTimestamp, ArrayCacheBySymbolById, ArrayCacheBySymbolBySide } from '../base/ws/Cache.js';
 import { sha256 } from '../static_dependencies/noble-hashes/sha256.js';
 import type { Int, OrderSide, OrderType, Str, Strings, OrderBook, Order, Trade, Ticker, OHLCV, Position, Balances, Num } from '../base/types.js';
@@ -223,7 +223,7 @@ export default class cryptocom extends cryptocomRest {
  const previousNonce = this.safeInteger (data, 'pu');
  const currentNonce = orderbook['nonce'];
  if (currentNonce !== previousNonce) {
- throw new InvalidNonce (this.id + ' watchOrderBook() ' + symbol + ' ' + previousNonce + ' != ' + nonce);
+ this.orderBookSequenceErrorThrow (symbol, previousNonce, currentNonce);
  }
  }
  this.handleDeltas (orderbook['asks'], this.safeValue (books, 'asks', []));

ts/src/pro/gate.ts

@@ -2,7 +2,7 @@
 // ---------------------------------------------------------------------------
 
 import gateRest from '../gate.js';
-import { AuthenticationError, BadRequest, ArgumentsRequired, InvalidNonce } from '../base/errors.js';
+import { AuthenticationError, BadRequest, ArgumentsRequired } from '../base/errors.js';
 import { ArrayCache, ArrayCacheByTimestamp, ArrayCacheBySymbolById, ArrayCacheBySymbolBySide } from '../base/ws/Cache.js';
 import { sha512 } from '../static_dependencies/noble-hashes/sha512.js';
 import type { Int, Str, Strings, OrderBook, Order, Trade, Ticker, Tickers, OHLCV, Position, Balances } from '../base/types.js';
@@ -222,10 +222,9 @@ export default class gate extends gateRest {
  } else if (nonce >= deltaStart - 1) {
  this.handleDelta (storedOrderBook, delta);
  } else {
- const error = new InvalidNonce (this.id + ' orderbook update has a nonce bigger than u');
  delete client.subscriptions[messageHash];
  delete this.orderbooks[symbol];
- client.reject (error, messageHash);
+ this.orderBookSequenceErrorReject (client, messageHash, symbol, nonce, deltaStart);
  }
  client.resolve (storedOrderBook, messageHash);
  }

ts/src/pro/htx.ts

@@ -577,7 +577,7 @@ export default class htx extends htxRest {
  orderbook['nonce'] = version;
  }
  if ((prevSeqNum !== undefined) && prevSeqNum > orderbook['nonce']) {
- throw new InvalidNonce (this.id + ' watchOrderBook() received a mesage out of order');
+ this.orderBookSequenceErrorThrow (symbol, prevSeqNum, orderbook['nonce']);
  }
  const spotConditon = market['spot'] && (prevSeqNum === orderbook['nonce']);
  const nonSpotCondition = market['contract'] && (version - 1 === orderbook['nonce']);

ts/src/pro/independentreserve.ts

@@ -1,7 +1,7 @@
 // ---------------------------------------------------------------------------
 
 import independentreserveRest from '../independentreserve.js';
-import { NotSupported, InvalidNonce } from '../base/errors.js';
+import { NotSupported } from '../base/errors.js';
 import { ArrayCache } from '../base/ws/Cache.js';
 import type { Int, OrderBook, Trade } from '../base/types.js';
 import Client from '../base/ws/Client.js';
@@ -28,7 +28,7 @@ export default class independentreserve extends independentreserveRest {
  },
  },
  'options': {
- 'checksum': false, // TODO: currently only working for snapshot
+ 'validateOrderBookSequences': false, // TODO: currently only working for snapshot
  },
  'streaming': {
  },
@@ -204,8 +204,7 @@ export default class independentreserve extends independentreserveRest {
  orderbook['timestamp'] = timestamp;
  orderbook['datetime'] = this.iso8601 (timestamp);
  }
- const checksum = this.safeBool (this.options, 'checksum', true);
- if (checksum && receivedSnapshot) {
+ if (receivedSnapshot) {
  const storedAsks = orderbook['asks'];
  const storedBids = orderbook['bids'];
  const asksLength = storedAsks.length;
@@ -224,8 +223,7 @@ export default class independentreserve extends independentreserveRest {
  const calculatedChecksum = this.crc32 (payload, true);
  const responseChecksum = this.safeInteger (orderBook, 'Crc32');
  if (calculatedChecksum !== responseChecksum) {
- const error = new InvalidNonce (this.id + ' invalid checksum');
- client.reject (error, messageHash);
+ this.orderBookSequenceErrorReject (client, messageHash, symbol, calculatedChecksum, responseChecksum);
  }
  }
  if (receivedSnapshot) {

ts/src/pro/kraken.ts

@@ -2,7 +2,7 @@
 // ---------------------------------------------------------------------------
 
 import krakenRest from '../kraken.js';
-import { ExchangeError, BadSymbol, PermissionDenied, AccountSuspended, BadRequest, InsufficientFunds, InvalidOrder, OrderNotFound, NotSupported, RateLimitExceeded, ExchangeNotAvailable, InvalidNonce, AuthenticationError } from '../base/errors.js';
+import { ExchangeError, BadSymbol, PermissionDenied, AccountSuspended, BadRequest, InsufficientFunds, InvalidOrder, OrderNotFound, NotSupported, RateLimitExceeded, ExchangeNotAvailable, AuthenticationError } from '../base/errors.js';
 import { ArrayCache, ArrayCacheByTimestamp, ArrayCacheBySymbolById } from '../base/ws/Cache.js';
 import { Precise } from '../base/Precise.js';
 import type { Int, Strings, OrderSide, OrderType, Str, OrderBook, Order, Trade, Ticker, Tickers, OHLCV, Num } from '../base/types.js';
@@ -50,7 +50,7 @@ export default class kraken extends krakenRest {
  'OHLCVLimit': 1000,
  'ordersLimit': 1000,
  'symbolsByOrderId': {},
- 'checksum': true,
+ 'validateOrderBookSequences': true,
  },
  'exceptions': {
  'ws': {
@@ -765,8 +765,8 @@ export default class kraken extends krakenRest {
  }
  // don't remove this line or I will poop on your face
  orderbook.limit ();
- const checksum = this.safeBool (this.options, 'checksum', true);
- if (checksum) {
+ const validate = this.safeBool2 (this.options, 'validateOrderBookSequences', 'checksum', true);
+ if (validate) {
  const priceString = this.safeString (example, 0);
  const amountString = this.safeString (example, 1);
  const priceParts = priceString.split ('.');
@@ -787,8 +787,7 @@ export default class kraken extends krakenRest {
  const payload = payloadArray.join ('');
  const localChecksum = this.crc32 (payload, false);
  if (localChecksum !== c) {
- const error = new InvalidNonce (this.id + ' invalid checksum');
- client.reject (error, messageHash);
+ this.orderBookSequenceErrorReject (client, messageHash, symbol, localChecksum, c);
  return;
  }
  }

ts/src/pro/okx.ts

@@ -2,7 +2,7 @@
 // ---------------------------------------------------------------------------
 
 import okxRest from '../okx.js';
-import { ArgumentsRequired, AuthenticationError, BadRequest, InvalidNonce } from '../base/errors.js';
+import { ArgumentsRequired, AuthenticationError, BadRequest } from '../base/errors.js';
 import { ArrayCache, ArrayCacheByTimestamp, ArrayCacheBySymbolById, ArrayCacheBySymbolBySide } from '../base/ws/Cache.js';
 import { sha256 } from '../static_dependencies/noble-hashes/sha256.js';
 import type { Int, OrderSide, OrderType, Str, Strings, OrderBook, Order, Trade, Ticker, Tickers, OHLCV, Position, Balances, Num } from '../base/types.js';
@@ -90,7 +90,7 @@ export default class okx extends okxRest {
  'ws': {
  // 'inflate': true,
  },
- 'checksum': true,
+ 'validateOrderBookSequences': true,
  },
  'streaming': {
  // okex does not support built-in ws protocol-level ping-pong
@@ -591,8 +591,8 @@ export default class okx extends okxRest {
  const storedBids = orderbook['bids'];
  this.handleDeltas (storedAsks, asks);
  this.handleDeltas (storedBids, bids);
- const checksum = this.safeBool (this.options, 'checksum', true);
- if (checksum) {
+ const validate = this.safeBool2 (this.options, 'validateOrderBookSequences', 'checksum', true);
+ if (validate) {
  const asksLength = storedAsks.length;
  const bidsLength = storedBids.length;
  const payloadArray = [];
@@ -610,8 +610,7 @@ export default class okx extends okxRest {
  const responseChecksum = this.safeInteger (message, 'checksum');
  const localChecksum = this.crc32 (payload, true);
  if (responseChecksum !== localChecksum) {
- const error = new InvalidNonce (this.id + ' invalid checksum');
- client.reject (error, messageHash);
+ this.orderBookSequenceErrorReject (client, messageHash, orderbook['symbol'], localChecksum, responseChecksum);
  }
  }
  const timestamp = this.safeInteger (message, 'ts');

ts/src/pro/poloniexfutures.ts

@@ -1,7 +1,7 @@
 // ---------------------------------------------------------------------------
 
 import poloniexfuturesRest from '../poloniexfutures.js';
-import { AuthenticationError, BadRequest, InvalidNonce } from '../base/errors.js';
+import { AuthenticationError, BadRequest } from '../base/errors.js';
 import { ArrayCache, ArrayCacheBySymbolById } from '../base/ws/Cache.js';
 import type { Int, Str, OrderBook, Order, Trade, Ticker, Balances } from '../base/types.js';
 import Client from '../base/ws/Client.js';
@@ -55,6 +55,7 @@ export default class poloniexfutures extends poloniexfuturesRest {
  'streamLimit': 5, // called tunnels by poloniexfutures docs
  'streamBySubscriptionsHash': {},
  'streamIndex': -1,
+ 'validateOrderBookSequences': true,
  },
  'streaming': {
  'keepAlive': 30000,
@@ -877,11 +878,8 @@ export default class poloniexfutures extends poloniexfuturesRest {
  const sequence = this.safeInteger (delta, 'sequence');
  const nonce = this.safeInteger (orderbook, 'nonce');
  if (nonce !== sequence - 1) {
- const checksum = this.safeBool (this.options, 'checksum', true);
- if (checksum) {
- // todo: client.reject from handleOrderBookMessage properly
- throw new InvalidNonce (this.id + ' watchOrderBook received an out-of-order nonce');
- }
+ // todo: client.reject from handleOrderBookMessage properly
+ this.orderBookSequenceErrorThrow (orderbook['symbol'], nonce, sequence);
  }
  const change = this.safeString (delta, 'change');
  const splitChange = change.split (',');