After some time of working, the PLC suddenly stopped responding to requests

[YuriKhachatryan]

It is usually stopped after some failed requests either
2023-06-29T06:41:29.365Z startGettingData Timed out ETIMEDOUT
or
2023-06-29T06:33:26.760Z startGettingData Port Not Open ECONNREFUSED
or
2023-07-06T05:27:19.498Z connect ETIMEDOUT {IP}:{PORT} -110

HMI continues working correctly. Our system gets the responses after restarting the PLC or plugging out/in the LAN cable.

It issues accrues when code is deployed in AWS ECS. During the local deployment, we didn't face it yet.

I attempted to read both individual registers and multiple registers simultaneously. The issue persists. Interestingly, even though the requests count to PLC is lowered drastically, the issue arises more frequently in the case of reading several registers at once.
PLC Model:

The following is my setup:

main.js

const modbus = require('./src/plc/plc');
const getters = require('./src/redis');
async function main() {
    try {
        ...
        await modbus.connect();
        getters.startGettingData();
        ...
    } catch (error) {
        console.log(new Date().toISOString(), 'Something goes wrong', error.message, error.errno);
        modbus.client.close();
        process.exit(1);
    }
};
main();

plc.js

const { ModbusTCPClient } = require("jsmodbus");
const net = require("net");
const { promisify } = require("util");
const { setTimeout } = require("timers/promises");

class JsModbusConnection {
  constructor() {
    this.socket = new net.Socket();
    this.options = {
      host: process.env.PLC_IP,
      port: parseInt(process.env.PLC_PORT),
    };
    this.client = new ModbusTCPClient(this.socket, 1);
  }

  async checkError() {
    console.log(
      new Date().toISOString(),
      "Checking jsmodbus error and reconnecting"
    );

    // Close port
    this.client.socket.end();
    await setTimeout(5000);

    // Re-open client
    this.client = new ModbusTCPClient(this.socket, 1);
    await setTimeout(2000);
    await this.connect();
  }
  async connect() {
    try {
      console.log(
        new Date().toISOString(),
        `Trying to connect: ${this.options.host}:${this.options.port}`
      );

      const connectPromise = promisify(this.socket.connect.bind(this.socket));
      await connectPromise(this.options);

      console.log("Connected to Modbus");
    } catch (error) {
      console.error("Error while connecting to Modbus:", error);
      throw error;
    }
  }

  async readHoldingRegisters(start, count) {
    try {
      const response = await this.client.readHoldingRegisters(start, count);
      return response.response._body._valuesAsArray;
    } catch (error) {
      console.error("Error while reading holding registers:", error);
      throw error;
    }
  }

  async writeSingleRegister(address, value) {
    try {
      const response = await this.client.writeSingleRegister(address, value);
      return response;
    } catch (error) {
      console.error("Error while writing single register:", error);
      throw error;
    }
  }

  async readCoils(start, count) {
    try {
      const response = await this.client.readCoils(start, count);
      for (let i = 0; i < response.response._body._valuesAsArray.length; i++) {
        if (response.response._body._valuesAsArray[i] === 1) {
          response.response._body._valuesAsArray[i] = true;
        } else {
          response.response._body._valuesAsArray[i] = false;
        }
      }
      return response.response._body._valuesAsArray;
    } catch (error) {
      console.error("Error while reading coils:", error);
      throw error;
    }
  }

  async writeSingleCoil(address, value) {
    try {
      const response = await this.client.writeSingleCoil(address, value);
      return response;
    } catch (error) {
      console.error("Error while writing single coil:", error);
      throw error;
    }
  }

  async close() {
    this.client.socket.end();
  }
}

module.exports = { JsModbusConnection };

redis/index.js

const plc = require('../plc/handlers/handlers');
const redisClient = require('./redisClient');

const getDeviceState = async () => {
    const devicesStates = await plc.getDevicesStates();
    await redisClient.hSet('devicesStates', devicesStates);
};

const timeout = ms => new Promise(resolve => setTimeout(resolve, ms));

const startGettingData = async () => {
    try {
        await getDeviceMode();
        await getDevicesRemoteMode();
        await getDeviceState();
        await getWindowsCurrentValues();
        await paramStates();
        await paramsConfigs();
        await timeout(constants.getParamsStatesInterval);
        console.log(new Date().toISOString(), 'startGettingData');
        return await startGettingData();
    } catch (error) {
        console.log(new Date().toISOString(), 'startGettingData', error.message, error.errno);
        await reconnect(error.errno);
        return await startGettingData();
    }
};

const reconnect = async (errorType) => {
    if (errorType !== 'ETIMEDOUT') {
        console.log('restart server')
        process.exit(1);
    }
};

module.exports = {
    startGettingData,
};

handlers.js – getting several register values at once

getDevicesStates: async () => {
    const deviceStateRegs = regTypeM_RW.deviceState;
    let devicesStates = {};
    // NOTE: Group windows related: holding registers
    let modbusRes = await modbus.client.readHoldingRegisters(min.num, (max.num - min.num + 1));
    for (let i = 0; i < modbusRes.data.length; i++) {
        const curRegNum = min.num + i;
        const foundRegs = windowsDeviceStates.filter(reg => reg.num === curRegNum);
        for (let j = 0; j < foundRegs.length; j++) {
            const deviceName = foundRegs[j].name;
            devicesStates[deviceName] = modbusRes.data[i];
        }
    }
    await timeout(1000);
    // NOTE: Group not windows related: coils
    await timeout(1000);
    return devicesStates;
}

handlers.js – getting one by one example

getDevicesStates: async () => {
    const deviceStateRegs = regTypeM_RW.deviceState;
    let devicesStates = {};
    for (let i = 0; i < deviceStateRegs.length; i++) {
        const deviceName = deviceStateRegs[i].name;
        const deviceNum = deviceStateRegs[i].num;
        let deviceState = {
            data: []
        };
        if (deviceName.includes('window')) {
            deviceState = await modbus.client.readHoldingRegisters(deviceNum, constants.registersReadLength);
        } else {
            deviceState = await modbus.client.readCoils(deviceNum, constants.registersReadLength);
        }
        devicesStates[deviceName] = deviceState.data[0];
        await timeout(1000);
    }
    return devicesStates;
}

Could you kindly provide me with guidance on the appropriate direction to dig? Is it something related to latency or PLC capabilities? Or maybe there are some subtleties which are not considered in the code.

[stefanpoeter]

You might want to format the code here so that it is readable. Then try to create a minimal reproducable example. Use the Modbus/TCP Server to see if it happens there too.

[YuriKhachatryan]

I formatted the code

[stefanpoeter]

Still too much code to look for an error. To me this doesn't seem to be related to the client logic. The line return await startGettingData(); doesn't seem healthy. Doesn't this stack up over time?

[aquiot-inc]

I noticed that there is a similar posting at yaacov/node-modbus-serial#502

I have had a problem with node-red-contrib-modbus (based on node-modbus) being "hung" after a number of days of running continuously (doing Modbus TCP only, no serial RTU). It's difficult to reproduce/debug because it's running in NodeRed and it takes days to show up. So I'm wondering if I may be be seeing something related to this.

[g-langlois]

How many times do you call readHoldingRegisters before this happens? Are you using this over an intermittent internet connection?

I have a very similar issue (ECONNREFUSED) and I have never been able to get to the bottom of it. The error rate is about 1 out of 100,000 and is impossible to replicate. I spent weeks troubleshooting it and at this point I am confident the issue isn't in my code. In my case it looks like a socket issue and its unclear if its caused by this library, node itself or the PLC.

[aquiot-inc]

How many times do you call readHoldingRegisters before this happens? Are you using this over an intermittent internet connection?

I have a very similar issue (ECONNREFUSED) and I have never been able to get to the bottom of it. The error rate is about 1 out of 100,000 and is impossible to replicate. I spent weeks troubleshooting it and at this point I am confident the issue isn't in my code. In my case it looks like a socket issue and its unclear if its caused by this library, node itself or the PLC.

My error condition is also after hundreds of thousands, even millions of Modbus reads, takes days to happen. Currently running with >10 million reads and only 71 reads failed (which is fine, I use a very short timeout with no retry) and without having the flow crash.

In my environment there is stable connection: all connected to a local Ethernet switch, not over WAN.

As an experiment, I was going to try similar Modbus polling using NodeJS directly (not Node Red). See if that can run reliably.

BTW @g-langlois: What hardware environment are you running in?

[stefanpoeter]

Has any of you tried to setup a test environment with debug logs enabled? I am curious if this is related to this library?