Add rBoot partition support

.gitmodules

@@ -63,7 +63,7 @@
  ignore = dirty
 [submodule "rboot"]
  path = Sming/Components/rboot/rboot
- url = https://github.com/raburton/rboot.git
+ url = https://github.com/mikee47/rboot
  ignore = dirty
 [submodule "spiffs"]
  path = Sming/Components/spiffs/spiffs

Sming/Arch/Esp8266/Components/esp8266/startup.cpp

@@ -55,9 +55,8 @@ extern "C" void WEAK_ATTR user_rf_pre_init(void)
 
 extern "C" uint32 ICACHE_FLASH_ATTR WEAK_ATTR user_rf_cal_sector_set(void)
 {
- // RF calibration stored in last sector of sysParam
- auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
- return ((sysParam.address() + sysParam.size()) / SPI_FLASH_SEC_SIZE) - 1;
+ auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal);
+ return rfCal.address();
 }
 
 #ifdef SDK_INTERNAL
@@ -71,16 +70,14 @@ extern "C" void ICACHE_FLASH_ATTR WEAK_ATTR user_pre_init(void)
  Storage::initialize();
 
  auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
+ auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal);
  auto phy = *Storage::findPartition(Storage::Partition::SubType::Data::phy);
 
- // RF calibration stored in last sector of sysParam
- auto sysParamSize = sysParam.size() - SPI_FLASH_SEC_SIZE;
-
  static const partition_item_t partitions[] = {
  {SYSTEM_PARTITION_BOOTLOADER, 0, SPI_FLASH_SEC_SIZE},
  {SYSTEM_PARTITION_PHY_DATA, phy.address(), phy.size()},
- {SYSTEM_PARTITION_SYSTEM_PARAMETER, sysParam.address(), sysParamSize},
- {SYSTEM_PARTITION_RF_CAL, sysParam.address() + sysParamSize, SPI_FLASH_SEC_SIZE},
+ {SYSTEM_PARTITION_SYSTEM_PARAMETER, sysParam.address(), sysParam.size()},
+ {SYSTEM_PARTITION_RF_CAL, rfCal.address(), rfCal.size()},
  };
 
  enum flash_size_map sizeMap = system_get_flash_size_map();

Sming/Arch/Esp8266/options.json

@@ -6,5 +6,23 @@
  "filename": "$(FLASH_INIT_DATA_VCC)"
  }
  }
+ },
+ "alternate": {
+ "description": "ESP8266 layout with critical partitions at start of flash",
+ "partition_table_offset": "0x2000",
+ "partitions": {
+ "rf_cal": {
+ "address": "0x3000"
+ },
+ "phy_init": {
+ "address": "0x4000"
+ },
+ "sys_param": {
+ "address": "0x5000"
+ },
+ "rom0": {
+ "address": "0x8000"
+ }
+ }
  }
 }

Sming/Arch/Esp8266/spiffs-two-roms.hw

@@ -2,11 +2,14 @@
  "name": "Two ROM slots with single SPIFFS",
  "base_config": "spiffs",
  "partitions": {
+ "rom0": {
+ "subtype": "ota_0"
+ },
  "rom1": {
  "address": "0x108000",
  "size": "992K",
  "type": "app",
- "subtype": "ota_0",
+ "subtype": "ota_1",
  "filename": "$(RBOOT_ROM_1_BIN)"
  }
  }

Sming/Arch/Esp8266/standard.hw

@@ -2,7 +2,7 @@
  "name": "Standard config with single ROM",
  "comment": "Should work with any Esp8266 variant",
  "arch": "Esp8266",
- "partition_table_offset": "0x2000",
+ "partition_table_offset": "self.devices[0].size - 0x6000",
  "devices": {
  "spiFlash": {
  "type": "flash",
@@ -12,25 +12,31 @@
  }
  },
  "partitions": {
+ "rom0": {
+ "address": "0x002000",
+ "size": "992K",
+ "type": "app",
+ "subtype": "factory",
+ "filename": "$(RBOOT_ROM_0_BIN)"
+ },
+ "rf_cal": {
+ "address": "self.device.size - 0x5000",
+ "size": "4K",
+ "type": "data",
+ "subtype": "rfcal"
+ },
  "phy_init": {
- "address": "0x003000",
+ "address": "self.device.size - 0x4000",
  "size": "4K",
  "type": "data",
  "subtype": "phy",
  "filename": "$(FLASH_INIT_DATA)"
  },
  "sys_param": {
- "address": "0x004000",
- "size": "16K",
+ "address": "self.device.size - 0x3000",
+ "size": "12K",
  "type": "data",
  "subtype": "sysparam"
- },
- "rom0": {
- "address": "0x008000",
- "size": "992K",
- "type": "app",
- "subtype": "factory",
- "filename": "$(RBOOT_ROM_0_BIN)"
  }
  }
-}
+}

Sming/Arch/Esp8266/two-rom-mode.hw

@@ -3,13 +3,14 @@
  "base_config": "standard",
  "partitions": {
  "rom0": {
+ "subtype": "ota_0",
  "size": "480K"
  },
  "rom1": {
  "address": "0x80000",
- "size": "512K",
+ "size": "488K",
  "type": "app",
- "subtype": "ota_0",
+ "subtype": "ota_1",
  "filename": "$(RBOOT_ROM_1_BIN)"
  }
  }

Sming/Components/Storage/README.rst

@@ -84,6 +84,16 @@ If using this approach, remember to updated your project's ``component.mk`` with
 and verify the layout is correct using ``make map``.
 
 
+OTA updates
+-----------
+
+When planning OTA updates please check that the displayed partition map corresponds to your project.
+For example, the partition table requires a free sector so must not overlap other partitions.
+
+Your OTA update process must include a step to write the partition table to the correct location.
+
+It is not necessary to update the bootloader. See :component:`rboot` for further information.
+
 
 Custom configurations
 ---------------------
@@ -173,6 +183,19 @@ To customise the hardware configuration for a project, for example 'my_project':
  This will flash everything: bootloader, partition table and all defined partitions (those with a ``filename`` entry).
 
 
+.. note::
+
+ The build system isn't smart enough to track dependencies for partition build targets.
+
+ To rebuild these manually type::
+
+ make partbuild
+
+ These will be removed when ``make clean`` is run, but you can also clean them separately thus::
+
+ make part-clean
+
+
 Partition maps
 --------------
 
@@ -356,7 +379,7 @@ you can take advantage of the partition API to manage them as follows:
 - Create an instance of your custom device and make a call to :cpp:func:`Storage::registerDevice`
  in your ``init()`` function (or elsewhere if more appropriate).
 
- 
+
 API
 ---
 

Sming/Components/Storage/Tools/hwconfig/config.py

@@ -46,6 +46,7 @@ def from_name(cls, name):
  config.load(name)
  if options != '':
  config.parse_options(options.split(','))
+ config.resolve_expressions()
  config.partitions.sort()
  return config
 
@@ -73,6 +74,9 @@ def parse_options(self, options):
  temp.pop('description', None)
  self.parse_dict(temp)
 
+ def resolve_expressions(self):
+ self.partitions.offset = eval(str(self.partitions.offset))
+
  def parse_dict(self, data):
  base_config = data.pop('base_config', None)
  if base_config is not None:
@@ -89,7 +93,7 @@ def parse_dict(self, data):
  elif k == 'arch':
  self.arch = v
  elif k == 'partition_table_offset':
- self.partitions.offset = parse_int(v)
+ self.partitions.offset = v
  elif k == 'devices':
  self.devices.parse_dict(v)
  elif k == 'comment':
@@ -132,7 +136,7 @@ def buildVars(self):
  return res
 
  def verify(self, secure):
- self.partitions.verify(self.arch, secure)
+ self.partitions.verify(self.arch, self.devices[0], secure)
 
  def map(self):
  return partition.Map(self.partitions, self.devices)

Sming/Components/Storage/Tools/hwconfig/hwconfig.py

@@ -44,7 +44,7 @@ def handle_flashcheck(args, config, part):
  for e in list:
  addr, filename = e.split('=')
  addr = int(addr, 0)
- part = config.partitions.find_by_address(addr)
+ part = config.partitions.find_by_address(config.devices[0], addr)
  if part is None:
  raise InputError("No partition contains address 0x%08x" % addr)
  if part.address != addr:
@@ -113,5 +113,5 @@ def main():
  try:
  main()
  except InputError as e:
- print(e, file=sys.stderr)
+ print("** ERROR! %s" % e, file=sys.stderr)
  sys.exit(2)

Sming/Components/Storage/Tools/hwconfig/partition.py

@@ -16,11 +16,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import struct, hashlib, storage, binascii
+import struct, hashlib, storage, binascii, copy
 from common import *
 
 MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature
 MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum
+FLASH_SECTOR_SIZE = 0x1000
 PARTITION_TABLE_SIZE = 0x1000 # Size of partition table
 PARTITION_ENTRY_SIZE = 32
 
@@ -84,6 +85,7 @@
  "nvs_keys": 0x04,
  "efuse": 0x05,
  "sysparam": 0x40,
+ "rfcal": 0x41,
  "esphttpd": 0x80,
  "fat": 0x81,
  "spiffs": 0x82,
@@ -154,6 +156,10 @@ def offset_str(self):
  def buildVars(self):
  dict = {}
  dict['PARTITION_NAMES'] = " ".join(p.name for p in self)
+ buildparts = [p for p in self if p.build is not None]
+ dict['PARTITIONS_WITH_TARGETS'] = " ".join(p.name for p in buildparts)
+ dict['PARTITION_BUILD_TARGETS'] = " ".join(p.filename for p in buildparts)
+
  for p in self:
  dict.update(p.buildVars())
  return dict
@@ -198,19 +204,28 @@ def find_by_name(self, name):
  return p
  return None
 
- def find_by_address(self, addr):
+ def find_by_address(self, device, addr):
  for p in self:
- if p.contains(addr):
+ if p.device == device and p.contains(addr):
  return p
  return None
 
- def verify(self, arch, secure):
+ def verify(self, arch, spiFlash, secure):
  """Verify partition layout
  """
  # verify each partition individually
  for p in self:
  p.verify(arch, secure)
 
+ if self.offset % FLASH_SECTOR_SIZE != 0:
+ raise InputError("Partition table offset not aligned to flash sector")
+
+ p = self.find_by_address(spiFlash, self.offset)
+ if p is None:
+ p = self.find_by_address(spiFlash, self.offset + PARTITION_TABLE_SIZE - 1)
+ if not p is None:
+ raise InputError("Partition table conflict with '%s'" % p.name)
+
  # check on duplicate name
  names = [p.name for p in self]
  duplicates = set(n for n in names if names.count(n) > 1)
@@ -224,7 +239,10 @@ def verify(self, arch, secure):
  raise InputError("Partition names must be unique")
 
  # check for overlaps
- minPartitionAddress = self.offset + PARTITION_TABLE_SIZE
+ if arch == 'Esp32':
+ minPartitionAddress = self.offset + PARTITION_TABLE_SIZE
+ else:
+ minPartitionAddress = 0x00002000
  dev = ''
  last = None
  for p in self:
@@ -345,7 +363,7 @@ def parse_dict(self, data, devices):
  if k == 'device':
  self.device = devices.find_by_name(v)
  elif k == 'address':
- self.address = parse_int(v)
+ self.address = eval(str(v))
  elif k == 'size':
  self.size = parse_int(v)
  elif k == 'filename':
@@ -523,13 +541,17 @@ def add_unused(address, last_end):
  if address > last_end + 1:
  add('(unused)', last_end + 1, address - last_end - 1)
 
+ partitions = copy.copy(table)
+
  if table.offset == 0:
  last = None
  else:
- add("Boot Sector", 0, table.offset)
- last = add("Partition Table", table.offset, PARTITION_TABLE_SIZE)
+ last = add('Boot Sector', 0, min(table.offset, partitions[0].address))
+ p = Entry(device, 'Partition Table', table.offset, PARTITION_TABLE_SIZE, 0xff, 0xff)
+ partitions.append(p)
+ partitions.sort()
 
- for p in table:
+ for p in partitions:
  if last is not None:
  if p.device != last.device:
  add_unused(last.device.size, last.end())