Try to avoid aborting when producing huge render output

The cmark_render_* functions may now return NULL instead of aborting
if the output exceeds the maximum internal buffer size.

This fix isn't bullet-proof but should help with smaller documents
that trigger quadratic output growth with the Commonmark renderer.
See commonmark#377.

src/buffer.c

@@ -41,10 +41,10 @@ void cmark_strbuf_grow(cmark_strbuf *buf, bufsize_t target_size) {
  if (target_size < buf->asize)
  return;
 
- if (target_size > (bufsize_t)(INT32_MAX / 2)) {
+ if (target_size > CMARK_BUF_MAX) {
  fprintf(stderr,
  "[cmark] cmark_strbuf_grow requests buffer with size > %d, aborting\n",
-  (INT32_MAX / 2));
+ CMARK_BUF_MAX);
  abort();
  }
 

src/buffer.h

@@ -15,6 +15,8 @@ extern "C" {
 
 typedef int32_t bufsize_t;
 
+#define CMARK_BUF_MAX (INT32_MAX / 2)
+
 typedef struct {
  cmark_mem *mem;
  unsigned char *ptr;

src/cmark.h

@@ -507,31 +507,36 @@ cmark_node *cmark_parse_file(FILE *f, int options);
 
 /** Render a 'node' tree as XML. It is the caller's responsibility
  * to free the returned buffer.
+ * May return NULL if internal resource limits are exceeded.
  */
 CMARK_EXPORT
 char *cmark_render_xml(cmark_node *root, int options);
 
 /** Render a 'node' tree as an HTML fragment. It is up to the user
  * to add an appropriate header and footer. It is the caller's
  * responsibility to free the returned buffer.
+ * May return NULL if internal resource limits are exceeded.
  */
 CMARK_EXPORT
 char *cmark_render_html(cmark_node *root, int options);
 
 /** Render a 'node' tree as a groff man page, without the header.
  * It is the caller's responsibility to free the returned buffer.
+ * May return NULL if internal resource limits are exceeded.
  */
 CMARK_EXPORT
 char *cmark_render_man(cmark_node *root, int options, int width);
 
 /** Render a 'node' tree as a commonmark document.
  * It is the caller's responsibility to free the returned buffer.
+ * May return NULL if internal resource limits are exceeded.
  */
 CMARK_EXPORT
 char *cmark_render_commonmark(cmark_node *root, int options, int width);
 
 /** Render a 'node' tree as a LaTeX document.
  * It is the caller's responsibility to free the returned buffer.
+ * May return NULL if internal resource limits are exceeded.
  */
 CMARK_EXPORT
 char *cmark_render_latex(cmark_node *root, int options, int width);

src/main.c

@@ -69,6 +69,10 @@ static void print_document(cmark_node *document, writer_format writer,
  fprintf(stderr, "Unknown format %d\n", writer);
  exit(1);
  }
+ if (result == NULL) {
+ fprintf(stderr, "Internal resource limit exceeded\n");
+ exit(1);
+ }
  printf("%s", result);
  document->mem->free(result);
 }

src/render.c

@@ -159,7 +159,7 @@ char *cmark_render(cmark_node *root, int options, int width,
  cmark_strbuf buf = CMARK_BUF_INIT(mem);
  cmark_node *cur;
  cmark_event_type ev_type;
- char *result;
+ char *result = NULL;
  cmark_iter *iter = cmark_iter_new(root);
 
  cmark_renderer renderer = {options,
@@ -176,6 +176,10 @@ char *cmark_render(cmark_node *root, int options, int width,
  // autolinks.
  cmark_iter_reset(iter, cur, CMARK_EVENT_EXIT);
  }
+ // try to avoid exceeding the maximum strbuf size
+ if (renderer.buffer->size > CMARK_BUF_MAX / 2) {
+ goto error;
+ }
  }
 
  // ensure final newline
@@ -185,6 +189,7 @@ char *cmark_render(cmark_node *root, int options, int width,
 
  result = (char *)cmark_strbuf_detach(renderer.buffer);
 
+error:
  cmark_iter_free(iter);
  cmark_strbuf_free(renderer.prefix);
  cmark_strbuf_free(renderer.buffer);