Harden allocation failure handling and size arithmetic

- check malloc/calloc/realloc results in constructors, the JSON and
  OpenStep writers, the XML text-part helpers and base64 decoding, and
  return PLIST_ERR_NO_MEM instead of asserting or dereferencing NULL
- guard size calculations against overflow in the UTF-8/UTF-16BE
  conversions, ptrarray, and bytearray
- make hash_table_insert() report failure instead of silently dropping
  the entry; writers propagate PLIST_ERR_NO_MEM, and dict lookup caches
  are discarded so lookups fall back to a linear search
- don't overwrite an earlier parse error code in the XML parser, and
  report PLIST_ERR_NO_MEM when reading text content fails for lack of memory
- OpenStep parser: check node/data/string allocations and dict/array
  insertion results

Based on #323 by bg0d-droid.

Co-authored-by: bg0d-droid <81825610+bg0d-droid@users.noreply.github.com>

1 files changed, 34 insertions, 4 deletions

diff --git a/src/bytearray.c b/src/bytearray.c
index 89c5ec7..19303e0 100644
--- a/src/bytearray.c
+++ b/src/bytearray.c
@@ -19,6 +19,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #include <string.h>
+#include <stdint.h>
 #include "bytearray.h"
 
 #define PAGE_SIZE 4096
@@ -26,8 +27,24 @@
 bytearray_t *byte_array_new(size_t initial)
 {
 	bytearray_t *a = (bytearray_t*)malloc(sizeof(bytearray_t));
-	a->capacity = (initial > PAGE_SIZE) ? (initial+(PAGE_SIZE-1)) & (~(PAGE_SIZE-1)) : PAGE_SIZE;
+	if (!a) {
+		return NULL;
+	}
+	if (initial > PAGE_SIZE) {
+		if (initial > SIZE_MAX - (PAGE_SIZE-1)) {
+			/* rounding up to the page size would overflow */
+			free(a);
+			return NULL;
+		}
+		a->capacity = (initial+(PAGE_SIZE-1)) & (~(PAGE_SIZE-1));
+	} else {
+		a->capacity = PAGE_SIZE;
+	}
 	a->data = malloc(a->capacity);
+	if (!a->data) {
+		free(a);
+		return NULL;
+	}
 	a->len = 0;
 	a->stream = NULL;
 	return a;
@@ -36,6 +53,9 @@ bytearray_t *byte_array_new(size_t initial)
 bytearray_t *byte_array_new_for_stream(FILE *stream)
 {
 	bytearray_t *a = (bytearray_t*)malloc(sizeof(bytearray_t));
+	if (!a) {
+		return NULL;
+	}
 	a->capacity = (size_t)-1;
 	a->data = NULL;
 	a->len = 0;
@@ -57,9 +77,15 @@ void byte_array_grow(bytearray_t *ba, size_t amount)
 	if (ba->stream || !ba->data) {
 		return;
 	}
-	size_t increase = (amount > PAGE_SIZE) ? (amount+(PAGE_SIZE-1)) & (~(PAGE_SIZE-1)) : PAGE_SIZE;
-	if (increase < amount || ba->capacity + increase < ba->capacity) {
-		/* size overflow */
+	size_t increase;
+	if (amount > PAGE_SIZE) {
+		/* round up to the page size, guarding against integer overflow */
+		increase = (amount <= SIZE_MAX - (PAGE_SIZE-1)) ? ((amount+(PAGE_SIZE-1)) & (~(PAGE_SIZE-1))) : 0;
+	} else {
+		increase = PAGE_SIZE;
+	}
+	if (increase == 0 || increase > SIZE_MAX - ba->capacity) {
+		/* integer overflow */
 		free(ba->data);
 		ba->data = NULL;
 		ba->len = 0;
@@ -101,6 +127,10 @@ void byte_array_append(bytearray_t *ba, void *buf, size_t len)
 #endif
 		}
 	} else {
+		if (len > SIZE_MAX - ba->len) {
+			/* integer overflow when computing the new total length */
+			return;
+		}
 		size_t remaining = ba->capacity-ba->len;
 		if (len > remaining) {
 			size_t needed = len - remaining;