calloc: check multiplication overflow

2 files changed, 13 insertions, 8 deletions

diff --git a/sys/src/ape/lib/ap/gen/calloc.c b/sys/src/ape/lib/ap/gen/calloc.c
index e52210d84..c080c7b3d 100644
--- a/sys/src/ape/lib/ap/gen/calloc.c
+++ b/sys/src/ape/lib/ap/gen/calloc.c
@@ -2,12 +2,14 @@
 #include <string.h>
 
 void *
-calloc(size_t nmemb, size_t size)
+calloc(size_t n, size_t s)
 {
-	void *mp;
+	void *v;
 
-	nmemb = nmemb*size;
-	if(mp = malloc(nmemb))
-		memset(mp, 0, nmemb);
-	return(mp);
+	if(n > 1 && ((size_t)-1)/n < s)
+		return 0;
+	n *= s;
+	if(v = malloc(n))
+		memset(v, 0, n);
+	return v;
 }
diff --git a/sys/src/libc/port/malloc.c b/sys/src/libc/port/malloc.c
index 741316926..e23e1f53b 100644
--- a/sys/src/libc/port/malloc.c
+++ b/sys/src/libc/port/malloc.c
@@ -280,10 +280,13 @@ msize(void *v)
 }
 
 void*
-calloc(ulong n, ulong szelem)
+calloc(ulong n, ulong s)
 {
 	void *v;
-	if(v = mallocz(n*szelem, 1))
+
+	if(n > 1 && ((ulong)-1)/n < s)
+		return nil;
+	if(v = mallocz(n*s, 1))
 		setmalloctag(v, getcallerpc(&n));
 	return v;
 }