Avoid potential integer overflow in Mmap.mmap (#41186)

There is a potential integer overflow in Mmap.mmap which can lead to an
out-of-bounds access. The size of the memory-mapped array `len` is
calculated as `prod(dims)`. If this multiplication overflows, the
allocated size will be too small and accesses towards the end of the
array will fail with e.g. a segfault or other errors. I noticed this
when using `dims` taken from a binary file header in UInt32 format.


To fix, use overflow-aware multiplication to determine the size of the
mmapped array and throw an error in case of overflow.

Author: kkretschmer

stdlib/Mmap/src/Mmap.jl

@@ -191,7 +191,11 @@ function mmap(io::IO,
     isopen(io) || throw(ArgumentError("$io must be open to mmap"))
     isbitstype(T)  || throw(ArgumentError("unable to mmap $T; must satisfy isbitstype(T) == true"))
 
-    len = prod(dims) * sizeof(T)
+    len = sizeof(T)
+    for l in dims
+        len, overflow = Base.Checked.mul_with_overflow(promote(len, l)...)
+        overflow && throw(ArgumentError("requested size prod($((sizeof(T), dims...))) too large, would overflow typeof(size(T)) == $(typeof(len))"))
+    end
     len >= 0 || throw(ArgumentError("requested size must be ≥ 0, got $len"))
     len == 0 && return Array{T}(undef, ntuple(x->0,Val(N)))
     len < typemax(Int) - PAGESIZE || throw(ArgumentError("requested size must be < $(typemax(Int)-PAGESIZE), got $len"))

stdlib/Mmap/test/runtests.jl

@@ -133,6 +133,7 @@ c = mmap(s, Vector{UInt8}, (UInt16(11),))
 finalize(c); c=nothing; GC.gc()
 @test_throws ArgumentError mmap(s, Vector{UInt8}, (Int16(-11),))
 @test_throws ArgumentError mmap(s, Vector{UInt8}, (typemax(UInt),))
+@test_throws ArgumentError mmap(s, Matrix{UInt8}, (typemax(Int) - Mmap.PAGESIZE - 1, 2)) # overflow
 close(s)
 s = open(file, "r+")
 @test isreadonly(s) == false