fix rem2pi for non-finite arguments (#46163)

Author: oscardssmith

base/math.jl

@@ -1239,7 +1239,7 @@ julia> rem2pi(7pi/4, RoundDown)
 """
 function rem2pi end
 function rem2pi(x::Float64, ::RoundingMode{:Nearest})
-    isnan(x) && return NaN
+    isfinite(x) || return NaN
 
     abs(x) < pi && return x
 
@@ -1264,7 +1264,7 @@ function rem2pi(x::Float64, ::RoundingMode{:Nearest})
     end
 end
 function rem2pi(x::Float64, ::RoundingMode{:ToZero})
-    isnan(x) && return NaN
+    isfinite(x) || return NaN
 
     ax = abs(x)
     ax <= 2*Float64(pi,RoundDown) && return x
@@ -1291,7 +1291,7 @@ function rem2pi(x::Float64, ::RoundingMode{:ToZero})
     copysign(z,x)
 end
 function rem2pi(x::Float64, ::RoundingMode{:Down})
-    isnan(x) && return NaN
+    isfinite(x) || return NaN
 
     if x < pi4o2_h
         if x >= 0
@@ -1322,7 +1322,7 @@ function rem2pi(x::Float64, ::RoundingMode{:Down})
     end
 end
 function rem2pi(x::Float64, ::RoundingMode{:Up})
-    isnan(x) && return NaN
+    isfinite(x) || return NaN
 
     if x > -pi4o2_h
         if x <= 0

test/numbers.jl

@@ -2676,10 +2676,12 @@ end
 end
 
 @testset "PR #36420 $T" for T in (Float16, Float32, Float64)
-    @test rem2pi(T(NaN), RoundToZero)  === T(NaN)
-    @test rem2pi(T(NaN), RoundNearest) === T(NaN)
-    @test rem2pi(T(NaN), RoundDown)    === T(NaN)
-    @test rem2pi(T(NaN), RoundUp)      === T(NaN)
+    for r in (RoundToZero, RoundNearest, RoundDown, RoundUp)
+        for x in (Inf, -Inf, NaN, -NaN)
+            @test isnan(rem2pi(T(x), r))
+            @test rem2pi(T(x), r) isa T
+        end
+    end
 end
 
 import Base.^