fix JuliaLang#39804, ABI handling of structs with String references (JuliaLang#39821)

Author: JeffBezanson

src/abi_aarch64.cpp

@@ -17,7 +17,7 @@ Type *get_llvm_vectype(jl_datatype_t *dt) const
 {
     // Assume jl_is_datatype(dt) && !jl_is_abstracttype(dt)
     // `!dt->mutabl && dt->pointerfree && !dt->haspadding && dt->nfields > 0`
-    if (dt->layout == NULL)
+    if (dt->layout == NULL || jl_is_layout_opaque(dt->layout))
         return nullptr;
     size_t nfields = dt->layout->nfields;
     assert(nfields > 0);

src/abi_ppc64le.cpp

@@ -83,7 +83,7 @@ unsigned isHFA(jl_datatype_t *ty, jl_datatype_t **ty0, bool *hva) const
     int n = 0;
     for (i = 0; i < l; i++) {
         jl_datatype_t *fld = (jl_datatype_t*)jl_field_type(ty, i);
-        if (!jl_is_datatype(fld) || ((jl_datatype_t*)fld)->layout == NULL)
+        if (!jl_is_datatype(fld) || ((jl_datatype_t*)fld)->layout == NULL || jl_is_layout_opaque(((jl_datatype_t*)fld)->layout))
             return 9;
         n += isHFA((jl_datatype_t*)fld, ty0, hva);
         if (n > 8)

src/abi_x86_64.cpp

@@ -147,11 +147,11 @@ void classifyType(Classification& accum, jl_datatype_t *dt, uint64_t offset) con
         accum.addField(offset, Sse);
     }
     // Other struct types
-    else if (jl_datatype_size(dt) <= 16) {
+    else if (jl_datatype_size(dt) <= 16 && dt->layout) {
         size_t i;
         for (i = 0; i < jl_datatype_nfields(dt); ++i) {
             jl_value_t *ty = jl_field_type(dt, i);
-            if (!jl_is_datatype(ty) || ((jl_datatype_t*)ty)->layout == NULL || jl_is_array_type(ty))
+            if (jl_field_isptr(dt, i))
                 ty = (jl_value_t*)jl_voidpointer_type;
             classifyType(accum, (jl_datatype_t*)ty, offset + jl_field_offset(dt, i));
         }

src/ccall.cpp

@@ -1011,7 +1011,7 @@ std::string generate_func_sig(const char *fname)
         abi->use_sret(jl_nothing_type);
     }
     else {
-        if (!jl_is_datatype(rt) || ((jl_datatype_t*)rt)->layout == NULL || jl_is_cpointer_type(rt) || jl_is_array_type(rt) || retboxed) {
+        if (!jl_is_datatype(rt) || ((jl_datatype_t*)rt)->layout == NULL || jl_is_layout_opaque(((jl_datatype_t*)rt)->layout) || jl_is_cpointer_type(rt) || retboxed) {
             prt = lrt; // passed as pointer
             abi->use_sret(jl_voidpointer_type);
         }
@@ -1072,7 +1072,7 @@ std::string generate_func_sig(const char *fname)
         }
 
         Type *pat;
-        if (!jl_is_datatype(tti) || ((jl_datatype_t*)tti)->layout == NULL || jl_is_array_type(tti)) {
+        if (!jl_is_datatype(tti) || ((jl_datatype_t*)tti)->layout == NULL || jl_is_layout_opaque(((jl_datatype_t*)tti)->layout)) {
             tti = (jl_value_t*)jl_voidpointer_type; // passed as pointer
         }
 

test/ccall.jl

@@ -1003,6 +1003,12 @@ unstable26078(x) = x > 0 ? x : "foo"
 handle26078 = @cfunction(unstable26078, Int32, (Int32,))
 @test ccall(handle26078, Int32, (Int32,), 1) == 1
 
+# issue #39804
+let f = @cfunction(Base.last, String, (Tuple{Int,String},))
+    # String inside a struct is a pointer even though String.size == 0
+    @test ccall(f, Ref{String}, (Tuple{Int,String},), (1, "a string?")) === "a string?"
+end
+
 # issue 17219
 function ccall_reassigned_ptr(ptr::Ptr{Cvoid})
     ptr = Libdl.dlsym(Libdl.dlopen(libccalltest), "test_echo_p")