Issue with nested device arrays

[SimonDanisch]

is the below supposed a bug, incorrectly implemented by me, or just not possible?

using KernelAbstractions
using pocl_jll, OpenCL
using OpenCL: CLArray

cl.device!(cl.devices(cl.platforms()[1])[1])

backend = OpenCL.OpenCLBackend()

KernelAbstractions.@kernel _dummy_kernel() = nothing
function to_device(backend, arr::CLArray)
    kernel = _dummy_kernel(backend)
    return KernelAbstractions.argconvert(kernel, arr)
end

# A composite element type (like RGBSpectrum)
struct MyColor
    c::NTuple{4, Float32}
end
MyColor(r, g, b) = MyColor((r, g, b, 1.0f0))

# A wrapper type (like Texture)
struct MyTexture{T}
    data::T
    default::MyColor
    isconst::Bool
end

# A material type (like MatteMaterial)
struct MyMaterial{T}
    albedo::MyTexture{T}
end

# Create test data
cpu_colors = [MyColor(1f0, 0f0, 0f0) MyColor(0f0, 1f0, 0f0);
              MyColor(0f0, 0f0, 1f0) MyColor(1f0, 1f0, 1f0)]
gpu_colors = CLArray(cpu_colors)

# Create material with GPU texture
gpu_tex = MyTexture(to_device(backend, gpu_colors), MyColor(0f0, 0f0, 0f0), false)
gpu_mat = MyMaterial(gpu_tex)

# Store material in a vector - THIS IS THE KEY
gpu_mats = CLArray([gpu_mat])

# Kernel that accesses nested data
@kernel function mwe_kernel!(result, materials)
    i = @index(Global)
    mat = materials[1]           # Extract material from vector
    tex = mat.albedo             # Get texture wrapper
    data = tex.data              # Get device matrix
    elem = data[1, 1]            # Get element (MyColor)
    result[i] = elem.c[1]        # Access field of element - FAILS
end

kernel! = mwe_kernel!(backend)
result = CLArray([0.0f0])

kernel!(result, gpu_mats; ndrange=1)

OpenCL.Device(gfx1100 on AMD Accelerated Parallel Processing @0x000000000f75e3d0)

error: line 234: Result type (OpTypePointer) does not match the type that results from indexing into the composite (OpTypePointer).
  %_fca_0_0_0_extract = OpCompositeExtract %_ptr_CrossWorkgroup_float %145 0 0

ERROR: Failed to validate generated SPIR-V.
If you think this is a bug, please file an issue and attach /tmp/jl_PE7wxaHevW.bc and /tmp/jl_rwR6vK5ABG.spv.
Stacktrace:
  [1] error(s::String)
    @ Base ./error.jl:44
  [2] mcgen(job::GPUCompiler.CompilerJob{…}, mod::LLVM.Module, format::LLVM.API.LLVMCodeGenFileType)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/spirv.jl:184
  [3] emit_asm(job::GPUCompiler.CompilerJob, ir::LLVM.Module, format::LLVM.API.LLVMCodeGenFileType)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:438
  [4] compile_unhooked(output::Symbol, job::GPUCompiler.CompilerJob; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:115
  [5] compile_unhooked
    @ ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:80 [inlined]
  [6] compile(target::Symbol, job::GPUCompiler.CompilerJob; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:67
  [7] compile
    @ ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:55 [inlined]
  [8] (::OpenCL.var"#compile##0#compile##1"{GPUCompiler.CompilerJob{…}})(ctx::LLVM.Context)
    @ OpenCL ~/.julia/packages/OpenCL/kWFms/src/compiler/compilation.jl:183
  [9] JuliaContext(f::OpenCL.var"#compile##0#compile##1"{GPUCompiler.CompilerJob{…}}; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:34
 [10] JuliaContext(f::Function)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:25
 [11] compile(job::GPUCompiler.CompilerJob)
    @ OpenCL ~/.julia/packages/OpenCL/kWFms/src/compiler/compilation.jl:182
 [12] actual_compilation(cache::Dict{…}, src::Core.MethodInstance, world::UInt64, cfg::GPUCompiler.CompilerConfig{…}, compiler::typeof(OpenCL.compile), linker::typeof(OpenCL.link))
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/execution.jl:245
 [13] cached_compilation(cache::Dict{…}, src::Core.MethodInstance, cfg::GPUCompiler.CompilerConfig{…}, compiler::Function, linker::Function)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/execution.jl:159
 [14] macro expansion
    @ ~/.julia/packages/OpenCL/kWFms/src/compiler/execution.jl:191 [inlined]
 [15] macro expansion
    @ ./lock.jl:376 [inlined]
 [16] clfunction(f::typeof(gpu_mwe_kernel!), tt::Type{Tuple{…}}; kwargs::@Kwargs{})
    @ OpenCL ~/.julia/packages/OpenCL/kWFms/src/compiler/execution.jl:186
 [17] clfunction(f::typeof(gpu_mwe_kernel!), tt::Type{Tuple{…}})
    @ OpenCL ~/.julia/packages/OpenCL/kWFms/src/compiler/execution.jl:182
 [18] macro expansion
    @ ~/.julia/packages/OpenCL/kWFms/src/compiler/execution.jl:66 [inlined]
 [19] (::KernelAbstractions.Kernel{…})(::CLArray{…}, ::Vararg{…}; ndrange::Int64, workgroupsize::Nothing)
    @ OpenCL.OpenCLKernels ~/.julia/packages/OpenCL/kWFms/src/OpenCLKernels.jl:104
 [20] top-level scope
    @ /sim/Programmieren/RayTracing/cl_compilation.jl:59

caused by: failed process: Process(setenv(`/home/simon/.julia/artifacts/8769e627f0911e0e6ab9b58f5961768a06de3e61/bin/spirv-val /tmp/jl_rwR6vK5ABG.spv`,["PATH=/home/simon/.julia/artifacts/8769e627f0911e0e6ab9b58f5961768a06de3e61/bin:/home/simon/.config/Code/User/globalStorage/github.copilot-chat/debugCommand:/home/simon/.config/Code/User/globalStorage/github.copilot-chat/copilotCli:/home/simon/.local/bin:/usr/local/bin:/usr/bin:/bin:/snap/bin", "ENV=/usr/share/Modules/init/profile.sh", "CPU=x86_64", "JRE_HOME=/usr/lib64/jvm/java-21-openjdk-21", "FROM_HEADER=", "G_FILENAME_ENCODING=@locale,UTF-8,ISO-8859-15,CP1252", "NO_AT_BRIDGE=1", "XNLSPATH=/usr/share/X11/nls", "POCL_ARGS_CLANG=-fuse-ld=lld;--ld-path=/home/simon/.julia/scratchspaces/627d6b7a-bbe6-5189-83e7-98cc0a5aeadd/bin/lld;-L;/home/simon/.julia/artifacts/dbb2d45cf83dd8f12f3ad2a436243a1ac76cb71a/share/lib;-L;/home/simon/.julia/scratchspaces/627d6b7a-bbe6-5189-83e7-98cc0a5aeadd/lib", "LD_LIBRARY_PATH=/home/simon/.julia/artifacts/8769e627f0911e0e6ab9b58f5961768a06de3e61/lib:/home/simon/.julia/juliaup/julia-1.12.4+0.x64.linux.gnu/bin/../lib/julia:/home/simon/.julia/juliaup/julia-1.12.4+0.x64.linux.gnu/bin/../lib"  …  "GTK2_RC_FILES=/etc/gtk-2.0/gtkrc:/home/simon/.gtkrc-2.0:/home/simon/.config/gtkrc-2.0", "PYTHONSTARTUP=/etc/pythonstart", "MORE=-sl", "GMAIL_APP_PASSWORD=rvrw cdbm hqfw cuae", "POCL_PATH_LLVM_SPIRV=/home/simon/.julia/scratchspaces/627d6b7a-bbe6-5189-83e7-98cc0a5aeadd/bin/llvm-spirv", "AUDIODRIVER=pulseaudio", "LS_COLORS=", "HOSTTYPE=x86_64", "KDE_SESSION_VERSION=6", "OPENBLAS_MAIN_FREE=1"]), ProcessExited(1)) [1]

Stacktrace:
  [1] pipeline_error
    @ ./process.jl:597 [inlined]
  [2] run(::Cmd; wait::Bool)
    @ Base ./process.jl:512
  [3] run
    @ ./process.jl:509 [inlined]
  [4] mcgen(job::GPUCompiler.CompilerJob{…}, mod::LLVM.Module, format::LLVM.API.LLVMCodeGenFileType)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/spirv.jl:182
  [5] emit_asm(job::GPUCompiler.CompilerJob, ir::LLVM.Module, format::LLVM.API.LLVMCodeGenFileType)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:438
  [6] compile_unhooked(output::Symbol, job::GPUCompiler.CompilerJob; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:115
  [7] compile_unhooked
    @ ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:80 [inlined]
  [8] compile(target::Symbol, job::GPUCompiler.CompilerJob; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:67
  [9] compile
    @ ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:55 [inlined]
 [10] (::OpenCL.var"#compile##0#compile##1"{GPUCompiler.CompilerJob{…}})(ctx::LLVM.Context)
    @ OpenCL ~/.julia/packages/OpenCL/kWFms/src/compiler/compilation.jl:183
 [11] JuliaContext(f::OpenCL.var"#compile##0#compile##1"{GPUCompiler.CompilerJob{…}}; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:34
 [12] JuliaContext(f::Function)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/driver.jl:25
 [13] compile(job::GPUCompiler.CompilerJob)
    @ OpenCL ~/.julia/packages/OpenCL/kWFms/src/compiler/compilation.jl:182
 [14] actual_compilation(cache::Dict{…}, src::Core.MethodInstance, world::UInt64, cfg::GPUCompiler.CompilerConfig{…}, compiler::typeof(OpenCL.compile), linker::typeof(OpenCL.link))
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/execution.jl:245
 [15] cached_compilation(cache::Dict{…}, src::Core.MethodInstance, cfg::GPUCompiler.CompilerConfig{…}, compiler::Function, linker::Function)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/j4HFa/src/execution.jl:159
 [16] macro expansion
    @ ~/.julia/packages/OpenCL/kWFms/src/compiler/execution.jl:191 [inlined]
 [17] macro expansion
    @ ./lock.jl:376 [inlined]
 [18] clfunction(f::typeof(gpu_mwe_kernel!), tt::Type{Tuple{…}}; kwargs::@Kwargs{})
    @ OpenCL ~/.julia/packages/OpenCL/kWFms/src/compiler/execution.jl:186
 [19] clfunction(f::typeof(gpu_mwe_kernel!), tt::Type{Tuple{…}})
    @ OpenCL ~/.julia/packages/OpenCL/kWFms/src/compiler/execution.jl:182
 [20] macro expansion
    @ ~/.julia/packages/OpenCL/kWFms/src/compiler/execution.jl:66 [inlined]
 [21] (::KernelAbstractions.Kernel{…})(::CLArray{…}, ::Vararg{…}; ndrange::Int64, workgroupsize::Nothing)
    @ OpenCL.OpenCLKernels ~/.julia/packages/OpenCL/kWFms/src/OpenCLKernels.jl:104
 [22] top-level scope
    @ /sim/Programmieren/RayTracing/cl_compilation.jl:59
Some type information was truncated. Use `show(err)` to see complete types.

The same does work with AMDGPU, pointing to an OpenCL bug.

[SimonDanisch]

Oh god, this seems to be a missing inbounds.... Still a pretty weird error I suppose.
Should we leave this open to throw better errors, or is that unrealistic to achieve?

[simeonschaub]

Thanks for reporting! Right now we don't have proper error handling, so this is unfortunately expected. #328 should eventually address this, but it's currently blocked due to SPIR-V not having a proper trap