GPU CMake: Improve architecture detection, split parameter header between device and non-device options

GPU/GPUTracking/Base/cuda/CMakeLists.txt

@@ -74,7 +74,7 @@ add_custom_command(
     COMMAND cat ${GPUDIR}/Base/GPUStdSystemHeaders.h >> ${GPU_RTC_BIN}.src
     COMMAND ${CMAKE_CUDA_COMPILER} ${GPU_RTC_DEFINES} ${GPU_RTC_INCLUDES} -std=c++${CMAKE_CUDA_STANDARD} -D__CUDA_ARCH__=${RTC_CUDA_ARCH} -Wno-deprecated-gpu-targets -D__CUDACC__ -x c++ -M -MD -MT ${GPU_RTC_BIN}.src -MF ${GPU_RTC_BIN}.src.d ${GPU_RTC_SRC}
     COMMAND ${CMAKE_CUDA_COMPILER} ${GPU_RTC_DEFINES} ${GPU_RTC_INCLUDES} -std=c++${CMAKE_CUDA_STANDARD} -D__CUDA_ARCH__=${RTC_CUDA_ARCH} -Wno-deprecated-gpu-targets -D__CUDACC__ -x c++ -E -Xcompiler "-nostdinc -P" ${GPU_RTC_SRC} >> ${GPU_RTC_BIN}.src
-    DEPENDS ${GPU_RTC_SRC} ${GPUDIR}/Base/GPUStdSystemHeaders.h ${GPUDIR}/Base/cuda/GPUReconstructionCUDAIncludesSystem.h ${GPUDIR}/Base/GPUStdSystemHeaders.h GPU_PARAM_HEADER_AUTO_ALL
+    DEPENDS ${GPU_RTC_SRC} ${GPUDIR}/Base/GPUStdSystemHeaders.h ${GPUDIR}/Base/cuda/GPUReconstructionCUDAIncludesSystem.h ${GPUDIR}/Base/GPUStdSystemHeaders.h
     DEPFILE ${GPU_RTC_BIN}.src.d
     COMMAND_EXPAND_LISTS
     COMMENT "Preparing CUDA RTC source file ${GPU_RTC_BIN}.src"

GPU/GPUTracking/Base/hip/CMakeLists.txt

@@ -125,7 +125,7 @@ add_custom_command(
     COMMAND cat ${GPUDIR}/Base/hip/GPUReconstructionHIPIncludesSystem.h | grep -v GPUStdSystemHeaders.h >> ${GPU_RTC_BIN}.src
     COMMAND cat ${GPUDIR}/Base/GPUStdSystemHeaders.h >> ${GPU_RTC_BIN}.src
     COMMAND ${CMAKE_HIP_COMPILER} ${GPU_RTC_DEFINES} ${GPU_RTC_INCLUDES} -std=c++${CMAKE_HIP_STANDARD} -D__HIPCC__ -D__HIP_DEVICE_COMPILE__ -x c++ -nostdinc -E -P ${GPU_RTC_SRC} -MD -MT ${GPU_RTC_BIN}.src -MF ${GPU_RTC_BIN}.src.d >> ${GPU_RTC_BIN}.src
-    DEPENDS ${GPU_RTC_SRC} ${GPUDIR}/Base/GPUStdSystemHeaders.h ${GPUDIR}/Base/hip/GPUReconstructionHIPIncludesSystem.h ${GPUDIR}/Base/GPUStdSystemHeaders.h ${MODULE}_HIPIFIED GPU_PARAM_HEADER_AUTO_ALL
+    DEPENDS ${GPU_RTC_SRC} ${GPUDIR}/Base/GPUStdSystemHeaders.h ${GPUDIR}/Base/hip/GPUReconstructionHIPIncludesSystem.h ${GPUDIR}/Base/GPUStdSystemHeaders.h ${MODULE}_HIPIFIED
     DEPFILE ${GPU_RTC_BIN}.src.d
     COMMAND_EXPAND_LISTS
     COMMENT "Preparing HIP RTC source file ${GPU_RTC_BIN}.src"

GPU/GPUTracking/CMakeLists.txt

@@ -107,31 +107,6 @@ set(SRCS_NO_H SectorTracker/GPUTPCTrackerDump.cxx
               Global/GPUChainTrackingDebugAndProfiling.cxx
               Global/GPUChainTrackingIO.cxx)
 
-if(GPUCA_OVERRIDE_PARAMETER_FILE)
-    set(GPU_PARAM_JSON ${GPUCA_OVERRIDE_PARAMETER_FILE})
-else()
-    set(GPU_PARAM_JSON ${CMAKE_CURRENT_SOURCE_DIR}/Definitions/Parameters/GPUParameters.csv)
-endif()
-set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${GPU_PARAM_JSON}")
-
-get_filename_component(GPU_PARAM_JSON_EXT ${GPU_PARAM_JSON} EXT)
-string(TOLOWER "${GPU_PARAM_JSON_EXT}" GPU_PARAM_JSON_EXT)
-if(GPU_PARAM_JSON_EXT STREQUAL .csv)
-    execute_process(
-        COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/Definitions/Parameters/csv_to_json.sh "${GPU_PARAM_JSON}"
-        OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/gpu_parameters.json
-        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-    )
-    message(STATUS "Converted ${GPU_PARAM_JSON} to ${CMAKE_CURRENT_BINARY_DIR}/gpu_parameters.json")
-    set(GPU_PARAM_JSON ${CMAKE_CURRENT_BINARY_DIR}/gpu_parameters.json)
-endif()
-
-set(ON_THE_FLY_DIR ${CMAKE_CURRENT_BINARY_DIR}/include_gpu_onthefly)
-file(MAKE_DIRECTORY ${ON_THE_FLY_DIR})
-include(cmake/gpu_param_header_generator.cmake)
-set(GPU_DEFAULT_PARAMS_HEADER ${ON_THE_FLY_DIR}/GPUDefParametersDefaults.h)
-generate_gpu_param_header("AUTO" ${GPU_DEFAULT_PARAMS_HEADER}) # generate header with default GPU parameters, arch selected by CMake variables
-
 set(HDRS_INSTALL
     ${HDRS_CINT_O2}
     ${HDRS_CINT_DATATYPES}
@@ -160,7 +135,6 @@ set(HDRS_INSTALL
     DataTypes/GPUO2ExternalUser.h
     Debug/GPUROOTDump.h
     Definitions/GPUDefConstantsAndSettings.h
-    ${GPU_DEFAULT_PARAMS_HEADER}
     Definitions/GPUDefParametersWrapper.h
     Definitions/GPUDefParametersConstants.h
     Definitions/GPUDef.h
@@ -256,6 +230,8 @@ if(ALIGPU_BUILD_TYPE STREQUAL "O2")
       DataTypes/GPUO2ConfigurableParam.cxx)
 endif()
 
+set(ON_THE_FLY_DIR ${CMAKE_CURRENT_BINARY_DIR}/include_gpu_onthefly)
+file(MAKE_DIRECTORY ${ON_THE_FLY_DIR})
 set(TEMPLATE_HEADER_LIST Base/GPUReconstructionKernelList.template.h
                          Base/GPUReconstructionKernelIncludes.template.h
                          Base/GPUReconstructionIncludesDeviceAll.template.h
@@ -286,6 +262,42 @@ add_custom_command(
 )
 list(APPEND GENERATED_HEADERS_LIST ${ON_THE_FLY_DIR}/GPUDefParametersLoadPrepare.h)
 
+if(GPUCA_OVERRIDE_PARAMETER_FILE)
+  set(GPU_PARAM_JSON ${GPUCA_OVERRIDE_PARAMETER_FILE})
+else()
+  set(GPU_PARAM_JSON ${CMAKE_CURRENT_SOURCE_DIR}/Definitions/Parameters/GPUParameters.csv)
+endif()
+set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${GPU_PARAM_JSON}")
+
+set(GPU_PARAM_JSON_FILES)
+set(GPU_PARAM_JSON_N_FILES 0)
+foreach(GPU_PARAM_JSON_FILE IN LISTS GPU_PARAM_JSON)
+  if(NOT EXISTS "${GPU_PARAM_JSON_FILE}")
+    message(FATAL_ERROR "Parameter file ${GPU_PARAM_JSON_FILE} does not exist")
+  endif()
+  get_filename_component(GPU_PARAM_JSON_EXT ${GPU_PARAM_JSON_FILE} EXT)
+  string(TOLOWER "${GPU_PARAM_JSON_EXT}" GPU_PARAM_JSON_EXT)
+  if(GPU_PARAM_JSON_EXT STREQUAL .csv)
+    get_filename_component(GPU_PARAM_JSON_NAME ${GPU_PARAM_JSON_FILE} NAME_WE)
+    set(CONVOUTFILE "GPUParameters_${GPU_PARAM_JSON_NAME}_${GPU_PARAM_JSON_N_FILES}.json")
+    execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/Definitions/Parameters/csv_to_json.sh "${GPU_PARAM_JSON_FILE}"
+                    OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/${CONVOUTFILE}
+                    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    )
+    message(STATUS "Converted ${GPU_PARAM_JSON_FILE} to ${CONVOUTFILE}")
+    list(APPEND GPU_PARAM_JSON_FILES ${CMAKE_CURRENT_BINARY_DIR}/${CONVOUTFILE})
+  else()
+    list(APPEND GPU_PARAM_JSON_FILES ${GPU_PARAM_JSON_FILE})
+  endif()
+  math(EXPR GPU_PARAM_JSON_N_FILES "${GPU_PARAM_JSON_N_FILES} + 1")
+endforeach()
+
+include(cmake/gpu_param_header_generator.cmake)
+set(GPU_DEFAULT_PARAMS_HEADER ${ON_THE_FLY_DIR}/GPUDefParametersDefaults.h)
+set(GPU_DEFAULT_PARAMS_HEADER_DEVICE ${ON_THE_FLY_DIR}/GPUDefParametersDefaultsDevice.h)
+generate_gpu_param_header("${GPU_PARAM_JSON_FILES}" "ALL" "${GPU_DEFAULT_PARAMS_HEADER}" "${GPU_DEFAULT_PARAMS_HEADER_DEVICE}" GPU_CONST_PARAM_ARCHITECTUES) # generate header with default GPU parameters, arch selected by CMake variables
+list(APPEND GENERATED_HEADERS_LIST ${GPU_DEFAULT_PARAMS_HEADER} ${GPU_DEFAULT_PARAMS_HEADER_DEVICE})
+
 set(HDRS_INSTALL ${HDRS_INSTALL} ${GENERATED_HEADERS_LIST})
 include(kernels.cmake)
 
@@ -449,20 +461,19 @@ if(CUDA_ENABLED OR OPENCL_ENABLED OR HIP_ENABLED)
     message(WARNING "GPU Tracking disabled on MacOS")
   else()
     make_directory(${CMAKE_CURRENT_BINARY_DIR}/genGPUArch)
-    set(GPU_CONST_PARAM_FILES "")
-    set(GPU_ARCH_PARAMS_HEADER ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch/GPUDefParametersDefaults_OnTheFly.h)
-    generate_gpu_param_header("ALL" ${GPU_ARCH_PARAMS_HEADER} "GPU_CONST_PARAM_ARCHITECTUES")
+    set(GPU_CONST_PARAM_FILES)
     foreach(GPU_ARCH ${GPU_CONST_PARAM_ARCHITECTUES})
       set(PARAMFILE ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch/gpu_const_param_${GPU_ARCH}.par)
       add_custom_command(
         OUTPUT ${PARAMFILE}
         COMMAND bash -c
-                "echo -e '#define GPUCA_GPUTYPE_${GPU_ARCH}\\n#define PARAMETER_FILE \"${GPU_ARCH_PARAMS_HEADER}\"\\ngInterpreter->AddIncludePath(\"${CMAKE_CURRENT_SOURCE_DIR}/Definitions\");\\ngInterpreter->AddIncludePath(\"${ON_THE_FLY_DIR}\");\\n.x ${CMAKE_CURRENT_SOURCE_DIR}/Standalone/tools/dumpGPUDefParam.C(\"${PARAMFILE}\")\\n.q\\n'"
+                "echo -e '#define GPUCA_GPUTYPE_${GPU_ARCH}\\n#define PARAMETER_FILE \"GPUDefParametersDefaults.h\"\\ngInterpreter->AddIncludePath(\"${CMAKE_CURRENT_SOURCE_DIR}/Definitions\");\\ngInterpreter->AddIncludePath(\"${ON_THE_FLY_DIR}\");\\n.x ${CMAKE_CURRENT_SOURCE_DIR}/Standalone/tools/dumpGPUDefParam.C(\"${PARAMFILE}\")\\n.q\\n'"
                 | root -l -b > /dev/null
         VERBATIM
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch
         MAIN_DEPENDENCY Standalone/tools/dumpGPUDefParam.C
-        DEPENDS ${GPU_ARCH_PARAMS_HEADER}
+        DEPENDS ${GPU_DEFAULT_PARAMS_HEADER}
+                ${GPU_DEFAULT_PARAMS_HEADER_DEVICE}
                 ${ON_THE_FLY_DIR}/GPUDefParametersLoadPrepare.h
                 ${ON_THE_FLY_DIR}/GPUDefParametersLoad.inc
         COMMENT "Generating GPU parameter set for architecture ${GPU_ARCH}")

GPU/GPUTracking/cmake/gpu_param_header_generator.cmake

@@ -12,110 +12,122 @@
 # file gpu_param_header_generator.cmake
 # author Gabriele Cimador
 
-function(generate_macros json_content output types arch_list arch_list_output)
-    foreach(arch IN LISTS arch_list)
-        set(OUTPUT_TMP_${arch} "")
-    endforeach()
-    set(arch_list_output_tmp)
-    list(FIND arch_list "ALL" do_all_architectures)
-    foreach(TYPE IN LISTS types)
-        string(JSON n_params LENGTH "${json_content}" "${TYPE}")
-        math(EXPR last "${n_params} - 1")
-        foreach(i RANGE 0 ${last})
-            string(JSON param_name MEMBER "${json_content}" "${TYPE}" "${i}")
-            string(JSON n_archs LENGTH "${JSON_CONTENT}" "${TYPE}" "${param_name}")
-            math(EXPR last_arch "${n_archs} - 1")
-            foreach(iArch RANGE 0 ${last_arch})
-                string(JSON arch MEMBER "${JSON_CONTENT}" "${TYPE}" "${param_name}" "${iArch}")
-                if(arch STREQUAL "default_cpu" AND NOT TYPE STREQUAL "PAR")
-                  message(FATAL_ERROR "Bogus entry ${param_name} for ${arch}")
-                endif()
-                if(do_all_architectures GREATER -1)
-                    if(arch_list_output AND NOT arch MATCHES ^default)
-                        list(APPEND arch_list_output_tmp "${arch}")
-                    endif()
-                    set(list_idx 0)
-                else()
-                    list(FIND arch_list "${arch}" list_idx)
-                endif()
-                if(list_idx GREATER -1)
-                    string(JSON param_values GET "${JSON_CONTENT}" "${TYPE}" "${param_name}" "${arch}")
-                    if(TYPE STREQUAL "LB")
-                        set(MACRO_NAME "GPUCA_LB_${param_name}")
-                    elseif(TYPE STREQUAL "PAR")
-                        set(MACRO_NAME "GPUCA_PAR_${param_name}")
-                    else()
-                        set(MACRO_NAME "GPUCA_${param_name}")
-                    endif()
-                    set(vals "${param_values}")
-                    string(REGEX REPLACE "^\\[ *" "" vals "${vals}")
-                    string(REGEX REPLACE " *\\]$" "" vals "${vals}")
-                    string(REGEX REPLACE "\"" "" vals "${vals}")
-                    set(MACRO_DEFINITION "#define ${MACRO_NAME} ${vals}")
-                    if(arch MATCHES ^default)
-                        # fallback defaults are wrapped in #ifndef
-                        string(APPEND OUTPUT_TMP_${arch} "#ifndef ${MACRO_NAME}\n  ${MACRO_DEFINITION}\n#endif\n\n")
-                    else()
-                        string(APPEND OUTPUT_TMP_${arch} "${MACRO_DEFINITION}\n")
-                    endif()
+function(generate_gpu_param_header GPU_PARAM_JSON_FILES ARCH_LIST OUT_HEADER OUT_HEADER_DEVICE)
+    list(FIND ARCH_LIST "ALL" do_all_architectures)
+    list(FIND ARCH_LIST "AUTO" do_auto_architectures)
+    if(do_all_architectures GREATER -1 OR do_auto_architectures GREATER -1)
+        if(do_auto_architectures GREATER -1)
+            detect_gpu_arch("AUTO")
+            list(REMOVE_ITEM ARCH_LIST "AUTO")
+        else()
+            detect_gpu_arch("ALL")
+        endif()
+        list(APPEND ARCH_LIST ${TARGET_ARCH})
+    endif()
+
+    # Types
+    set(TYPES CORE LB PAR)
+    set(ARCH_LIST_EXT "${ARCH_LIST};default;default_cpu")
+    # Per architecture definitions
+    set(JSON_ARCHITECTURES)
+
+    set(GPU_PARAM_JSON_N_FILES 0)
+    foreach(GPU_PARAM_JSON_FILE IN LISTS GPU_PARAM_JSON_FILES)
+        file(READ "${GPU_PARAM_JSON_FILE}" JSON_CONTENT)
+        foreach(TYPE IN LISTS TYPES)
+            string(JSON n_params LENGTH "${JSON_CONTENT}" "${TYPE}")
+            math(EXPR last "${n_params} - 1")
+            foreach(i RANGE 0 ${last})
+                string(JSON param_name MEMBER "${JSON_CONTENT}" "${TYPE}" "${i}")
+                string(JSON n_archs LENGTH "${JSON_CONTENT}" "${TYPE}" "${param_name}")
+                if(n_archs GREATER 0)
+                math(EXPR last_arch "${n_archs} - 1")
+                    foreach(iArch RANGE 0 ${last_arch})
+                        string(JSON arch MEMBER "${JSON_CONTENT}" "${TYPE}" "${param_name}" "${iArch}")
+                        if(arch STREQUAL "default_cpu" AND NOT TYPE STREQUAL "PAR")
+                            message(FATAL_ERROR "Bogus entry ${param_name} for ${arch}")
+                        endif()
+                        if(arch MATCHES ^default AND GPU_PARAM_JSON_N_FILES GREATER 0)
+                            message(FATAL_ERROR "Defaults must be provided in first parameter file")
+                        endif()
+                        if(do_all_architectures GREATER -1)
+                            if(NOT arch MATCHES ^default)
+                                list(APPEND JSON_ARCHITECTURES "${arch}")
+                            endif()
+                            set(list_idx 0)
+                        else()
+                            list(FIND ARCH_LIST_EXT "${arch}" list_idx)
+                        endif()
+                        if(list_idx GREATER -1)
+                            string(JSON param_values GET "${JSON_CONTENT}" "${TYPE}" "${param_name}" "${arch}")
+                            if(TYPE STREQUAL "LB")
+                                set(MACRO_NAME "GPUCA_LB_${param_name}")
+                            elseif(TYPE STREQUAL "PAR")
+                                set(MACRO_NAME "GPUCA_PAR_${param_name}")
+                            else()
+                                set(MACRO_NAME "GPUCA_${param_name}")
+                            endif()
+                            set(vals "${param_values}")
+                            string(REGEX REPLACE "^\\[ *" "" vals "${vals}")
+                            string(REGEX REPLACE " *\\]$" "" vals "${vals}")
+                            string(REGEX REPLACE "\"" "" vals "${vals}")
+                            set(MACRO_DEFINITION "#define ${MACRO_NAME} ${vals}")
+                            if(arch MATCHES ^default)
+                                # fallback defaults are wrapped in #ifndef
+                                string(APPEND generate_gpu_param_header_OUTPUT_TMP_${arch} "#ifndef ${MACRO_NAME}\n  ${MACRO_DEFINITION}\n#endif\n\n")
+                            else()
+                                string(APPEND generate_gpu_param_header_OUTPUT_TMP_${arch} "${MACRO_DEFINITION}\n")
+                            endif()
+                        endif()
+                    endforeach()
                 endif()
             endforeach()
         endforeach()
+        math(EXPR GPU_PARAM_JSON_N_FILES "${GPU_PARAM_JSON_N_FILES} + 1")
     endforeach()
-    foreach(arch IN LISTS arch_list)
-        set(${output}_${arch} "${OUTPUT_TMP_${arch}}" PARENT_SCOPE)
-    endforeach()
-    if(arch_list_output)
-        list(REMOVE_DUPLICATES arch_list_output_tmp)
-        list(SORT arch_list_output_tmp)
-        set(${arch_list_output} "${arch_list_output_tmp}" PARENT_SCOPE)
-    endif()
-endfunction()
 
-function(generate_gpu_param_header GPU_ARCH OUT_HEADER)
-    set(TARGET_ARCH "UNKNOWN")
-    if(GPU_ARCH STREQUAL "AUTO")
-        detect_gpu_arch("ALL")
-    else()
-        set(TARGET_ARCH ${GPU_ARCH})
+    list(REMOVE_DUPLICATES JSON_ARCHITECTURES)
+    list(SORT JSON_ARCHITECTURES)
+    if(ARGC GREATER 4)
+        set(${ARGV4} "${JSON_ARCHITECTURES}" PARENT_SCOPE)
+    endif()
+    if(do_all_architectures GREATER -1)
+        list(REMOVE_ITEM ARCH_LIST "ALL")
+        list(APPEND ARCH_LIST ${JSON_ARCHITECTURES})
     endif()
-    file(READ "${GPU_PARAM_JSON}" JSON_CONTENT)
+    list(REMOVE_DUPLICATES ARCH_LIST)
+    list(SORT ARCH_LIST)
+
+    get_filename_component(DEVICE_HEADER_FILE "${OUT_HEADER_DEVICE}" NAME)
+
     set(TMP_HEADER "#ifndef GPUDEFPARAMETERSDEFAULTS_H\n#define GPUDEFPARAMETERSDEFAULTS_H\n\n")
+    set(TMP_HEADER_DEVICE "#ifndef GPUDEFPARAMETERSDEFAULTSDEVICE_H\n#define GPUDEFPARAMETERSDEFAULTSDEVICE_H\n\n")
     string(APPEND TMP_HEADER "// This file is auto-generated from gpu_params.json. Do not edit directly.\n")
-    string(REPLACE "," ";" ARCH_LIST "${TARGET_ARCH}")
-    string(APPEND TMP_HEADER "// Architectures: ${TARGET_ARCH}\n\n")
+    string(APPEND TMP_HEADER_DEVICE "// This file is auto-generated from gpu_params.json. Do not edit directly.\n")
+    string(APPEND TMP_HEADER_DEVICE "// Architectures: ${TARGET_ARCH}\n\n")
     string(APPEND TMP_HEADER "#if defined(GPUCA_GPUCODE) && !defined(GPUCA_GPUCODE_GENRTC) && !defined(GPUCA_GPUCODE_NO_LAUNCH_BOUNDS) // Avoid including for RTC generation besides normal include protection.\n\n")
+    string(APPEND TMP_HEADER "#include \"${DEVICE_HEADER_FILE}\"\n")
 
-    # Types
-    set(TYPES CORE LB PAR)
-    # Per architecture definitions
-    generate_macros("${JSON_CONTENT}" TMP_OUTPUT "${TYPES}" "${ARCH_LIST};default;default_cpu" "JSON_ARCHITECTURES")
-    list(FIND ARCH_LIST "ALL" do_all_architectures)
-    if(ARGC GREATER 2)
-        set(${ARGV2} "${JSON_ARCHITECTURES}" PARENT_SCOPE)
-    endif()
-    if(do_all_architectures GREATER -1)
-        set(ARCH_LIST ${JSON_ARCHITECTURES})
-    endif()
-    string(APPEND TMP_HEADER "#if 0\n")
+    string(APPEND TMP_HEADER_DEVICE "#if 0\n")
     foreach(ARCH IN LISTS ARCH_LIST)
-        string(APPEND TMP_HEADER "\n#elif defined(GPUCA_GPUTYPE_${ARCH})\n")
-        string(APPEND TMP_HEADER ${TMP_OUTPUT_${ARCH}})
+        string(APPEND TMP_HEADER_DEVICE "\n#elif defined(GPUCA_GPUTYPE_${ARCH})\n")
+        string(APPEND TMP_HEADER_DEVICE ${generate_gpu_param_header_OUTPUT_TMP_${ARCH}})
     endforeach()
-    string(APPEND TMP_HEADER "#else\n#error GPU TYPE NOT SET\n#endif\n")
+    string(APPEND TMP_HEADER_DEVICE "#else\n#error GPU TYPE NOT SET\n#endif\n")
 
     # Default parameters
     string(APPEND TMP_HEADER "\n// Default parameters if not defined for the target architecture\n\n")
-    string(APPEND TMP_HEADER ${TMP_OUTPUT_default})
+    string(APPEND TMP_HEADER ${generate_gpu_param_header_OUTPUT_TMP_default})
     string(APPEND TMP_HEADER "#endif // defined(GPUCA_GPUCODE) && !defined(GPUCA_GPUCODE_GENRTC) && !defined(GPUCA_GPUCODE_NO_LAUNCH_BOUNDS)\n\n")
 
     # CPU fallback
     string(APPEND TMP_HEADER "#ifndef GPUCA_GPUCODE_GENRTC // Defaults for non-LB parameters also for CPU fallback\n\n")
-    string(APPEND TMP_HEADER ${TMP_OUTPUT_default_cpu})
+    string(APPEND TMP_HEADER ${generate_gpu_param_header_OUTPUT_TMP_default_cpu})
     string(APPEND TMP_HEADER "\n#endif // GPUCA_GPUCODE_GENRTC\n")
 
     string(APPEND TMP_HEADER "\n#endif // GPUDEFPARAMETERSDEFAULTS_H\n")
+    string(APPEND TMP_HEADER_DEVICE "\n#endif // GPUDEFPARAMETERSDEFAULTSDEVICE_H\n")
     file(GENERATE OUTPUT "${OUT_HEADER}" CONTENT "${TMP_HEADER}")
-    message(STATUS "Generated ${OUT_HEADER}")
-    add_custom_target(GPU_PARAM_HEADER_${GPU_ARCH}_ALL ALL DEPENDS ${OUT_HEADER} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/gpu_param_header_generator.cmake ${GPU_PARAM_JSON})
+    file(GENERATE OUTPUT "${OUT_HEADER_DEVICE}" CONTENT "${TMP_HEADER_DEVICE}")
+    message(STATUS "Generated ${OUT_HEADER} and ${OUT_HEADER_DEVICE}")
 endfunction()