Skip to content

Develop 126 add gpu testing #130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cguzman95 wants to merge 4 commits into
base: chem_mod
Choose a base branch
from
Open

Develop 126 add gpu testing #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a1de7d6
Added compilation of the tests on GPU: For the moment, only unit_test…
cguzman95 Oct 22, 2019
eb2d007
Fixed new unit_test_arrhenius on gpu by moving the global gpu variabl…
cguzman95 Oct 22, 2019
c0b2f63
Cleaning code. Prepared a folder debug for other issue
cguzman95 Oct 26, 2019
ea78eec
Fixing format
cguzman95 Oct 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 107 additions & 48 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,11 +170,21 @@ endif()
######################################################################
# GPU support

if (ENABLE_GPU)
add_definitions(-DPMC_USE_GPU)
enable_language(CUDA)

#Compile GPU tests if CUDA detected
find_package(CUDA)
if (CUDA_FOUND)
enable_language(CUDA)
#If enable_gpu, all will be executed on GPU
if (ENABLE_GPU)
#Define global variable of use gpu no matter what test to include user main test
add_definitions(-DPMC_USE_GPU)
endif()
endif()

if(NOT CUDA_FOUND AND ENABLE_GPU)
message( WARNING "Not CUDA module found, can't support GPU computation")
endif()

######################################################################
# debugging options
Expand Down Expand Up @@ -262,6 +272,15 @@ if (ENABLE_TESTS)
add_test(test_MONARCH_1 ${CMAKE_BINARY_DIR}/test_run/monarch/test_monarch_1.sh ${MPI_TEST_FLAG})
add_test(test_MONARCH_2 ${CMAKE_BINARY_DIR}/test_run/monarch/test_monarch_2.sh ${MPI_TEST_FLAG})

if (CUDA_FOUND AND NOT ENABLE_GPU)
#todo: new unit_tests with gpu
add_test(test_rxn_arrhenius_mech_gpu ${CMAKE_BINARY_DIR}/test_run/unit_tests/input_files/run_rxn_arrhenius_gpu.sh ${MPI_TEST_FLAG})

#Relevant mechanisms tests
add_test(test_chemistry_cb05cl_ae5_gpu ${CMAKE_BINARY_DIR}/test_run/chemistry/cb05cl_ae5/test_chemistry_cb05cl_ae5_gpu.sh ${MPI_TEST_FLAG})

endif()

endif()

add_test(test_additive_1 ${CMAKE_BINARY_DIR}/test_run/additive/test_additive_1.sh)
Expand Down Expand Up @@ -411,6 +430,7 @@ endif()
# partmc library

set(STD_C_FLAGS "-std=c99")

set(STD_CUDA_FLAGS "-dc -arch=compute_70 -code=sm_70")

if(${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel")
Expand Down Expand Up @@ -491,14 +511,16 @@ set(SUB_MODELS_SRC ${SUB_MODELS_F_SRC} ${SUB_MODELS_C_SRC})

set(CAMP_C_SRC
src/camp_solver.c src/rxn_solver.c src/aero_phase_solver.c
src/aero_rep_solver.c src/sub_model_solver.c)
src/aero_rep_solver.c src/sub_model_solver.c
src/debug_and_stats/camp_debug_2.c)

set_source_files_properties(${CAMP_C_SRC} PROPERTIES COMPILE_FLAGS
${STD_C_FLAGS})

set(CAMP_CXX_SRC "")

if(ENABLE_GPU)
if (CUDA_FOUND)
#if(ENABLE_GPU)
set(CAMP_CUDA_SRC
src/cuda/camp_gpu_solver.cu
src/cuda/rxns_gpu/rxn_aqueous_equilibrium.cu
Expand All @@ -521,33 +543,38 @@ if(ENABLE_GPU)

set_source_files_properties(${CAMP_CUDA_SRC} PROPERTIES COMPILE_FLAGS
${STD_CUDA_FLAGS})

set_source_files_properties(${CAMP_CUDA_SRC} PROPERTIES LANGUAGE CUDA)
else()
set(CAMP_CUDA_SRC "")
# set(CAMP_CUDA_SRC "")
endif()

add_library(partmclib src/aero_state.F90 src/integer_varray.F90
src/integer_rmap.F90 src/integer_rmap2.F90 src/aero_sorted.F90
src/aero_binned.F90 src/bin_grid.F90 src/constants.F90
src/scenario.F90 src/env_state.F90 src/aero_mode.F90
src/aero_dist.F90 src/aero_weight.F90 src/aero_weight_array.F90
src/coag_kernel_additive.F90 src/coag_kernel_sedi.F90
src/coag_kernel_constant.F90 src/coag_kernel_brown.F90
src/coag_kernel_zero.F90 src/coag_kernel_brown_free.F90
src/coag_kernel_brown_cont.F90 src/aero_data.F90 src/run_exact.F90
src/run_part.F90 src/util.F90 src/stats.F90 src/run_sect.F90 src/output.F90
src/mosaic.F90 src/gas_data.F90 src/gas_state.F90
src/coagulation.F90 src/exact_soln.F90 src/coagulation_dist.F90
src/coag_kernel.F90 src/spec_line.F90 src/spec_file.F90 src/rand.F90
src/aero_particle.F90 src/aero_particle_array.F90 src/mpi.F90
src/netcdf.F90 src/aero_info.F90 src/aero_info_array.F90
src/nucleate.F90 src/condense.F90 src/fractal.F90 src/chamber.F90
src/property.F90 src/chem_spec_data.F90
src/rxn_data.F90 src/camp_state.F90 src/mechanism_data.F90
src/camp_core.F90 src/camp_solver_data.F90 src/aero_rep_data.F90
src/aero_phase_data.F90 src/aero_rep_factory.F90 src/camp_interface.F90
src/rxn_factory.F90 src/sub_model_data.F90 src/sub_model_factory.F90
src/solver_stats.F90 src/camp_box_model_data.F90
set(CAMP_F_SRC
src/aero_state.F90 src/integer_varray.F90
src/integer_rmap.F90 src/integer_rmap2.F90 src/aero_sorted.F90
src/aero_binned.F90 src/bin_grid.F90 src/constants.F90
src/scenario.F90 src/env_state.F90 src/aero_mode.F90
src/aero_dist.F90 src/aero_weight.F90 src/aero_weight_array.F90
src/coag_kernel_additive.F90 src/coag_kernel_sedi.F90
src/coag_kernel_constant.F90 src/coag_kernel_brown.F90
src/coag_kernel_zero.F90 src/coag_kernel_brown_free.F90
src/coag_kernel_brown_cont.F90 src/aero_data.F90 src/run_exact.F90
src/run_part.F90 src/util.F90 src/stats.F90 src/run_sect.F90 src/output.F90
src/mosaic.F90 src/gas_data.F90 src/gas_state.F90
src/coagulation.F90 src/exact_soln.F90 src/coagulation_dist.F90
src/coag_kernel.F90 src/spec_line.F90 src/spec_file.F90 src/rand.F90
src/aero_particle.F90 src/aero_particle_array.F90 src/mpi.F90
src/netcdf.F90 src/aero_info.F90 src/aero_info_array.F90
src/nucleate.F90 src/condense.F90 src/fractal.F90 src/chamber.F90
src/property.F90 src/chem_spec_data.F90
src/rxn_data.F90 src/camp_state.F90 src/mechanism_data.F90
src/camp_core.F90 src/camp_solver_data.F90 src/aero_rep_data.F90
src/aero_phase_data.F90 src/aero_rep_factory.F90 src/camp_interface.F90
src/rxn_factory.F90 src/sub_model_data.F90 src/sub_model_factory.F90
src/solver_stats.F90 src/camp_box_model_data.F90
)

add_library(partmclib ${CAMP_F_SRC}
${CAMP_C_SRC} ${AEROSOL_REPS_SRC} ${SUB_MODELS_SRC} ${REACTIONS_SRC}
${SUNDIALS_SRC} ${GSL_SRC} ${C_SORT_SRC} ${CAMP_CUDA_SRC} ${CAMP_CXX_SRC} )

Expand All @@ -556,6 +583,24 @@ target_link_libraries(partmclib ${NETCDF_LIBS} ${SUNDIALS_LIBS}

set_target_properties(partmclib PROPERTIES OUTPUT_NAME partmc)

if (CUDA_FOUND AND NOT ENABLE_GPU)
add_library(partmclib_gpu ${CAMP_F_SRC}
${CAMP_C_SRC} ${AEROSOL_REPS_SRC} ${SUB_MODELS_SRC} ${REACTIONS_SRC}
${SUNDIALS_SRC} ${GSL_SRC} ${C_SORT_SRC} ${CAMP_CUDA_SRC} ${CAMP_CXX_SRC} )

target_compile_definitions(partmclib_gpu PRIVATE
"PMC_USE_GPU=ON")

target_link_libraries(partmclib_gpu ${NETCDF_LIBS} ${SUNDIALS_LIBS}
${MOSAIC_LIB} ${GSL_LIBS} ${JSON_LIB})

set_target_properties(partmclib_gpu PROPERTIES OUTPUT_NAME partmc_gpu)

add_executable(partmc_gpu src/partmc.F90)

target_link_libraries(partmc_gpu partmclib_gpu)
endif()

######################################################################
# partmc executable

Expand Down Expand Up @@ -614,13 +659,21 @@ set_source_files_properties(${CB5_EBI_SOLVER} PROPERTIES COMPILE_FLAGS
${CB5_EBI_FLAGS})

######################################################################
# test_chemistry_cb05cl_ae5_big
# test_chemistry_cb05cl_ae5

add_executable(test_chemistry_cb05cl_ae5_big
test/chemistry/cb05cl_ae5/test_cb05cl_ae5_big.F90
add_executable(test_chemistry_cb05cl_ae5
test/chemistry/cb05cl_ae5/test_cb05cl_ae5.F90
${CB5_EBI_SOLVER} ${CB5_KPP_SOLVER}
test/chemistry/cb05cl_ae5/module_BSC_CHEM_DATA.F90)
target_link_libraries(test_chemistry_cb05cl_ae5_big partmclib)
target_link_libraries(test_chemistry_cb05cl_ae5 partmclib)

if (CUDA_FOUND AND NOT ENABLE_GPU)
add_executable(test_chemistry_cb05cl_ae5_gpu
test/chemistry/cb05cl_ae5/test_cb05cl_ae5.F90
${CB5_EBI_SOLVER} ${CB5_KPP_SOLVER}
test/chemistry/cb05cl_ae5/module_BSC_CHEM_DATA.F90)
target_link_libraries(test_chemistry_cb05cl_ae5_gpu partmclib_gpu)
endif()

######################################################################
# MONARCH interface
Expand All @@ -633,20 +686,19 @@ target_link_libraries(mock_monarch partmclib)
if (ENABLE_TESTS)

######################################################################
# test_chemistry_cb05cl_ae5
# test_chemistry_cb05cl_ae5_big

add_executable(test_chemistry_cb05cl_ae5
test/chemistry/cb05cl_ae5/test_cb05cl_ae5.F90
${CB5_EBI_SOLVER} ${CB5_KPP_SOLVER}
test/chemistry/cb05cl_ae5/module_BSC_CHEM_DATA.F90)
target_link_libraries(test_chemistry_cb05cl_ae5 partmclib)
add_executable(test_chemistry_cb05cl_ae5_big
test/chemistry/cb05cl_ae5/test_cb05cl_ae5_big.F90
${CB5_EBI_SOLVER} ${CB5_KPP_SOLVER}
test/chemistry/cb05cl_ae5/module_BSC_CHEM_DATA.F90)
target_link_libraries(test_chemistry_cb05cl_ae5_big partmclib)

######################################################################
# test_chemistry_eqsam_v03d

set(EQSAM_V03D
test/chemistry/eqsam_v03d/eqsam_v03d.F90
)
test/chemistry/eqsam_v03d/eqsam_v03d.F90)

add_executable(test_chemistry_eqsam_v03d test/chemistry/eqsam_v03d/test_eqsam_v03d.F90
${EQSAM_V03D})
Expand Down Expand Up @@ -709,19 +761,26 @@ target_link_libraries(test_sub_model_ZSR_aerosol_water partmclib)
# New unit tests (UNDER DEVELOPMENT)

set(UNIT_TEST_SRC
test/unit_tests/unit_test_data.F90
test/unit_tests/unit_test_driver.F90)
test/unit_tests/unit_test_data.F90
test/unit_tests/unit_test_driver.F90)

set(UNIT_TEST_RXN_ARRHENIUS_SRC
test/unit_tests/rxns/unit_test_rxn_arrhenius.F90 ${UNIT_TEST_SRC})
set_source_files_properties(${UNIT_TEST_RXN_ARRHENIUS_SRC} PROPERTIES COMPILE_DEFINITIONS
"UNIT_TEST_MODULE_=pmc_unit_test_rxn_arrhenius \
;UNIT_TEST_TYPE_=unit_test_rxn_arrhenius_t()")
set_source_files_properties(${UNIT_TEST_RXN_ARRHENIUS_SRC} PROPERTIES COMPILE_FLAGS
${STD_F_FLAGS})
test/unit_tests/rxns/unit_test_rxn_arrhenius.F90 ${UNIT_TEST_SRC})
set_source_files_properties(${UNIT_TEST_RXN_ARRHENIUS_SRC} PROPERTIES COMPILE_FLAGS ${STD_F_FLAGS})
add_executable(unit_test_rxn_arrhenius ${UNIT_TEST_RXN_ARRHENIUS_SRC})
target_compile_definitions(unit_test_rxn_arrhenius PRIVATE
"UNIT_TEST_MODULE_=pmc_unit_test_rxn_arrhenius \
;UNIT_TEST_TYPE_=unit_test_rxn_arrhenius_t()")
target_link_libraries(unit_test_rxn_arrhenius partmclib)

if (CUDA_FOUND AND NOT ENABLE_GPU)
add_executable(unit_test_rxn_arrhenius_gpu ${UNIT_TEST_RXN_ARRHENIUS_SRC})
target_compile_definitions(unit_test_rxn_arrhenius_gpu PRIVATE
"UNIT_TEST_MODULE_=pmc_unit_test_rxn_arrhenius \
;UNIT_TEST_TYPE_=unit_test_rxn_arrhenius_t()")
target_link_libraries(unit_test_rxn_arrhenius_gpu partmclib_gpu)
endif()

######################################################################
# test_camp_core

Expand Down
30 changes: 30 additions & 0 deletions src/camp_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,12 @@ typedef struct {
int param_id; // sub model Jacobian id
} JacMap;

/* GPU data structure */
typedef struct {
size_t deriv_size;

} DataGPU;

/* Model data structure */
typedef struct {
int n_per_cell_state_var; // number of state variables per grid cell
Expand Down Expand Up @@ -167,6 +173,30 @@ typedef struct {
// for the current grid cell
int n_sub_model_env_data; // Number of sub model environmental parameters
// from all sub models
//#ifdef CUDA_FOUND
// GPU data
double *deriv_gpu_data;
double *deriv_cpu;
double *jac_gpu_data;
double *jac_cpu;
size_t deriv_size;
size_t jac_size;
size_t state_size;
size_t env_size;
size_t rate_constants_size;
size_t rate_constants_idx_size;
bool few_data;
bool implemented_all;
int *int_pointer;
int *int_pointer_gpu;
double *double_pointer;
double *double_pointer_gpu;
double *state_gpu;
double *env_gpu;
double *rate_constants_gpu;
int *rate_constants_idx_gpu;

//#endif
} ModelData;

/* Solver data structure */
Expand Down
4 changes: 4 additions & 0 deletions src/camp_debug.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
* Header file with some debugging functions for use with camp_solver.c
*
*/
#ifndef CAMP_DEBUG_H
#define CAMP_DEBUG_H
#ifdef PMC_DEBUG
#define PMC_DEBUG_SPEC_ 0
#define PMC_DEBUG_PRINT(x) \
Expand Down Expand Up @@ -146,3 +148,5 @@ static void print_derivative(N_Vector deriv) {
printf(" index: %d \n", i);
}
}

#endif
28 changes: 13 additions & 15 deletions src/camp_solver.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,8 +337,9 @@ void *solver_new(int n_state_var, int n_cells, int *var_type, int n_rxn,
sd->model_data.sub_model_env_idx[0] = 0;

#ifdef PMC_USE_GPU
solver_new_gpu_cu(n_dep_var, n_state_var, n_rxn, n_rxn_int_param,
n_rxn_float_param, n_rxn_env_param, n_cells);
solver_new_gpu_cu(&(sd->model_data), n_dep_var, n_state_var, n_rxn,
n_rxn_int_param, n_rxn_float_param, n_rxn_env_param,
n_cells);
#endif

#ifdef PMC_DEBUG
Expand Down Expand Up @@ -452,11 +453,6 @@ void solver_initialize(void *solver_data, double *abs_tol, double rel_tol,
flag = CVodeSetDlsGuessHelper(sd->cvode_mem, guess_helper);
check_flag_fail(&flag, "CVodeSetDlsGuessHelper", 1);

// Allocate Jacobian on GPU
#ifdef PMC_USE_GPU
allocate_jac_gpu(sd->model_data.n_per_cell_solver_jac_elem, n_cells);
#endif

// Set gpu rxn values
#ifdef PMC_USE_GPU
solver_set_rxn_data_gpu(&(sd->model_data));
Expand Down Expand Up @@ -969,7 +965,8 @@ int Jac(realtype t, N_Vector y, N_Vector deriv, SUNMatrix J, void *solver_data,

#ifdef PMC_USE_GPU
// Calculate the Jacobian
rxn_calc_jac_gpu(md, J, time_step);
// TODO: Fix jacobian mapping with jac_map[i_map].rxn_id
// rxn_calc_jac_gpu(md, J, time_step);
#endif

#ifdef PMC_DEBUG
Expand All @@ -978,7 +975,6 @@ int Jac(realtype t, N_Vector y, N_Vector deriv, SUNMatrix J, void *solver_data,
#endif

// Solving on CPU only

// Loop over the grid cells to calculate sub-model and rxn Jacobians
for (int i_cell = 0; i_cell < n_cells; ++i_cell) {
// Set the grid cell state pointers
Expand Down Expand Up @@ -1009,18 +1005,20 @@ int Jac(realtype t, N_Vector y, N_Vector deriv, SUNMatrix J, void *solver_data,
clock_t start = clock();
#endif

#ifndef PMC_USE_GPU
//#ifdef PMC_USE_GPU

// Calculate the reaction Jacobian
rxn_calc_jac(md, J_rxn_data, time_step);
PMC_DEBUG_JAC(md->J_rxn, "reaction Jacobian");

#else
// Add contributions from reactions not implemented on GPU
rxn_calc_jac_specific_types(md, J_rxn_data, time_step);
/*#else
// Add contributions from reactions not implemented on GPU
rxn_calc_jac_specific_types(md, J_rxn_data, time_step);
PMC_DEBUG_JAC(md->J_rxn, "reaction Jacobian");
#endif

// rxn_calc_jac_specific_types(md, J_rxn_data, time_step);
rxn_calc_jac_specific_types(md, J_rxn_data, time_step);
*/
#ifdef PMC_DEBUG
clock_t end = clock();
sd->timeJac += (end - start);
Expand Down Expand Up @@ -1868,7 +1866,7 @@ void error_handler(int error_code, const char *module, const char *function,
*/
void model_free(ModelData model_data) {
#ifdef PMC_USE_GPU
// free_gpu_cu();
free_gpu_cu();
#endif

#ifdef PMC_USE_SUNDIALS
Expand Down
3 changes: 2 additions & 1 deletion src/camp_solver_data.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -888,7 +888,8 @@ subroutine solve(this, camp_state, t_initial, t_final, solver_stats)
end if

! Reset the solver function timers
call this%reset_timers( )
!Move this to intialization: I want to print only 1 time the deriv, not all times we call solve...
!call this%reset_timers( )
#endif

! Run the solver
Expand Down
Loading