Skip to content

Make --mixed build again, and fix its multi-rank parallel I/O #2106

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ruru25254649 wants to merge 3 commits into from
+287 −44
Closed

Make --mixed build again, and fix its multi-rank parallel I/O #2106

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0906107
Make --mixed build again: match stp and wp at call sites
ruru25254649 Oct 24, 2026
2a8dc39
Describe --mixed parallel I/O file views in stored cells
ruru25254649 Oct 24, 2026
c42f58b
Convert --mixed operands to wp before adding or dividing them
ruru25254649 Oct 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions src/common/m_derived_types.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,8 @@ module m_derived_types

!> Derived type annexing an integer scalar field (SF)
type integer_field
#ifdef MFC_MIXED_PRECISION
integer(kind=1), pointer, dimension(:,:,:) :: sf => null()
#else
! Patch encodings, GPU atomics and MPI_INTEGER I/O require default-width integer metadata.
integer, pointer, dimension(:,:,:) :: sf => null()
#endif
end type integer_field

!> Derived type for levelset
Expand Down
37 changes: 26 additions & 11 deletions src/common/m_mpi_common.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ module m_mpi_common

implicit none

private :: s_apply_decomposition_policies
private :: s_apply_decomposition_policies, s_create_mpi_field_subarray

integer, private :: v_size
$:GPU_DECLARE(create='[v_size]')
Expand Down Expand Up @@ -175,16 +175,12 @@ contains

! Define the view for each variable
do i = 1, alt_sys
call MPI_TYPE_CREATE_SUBARRAY(num_dims, sizes_glb, sizes_loc, start_idx, MPI_ORDER_FORTRAN, mpi_p, &
& MPI_IO_DATA%view(i), ierr)
call MPI_TYPE_COMMIT(MPI_IO_DATA%view(i), ierr)
call s_create_mpi_field_subarray(sizes_glb, sizes_loc, start_idx, MPI_IO_DATA%view(i))
end do

if (bind_qbmm_fields) then
do i = sys_size + 1, sys_size + 2*nb*nnode
call MPI_TYPE_CREATE_SUBARRAY(num_dims, sizes_glb, sizes_loc, start_idx, MPI_ORDER_FORTRAN, mpi_p, &
& MPI_IO_DATA%view(i), ierr)
call MPI_TYPE_COMMIT(MPI_IO_DATA%view(i), ierr)
call s_create_mpi_field_subarray(sizes_glb, sizes_loc, start_idx, MPI_IO_DATA%view(i))
end do
end if

Expand All @@ -211,7 +207,7 @@ contains
integer, dimension(3) :: sf_start_idx

#ifdef MFC_MPI
integer :: i, ierr
integer :: i

sf_start_idx = (/0, 0, 0/)

Expand All @@ -231,14 +227,33 @@ contains

! Define the view for each variable
do i = 1, sys_size
call MPI_TYPE_CREATE_SUBARRAY(num_dims, sizes_loc, sizes_loc, sf_start_idx, MPI_ORDER_FORTRAN, mpi_p, &
& MPI_IO_DATA%view(i), ierr)
call MPI_TYPE_COMMIT(MPI_IO_DATA%view(i), ierr)
call s_create_mpi_field_subarray(sizes_loc, sizes_loc, sf_start_idx, MPI_IO_DATA%view(i))
end do
#endif

end subroutine s_initialize_mpi_data_ds

!> File views advance by stored cells; half-precision cells contain two MPI_BYTE values.
subroutine s_create_mpi_field_subarray(sizes_glb, sizes_loc, starts, view)

integer, dimension(num_dims), intent(in) :: sizes_glb, sizes_loc, starts
integer, intent(out) :: view

#ifdef MFC_MPI
integer :: cell_type, ierr

cell_type = mpi_io_p
if (mpi_io_type /= 1) then
call MPI_TYPE_CONTIGUOUS(mpi_io_type, mpi_io_p, cell_type, ierr)
call MPI_TYPE_COMMIT(cell_type, ierr)
end if
call MPI_TYPE_CREATE_SUBARRAY(num_dims, sizes_glb, sizes_loc, starts, MPI_ORDER_FORTRAN, cell_type, view, ierr)
call MPI_TYPE_COMMIT(view, ierr)
if (mpi_io_type /= 1) call MPI_TYPE_FREE(cell_type, ierr)
#endif

end subroutine s_create_mpi_field_subarray

!> Gather variable-length real vectors from all MPI ranks onto the root process.
impure subroutine s_mpi_gather_data(my_vector, counts, gathered_vector, root)

Expand Down
2 changes: 1 addition & 1 deletion src/post_process/m_data_input.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,7 @@ impure subroutine s_read_parallel_conservative_data(t_step, m_MOK, n_MOK, p_MOK,

disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1)

call MPI_FILE_SET_VIEW(ifile, disp, mpi_p, MPI_IO_DATA%view(i), 'native', mpi_info_int, ierr)
call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), 'native', mpi_info_int, ierr)
call MPI_FILE_READ_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
end do

Expand Down
6 changes: 3 additions & 3 deletions src/post_process/m_start_up.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -552,9 +552,9 @@ contains
do k = -offset_z%beg, p + offset_z%end
do j = -offset_y%beg, n + offset_y%end
do i = -offset_x%beg, m + offset_x%end
call s_phase_temperature(q_prim_vf(eqn_idx%cont%beg + l - 1)%sf(i, j, &
& k)/max(q_prim_vf(eqn_idx%E + l)%sf(i, j, k), sgm_eps), &
& q_prim_vf(eqn_idx%E)%sf(i, j, k), l, T)
call s_phase_temperature(real(q_prim_vf(eqn_idx%cont%beg + l - 1)%sf(i, j, k), &
& kind=wp)/max(real(q_prim_vf(eqn_idx%E + l)%sf(i, j, k), kind=wp), &
& sgm_eps), real(q_prim_vf(eqn_idx%E)%sf(i, j, k), kind=wp), l, T)
out%q_sf(i, j, k) = T
end do
end do
Expand Down
8 changes: 4 additions & 4 deletions src/pre_process/m_start_up.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,8 +469,8 @@ contains
! Initial displacement to skip at beginning of file
disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1)

call MPI_FILE_SET_VIEW(ifile, disp, mpi_p, MPI_IO_DATA%view(i), 'native', mpi_info_int, ierr)
call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size, mpi_p, status, ierr)
call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), 'native', mpi_info_int, ierr)
call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
end do

if (qbmm .and. .not. polytropic) then
Expand All @@ -480,8 +480,8 @@ contains
! Initial displacement to skip at beginning of file
disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1)

call MPI_FILE_SET_VIEW(ifile, disp, mpi_p, MPI_IO_DATA%view(i), 'native', mpi_info_int, ierr)
call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size, mpi_p, status, ierr)
call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), 'native', mpi_info_int, ierr)
call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
end do
end if

Expand Down
2 changes: 1 addition & 1 deletion src/simulation/m_acoustic_src.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ contains
end do
c = c/myRho
else
c = f_bulk_modulus(q_prim_vf(eqn_idx%E)%sf(j, k, l), gamma_mix, pi_inf_mix)/myRho
c = f_bulk_modulus(real(q_prim_vf(eqn_idx%E)%sf(j, k, l), kind=wp), gamma_mix, pi_inf_mix)/myRho
end if
if (model_eqns == model_eqns_5eq .and. bubbles_euler .and. .not. (mpp_lim .and. num_fluids > 1)) then
c = c/(1._wp - myalpha(num_fluids))
Expand Down
6 changes: 3 additions & 3 deletions src/simulation/m_bubbles_EE.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,14 +265,14 @@ contains
! Adaptive time stepping
if (adap_dt) then
adap_dt_stop = f_advance_step(myRho, myP, myR, myV, R0(q), pb_local, pbdot, alf, n_tait, B_tait, &
& bub_adv_src(j, k, l), divu_in%sf(j, k, l), dmBub_id, dmMass_v, &
& dmMass_n, dmBeta_c, dmBeta_t, dmCson)
& bub_adv_src(j, k, l), real(divu_in%sf(j, k, l), kind=wp), &
& dmBub_id, dmMass_v, dmMass_n, dmBeta_c, dmBeta_t, dmCson)

q_cons_vf(rs(q))%sf(j, k, l) = nbub*myR
q_cons_vf(vs(q))%sf(j, k, l) = nbub*myV
else
rddot = f_rddot(myRho, myP, myR, myV, R0(q), pb_local, pbdot, alf, n_tait, B_tait, bub_adv_src(j, &
& k, l), divu_in%sf(j, k, l), dmCson)
& k, l), real(divu_in%sf(j, k, l), kind=wp), dmCson)
bub_v_src(j, k, l, q) = nbub*rddot
bub_r_src(j, k, l, q) = q_cons_vf(vs(q))%sf(j, k, l)
end if
Expand Down
8 changes: 4 additions & 4 deletions src/simulation/m_data_output.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -991,7 +991,7 @@ contains

disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1)

call MPI_FILE_SET_VIEW(ifile, disp, mpi_p, MPI_IO_DATA%view(i), 'native', mpi_info_int, ierr)
call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), 'native', mpi_info_int, ierr)
call MPI_FILE_WRITE_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
end do
if (qbmm .and. .not. polytropic) then
Expand All @@ -1000,7 +1000,7 @@ contains

disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1)

call MPI_FILE_SET_VIEW(ifile, disp, mpi_p, MPI_IO_DATA%view(i), 'native', mpi_info_int, ierr)
call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), 'native', mpi_info_int, ierr)
call MPI_FILE_WRITE_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
end do
end if
Expand All @@ -1010,7 +1010,7 @@ contains

disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1)

call MPI_FILE_SET_VIEW(ifile, disp, mpi_p, MPI_IO_DATA%view(i), 'native', mpi_info_int, ierr)
call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), 'native', mpi_info_int, ierr)
call MPI_FILE_WRITE_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
end do
end if
Expand All @@ -1020,7 +1020,7 @@ contains

disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1)

call MPI_FILE_SET_VIEW(ifile, disp, mpi_p, MPI_IO_DATA%view(sys_size + 1), 'native', mpi_info_int, ierr)
call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(sys_size + 1), 'native', mpi_info_int, ierr)
call MPI_FILE_WRITE_ALL(ifile, MPI_IO_DATA%var(sys_size + 1)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
end if

Expand Down
15 changes: 8 additions & 7 deletions src/simulation/m_igr.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,16 @@ module m_igr

!> @cond
#ifdef __NVCOMPILER_GPU_UNIFIED_MEM
integer, dimension(3) :: nv_uvm_temp_on_gpu
real(wp), pointer, contiguous, dimension(:,:,:) :: jac, jac_rhs, jac_old
real(wp), allocatable, dimension(:,:,:), pinned, target :: jac_host
real(wp), allocatable, dimension(:,:,:), pinned, target :: jac_rhs_host
real(wp), allocatable, dimension(:,:,:), pinned, target :: jac_old_host
integer, dimension(3) :: nv_uvm_temp_on_gpu
real(stp), pointer, contiguous, dimension(:,:,:) :: jac
real(wp), pointer, contiguous, dimension(:,:,:) :: jac_rhs, jac_old
real(stp), allocatable, dimension(:,:,:), pinned, target :: jac_host
real(wp), allocatable, dimension(:,:,:), pinned, target :: jac_rhs_host
real(wp), allocatable, dimension(:,:,:), pinned, target :: jac_old_host
#else
!> @endcond
real(wp), allocatable, target, dimension(:,:,:) :: jac
real(wp), allocatable, dimension(:,:,:) :: jac_rhs, jac_old
real(stp), allocatable, target, dimension(:,:,:) :: jac
real(wp), allocatable, dimension(:,:,:) :: jac_rhs, jac_old
$:GPU_DECLARE(create='[jac, jac_rhs, jac_old]')
!> @cond
#endif
Expand Down
14 changes: 8 additions & 6 deletions src/simulation/m_viscous.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,14 +70,16 @@ contains
integer :: i, q

if (any_non_newtonian) then
gamma_dot_c = f_compute_shear_rate_from_components(grad_x_vf(1)%sf(j, k, l), grad_y_vf(2)%sf(j, k, l), 0._wp, &
& 0.5_wp*(grad_y_vf(1)%sf(j, k, l) + grad_x_vf(2)%sf(j, k, l)), 0._wp, 0._wp)
gamma_dot_c = f_compute_shear_rate_from_components(real(grad_x_vf(1)%sf(j, k, l), kind=wp), real(grad_y_vf(2)%sf(j, &
& k, l), kind=wp), 0._wp, 0.5_wp*(real(grad_y_vf(1)%sf(j, k, l), kind=wp) + real(grad_x_vf(2)%sf(j, k, l), &
& kind=wp)), 0._wp, 0._wp)
#:if not MFC_CASE_OPTIMIZATION or num_dims > 2
if (p > 0) then
gamma_dot_c = f_compute_shear_rate_from_components(grad_x_vf(1)%sf(j, k, l), grad_y_vf(2)%sf(j, k, l), &
& grad_z_vf(3)%sf(j, k, l), 0.5_wp*(grad_y_vf(1)%sf(j, k, l) + grad_x_vf(2)%sf(j, k, l)), &
& 0.5_wp*(grad_z_vf(1)%sf(j, k, l) + grad_x_vf(3)%sf(j, k, l)), 0.5_wp*(grad_z_vf(2)%sf(j, k, &
& l) + grad_y_vf(3)%sf(j, k, l)))
gamma_dot_c = f_compute_shear_rate_from_components(real(grad_x_vf(1)%sf(j, k, l), kind=wp), &
& real(grad_y_vf(2)%sf(j, k, l), kind=wp), real(grad_z_vf(3)%sf(j, k, l), kind=wp), &
& 0.5_wp*(real(grad_y_vf(1)%sf(j, k, l), kind=wp) + real(grad_x_vf(2)%sf(j, k, l), kind=wp)), &
& 0.5_wp*(real(grad_z_vf(1)%sf(j, k, l), kind=wp) + real(grad_x_vf(3)%sf(j, k, l), kind=wp)), &
& 0.5_wp*(real(grad_z_vf(2)%sf(j, k, l), kind=wp) + real(grad_y_vf(3)%sf(j, k, l), kind=wp)))
end if
#:endif
call s_compute_mixture_inv_re(alpha_visc, gamma_dot_c, Res_viscous, Re_visc)
Expand Down
Loading
Loading