Skip to content

File-Per-Process IB markers #1802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
danieljvickers wants to merge 1 commit into
base: master
Choose a base branch
from
+52 −18
Draft

File-Per-Process IB markers #1802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 22 additions & 7 deletions src/post_process/m_data_input.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,30 +105,45 @@ end subroutine s_setup_mpi_io_params
!> Helper subroutine to read IB data files
impure subroutine s_read_ib_data_files(file_loc_base, t_step)

character(len=*), intent(in) :: file_loc_base
integer, intent(in), optional :: t_step
character(LEN=len_trim(file_loc_base) + 20) :: file_loc
logical :: file_exist
integer :: ifile, ierr, data_size
character(len=*), intent(in) :: file_loc_base
integer, intent(in), optional :: t_step
character(LEN=path_len + 2*name_len) :: file_loc
logical :: file_exist
integer :: ifile, ierr, data_size

#ifdef MFC_MPI
integer, dimension(MPI_STATUS_SIZE) :: status
integer(KIND=MPI_OFFSET_KIND) :: disp
integer(KIND=MPI_OFFSET_KIND) :: m_MOK, n_MOK, p_MOK, MOK, WP_MOK, var_MOK
integer :: save_index
character(len=10) :: t_step_string
#endif

if (.not. ib) return

if (parallel_io) then
if (parallel_io .and. file_per_process) then
call s_int_to_str(t_step, t_step_string)
write (file_loc, '(A,I0,A,i7.7,A)') 'ib_markers_', t_step, '_', proc_rank, '.dat'
file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc)
else if (parallel_io) then
write (file_loc, '(A)') trim(file_loc_base) // 'ib.dat'
else
write (file_loc, '(A)') trim(file_loc_base) // '/ib_data.dat'
end if
inquire (FILE=trim(file_loc), EXIST=file_exist)

if (file_exist) then
if (parallel_io) then
if (parallel_io .and. file_per_process) then
#ifdef MFC_MPI
call MPI_FILE_OPEN(MPI_COMM_SELF, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr)

data_size = (m + 1)*(n + 1)*(p + 1)

call MPI_FILE_READ(ifile, MPI_IO_IB_DATA%var%sf, data_size, MPI_INTEGER, status, ierr)

call MPI_FILE_CLOSE(ifile, ierr)
#endif
else if (parallel_io) then
#ifdef MFC_MPI
call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr)

Expand Down
41 changes: 30 additions & 11 deletions src/simulation/m_data_output.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -881,6 +881,7 @@ contains
integer(kind=MPI_OFFSET_kind) :: WP_MOK, var_MOK, MOK
integer :: ifile, ierr, data_size
integer, dimension(MPI_STATUS_SIZE) :: status
character(len=10) :: t_step_string

$:GPU_UPDATE(host='[ib_markers%sf]')

Expand All @@ -891,21 +892,39 @@ contains
WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND)
MOK = int(1._wp, MPI_OFFSET_KIND)

write (file_loc, '(A)') 'ib.dat'
file_loc = trim(case_dir) // '/restart_data' // trim(mpiiofs) // trim(file_loc)
if (file_per_process) then
call s_int_to_str(time_step, t_step_string)

call s_mpi_barrier()
call s_delay_file_access(proc_rank)
if (proc_rank == 0) then
file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string)
call s_create_directory(trim(file_loc))
end if
call s_mpi_barrier()
call s_delay_file_access(proc_rank)

write (file_loc, '(A,I0,A,i7.7,A)') 'ib_markers_', time_step, '_', proc_rank, '.dat'
file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc)

call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, ior(MPI_MODE_WRONLY, MPI_MODE_CREATE), mpi_info_int, ifile, ierr)
call MPI_FILE_OPEN(MPI_COMM_SELF, file_loc, ior(MPI_MODE_WRONLY, MPI_MODE_CREATE), mpi_info_int, ifile, ierr)
call MPI_FILE_WRITE_ALL(ifile, MPI_IO_IB_DATA%var%sf, data_size, MPI_INTEGER, status, ierr)
call MPI_FILE_CLOSE(ifile, ierr)
else
write (file_loc, '(A)') 'ib.dat'
file_loc = trim(case_dir) // '/restart_data' // trim(mpiiofs) // trim(file_loc)

var_MOK = int(sys_size + 1, MPI_OFFSET_KIND)
disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1 + int(time_step/t_step_save))
if (time_step == 0) disp = 0
call s_mpi_barrier()
call s_delay_file_access(proc_rank)

call MPI_FILE_SET_VIEW(ifile, disp, MPI_INTEGER, MPI_IO_IB_DATA%view, 'native', mpi_info_int, ierr)
call MPI_FILE_WRITE_ALL(ifile, MPI_IO_IB_DATA%var%sf, data_size, MPI_INTEGER, status, ierr)
call MPI_FILE_CLOSE(ifile, ierr)
call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, ior(MPI_MODE_WRONLY, MPI_MODE_CREATE), mpi_info_int, ifile, ierr)

var_MOK = int(sys_size + 1, MPI_OFFSET_KIND)
disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1 + int(time_step/t_step_save))
if (time_step == 0) disp = 0

call MPI_FILE_SET_VIEW(ifile, disp, MPI_INTEGER, MPI_IO_IB_DATA%view, 'native', mpi_info_int, ierr)
call MPI_FILE_WRITE_ALL(ifile, MPI_IO_IB_DATA%var%sf, data_size, MPI_INTEGER, status, ierr)
call MPI_FILE_CLOSE(ifile, ierr)
end if
#endif

end subroutine s_write_parallel_ib_data
Expand Down
Loading