diff --git a/src/post_process/m_data_input.f90 b/src/post_process/m_data_input.f90 index 9617766a2..19de8e9df 100644 --- a/src/post_process/m_data_input.f90 +++ b/src/post_process/m_data_input.f90 @@ -104,22 +104,27 @@ end subroutine s_setup_mpi_io_params !> Helper subroutine to read IB data files impure subroutine s_read_ib_data_files(file_loc_base, t_step) - character(len=*), intent(in) :: file_loc_base - integer, intent(in), optional :: t_step - character(LEN=len_trim(file_loc_base) + 20) :: file_loc - logical :: file_exist - integer :: ifile, ierr, data_size + character(len=*), intent(in) :: file_loc_base + integer, intent(in), optional :: t_step + character(LEN=path_len + 2*name_len) :: file_loc + logical :: file_exist + integer :: ifile, ierr, data_size #ifdef MFC_MPI integer, dimension(MPI_STATUS_SIZE) :: status integer(KIND=MPI_OFFSET_KIND) :: disp integer(KIND=MPI_OFFSET_KIND) :: m_MOK, n_MOK, p_MOK, MOK, WP_MOK, var_MOK integer :: save_index + character(len=10) :: t_step_string #endif if (.not. ib) return - if (parallel_io) then + if (parallel_io .and. file_per_process) then + call s_int_to_str(t_step, t_step_string) + write (file_loc, '(A,I0,A,i7.7,A)') 'ib_markers_', t_step, '_', proc_rank, '.dat' + file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc) + else if (parallel_io) then write (file_loc, '(A)') trim(file_loc_base) // 'ib.dat' else write (file_loc, '(A)') trim(file_loc_base) // '/ib_data.dat' @@ -127,7 +132,17 @@ impure subroutine s_read_ib_data_files(file_loc_base, t_step) inquire (FILE=trim(file_loc), EXIST=file_exist) if (file_exist) then - if (parallel_io) then + if (parallel_io .and. file_per_process) then +#ifdef MFC_MPI + call MPI_FILE_OPEN(MPI_COMM_SELF, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr) + + data_size = (m + 1)*(n + 1)*(p + 1) + + call MPI_FILE_READ(ifile, MPI_IO_IB_DATA%var%sf, data_size, MPI_INTEGER, status, ierr) + + call MPI_FILE_CLOSE(ifile, ierr) +#endif + else if (parallel_io) then #ifdef MFC_MPI call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr) diff --git a/src/simulation/m_data_output.fpp b/src/simulation/m_data_output.fpp index 6c923b334..239d8df08 100644 --- a/src/simulation/m_data_output.fpp +++ b/src/simulation/m_data_output.fpp @@ -880,6 +880,7 @@ contains integer(kind=MPI_OFFSET_kind) :: WP_MOK, var_MOK, MOK integer :: ifile, ierr, data_size integer, dimension(MPI_STATUS_SIZE) :: status + character(len=10) :: t_step_string $:GPU_UPDATE(host='[ib_markers%sf]') @@ -890,21 +891,39 @@ contains WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) - write (file_loc, '(A)') 'ib.dat' - file_loc = trim(case_dir) // '/restart_data' // trim(mpiiofs) // trim(file_loc) + if (file_per_process) then + call s_int_to_str(time_step, t_step_string) - call s_mpi_barrier() - call s_delay_file_access(proc_rank) + if (proc_rank == 0) then + file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) + call s_create_directory(trim(file_loc)) + end if + call s_mpi_barrier() + call s_delay_file_access(proc_rank) + + write (file_loc, '(A,I0,A,i7.7,A)') 'ib_markers_', time_step, '_', proc_rank, '.dat' + file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc) - call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, ior(MPI_MODE_WRONLY, MPI_MODE_CREATE), mpi_info_int, ifile, ierr) + call MPI_FILE_OPEN(MPI_COMM_SELF, file_loc, ior(MPI_MODE_WRONLY, MPI_MODE_CREATE), mpi_info_int, ifile, ierr) + call MPI_FILE_WRITE_ALL(ifile, MPI_IO_IB_DATA%var%sf, data_size, MPI_INTEGER, status, ierr) + call MPI_FILE_CLOSE(ifile, ierr) + else + write (file_loc, '(A)') 'ib.dat' + file_loc = trim(case_dir) // '/restart_data' // trim(mpiiofs) // trim(file_loc) - var_MOK = int(sys_size + 1, MPI_OFFSET_KIND) - disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1 + int(time_step/t_step_save)) - if (time_step == 0) disp = 0 + call s_mpi_barrier() + call s_delay_file_access(proc_rank) - call MPI_FILE_SET_VIEW(ifile, disp, MPI_INTEGER, MPI_IO_IB_DATA%view, 'native', mpi_info_int, ierr) - call MPI_FILE_WRITE_ALL(ifile, MPI_IO_IB_DATA%var%sf, data_size, MPI_INTEGER, status, ierr) - call MPI_FILE_CLOSE(ifile, ierr) + call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, ior(MPI_MODE_WRONLY, MPI_MODE_CREATE), mpi_info_int, ifile, ierr) + + var_MOK = int(sys_size + 1, MPI_OFFSET_KIND) + disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1 + int(time_step/t_step_save)) + if (time_step == 0) disp = 0 + + call MPI_FILE_SET_VIEW(ifile, disp, MPI_INTEGER, MPI_IO_IB_DATA%view, 'native', mpi_info_int, ierr) + call MPI_FILE_WRITE_ALL(ifile, MPI_IO_IB_DATA%var%sf, data_size, MPI_INTEGER, status, ierr) + call MPI_FILE_CLOSE(ifile, ierr) + end if #endif end subroutine s_write_parallel_ib_data