commit b304fcd71625e8b630831e54bf20c168036bec24
parent 13595a239be0c313f269dc69916ce97b1baccd47
Author: Vincent Forest <vincent.forest@meso-star.com>
Date: Thu, 25 Oct 2018 17:57:00 +0200
Report the progress status of each MPI process
Diffstat:
5 files changed, 180 insertions(+), 78 deletions(-)
diff --git a/src/htrdr.c b/src/htrdr.c
@@ -248,6 +248,7 @@ spherical_to_cartesian_dir
static res_T
init_mpi(struct htrdr* htrdr)
{
+ size_t n;
int err;
res_T res = RES_OK;
ASSERT(htrdr);
@@ -280,6 +281,29 @@ init_mpi(struct htrdr* htrdr)
goto error;
}
+ /* Allocate #processes progress statuses on the master process and only 1
+ * progress status on the other ones: the master process will gather the
+ * status of the other processes to report their progression. */
+ n = (size_t)(htrdr->mpi_rank == 0 ? htrdr->mpi_nprocs : 1);
+
+ htrdr->mpi_progress_octree = MEM_CALLOC
+ (htrdr->allocator, n, sizeof(*htrdr->mpi_progress_octree));
+ if(!htrdr->mpi_progress_octree) {
+ htrdr_log_err(htrdr,
+ "could not allocate the progress state of the octree building.\n");
+ res = RES_MEM_ERR;
+ goto error;
+ }
+
+ htrdr->mpi_progress_render = MEM_CALLOC
+ (htrdr->allocator, n, sizeof(*htrdr->mpi_progress_render));
+ if(!htrdr->mpi_progress_render) {
+ htrdr_log_err(htrdr,
+ "could not allocate the progress state of the scene rendering.\n");
+ res = RES_MEM_ERR;
+ goto error;
+ }
+
exit:
return res;
error:
@@ -451,6 +475,12 @@ htrdr_release(struct htrdr* htrdr)
if(htrdr->cam) htrdr_camera_ref_put(htrdr->cam);
if(htrdr->buf) htrdr_buffer_ref_put(htrdr->buf);
if(htrdr->mpi_err_str) MEM_RM(htrdr->allocator, htrdr->mpi_err_str);
+ if(htrdr->mpi_progress_octree) {
+ MEM_RM(htrdr->allocator, htrdr->mpi_progress_octree);
+ }
+ if(htrdr->mpi_progress_render) {
+ MEM_RM(htrdr->allocator, htrdr->mpi_progress_render);
+ }
if(htrdr->lifo_allocators) {
size_t i;
FOR_EACH(i, 0, htrdr->nthreads) {
@@ -757,3 +787,84 @@ error:
goto exit;
}
+void
+fetch_mpi_progress(struct htrdr* htrdr, const enum htrdr_mpi_progress tag)
+{
+ int8_t* progress = NULL;
+ int iproc;
+ ASSERT(htrdr && progress && htrdr->mpi_rank == 0);
+
+ switch(tag) {
+ case HTRDR_MPI_PROGRESS_BUILD_OCTREE:
+ progress = htrdr->mpi_progress_octree;
+ break;
+ case HTRDR_MPI_PROGRESS_RENDERING:
+ progress = htrdr->mpi_progress_render;
+ break;
+ default: FATAL("Unreachable code.\n"); break;
+ }
+
+ FOR_EACH(iproc, 1, htrdr->mpi_nprocs) {
+
+ /* Flush the last sent percentage of the process `iproc' */
+ for(;;) {
+ int flag;
+
+ CHK(MPI_Iprobe
+ (iproc, tag, MPI_COMM_WORLD, &flag, MPI_STATUS_IGNORE) == MPI_SUCCESS);
+ if(flag == 0) break; /* No more message */
+
+ CHK(MPI_Recv(&progress[iproc], sizeof(size_t), MPI_CHAR, iproc, tag,
+ MPI_COMM_WORLD, MPI_STATUS_IGNORE) == MPI_SUCCESS);
+ }
+ }
+}
+
+void
+print_mpi_progress(struct htrdr* htrdr, const enum htrdr_mpi_progress tag)
+{
+ int iproc;
+ ASSERT(htrdr && htrdr->mpi_rank == 0);
+ FOR_EACH(iproc, 0, htrdr->mpi_nprocs) {
+ switch(tag) {
+ case HTRDR_MPI_PROGRESS_BUILD_OCTREE:
+ htrdr_fprintf(htrdr, stderr,
+ "\033[2K\rProcess %d -- building octree: %3d%%\n",
+ iproc, htrdr->mpi_progress_octree[iproc]);
+ break;
+ case HTRDR_MPI_PROGRESS_RENDERING:
+ htrdr_fprintf(htrdr, stderr,
+ "\033[2K\rProcess %d -- rendering: %3d%%\n",
+ iproc, htrdr->mpi_progress_render[iproc]);
+ break;
+ default: FATAL("Unreachable code.\n"); break;
+ }
+ }
+}
+
+int8_t
+total_mpi_progress(const struct htrdr* htrdr, const enum htrdr_mpi_progress tag)
+{
+ const int8_t* progress = NULL;
+ int total = 0;
+ int iproc;
+ ASSERT(htrdr && progress && htrdr->mpi_rank == 0);
+
+ switch(tag) {
+ case HTRDR_MPI_PROGRESS_BUILD_OCTREE:
+ progress = htrdr->mpi_progress_octree;
+ break;
+ case HTRDR_MPI_PROGRESS_RENDERING:
+ progress = htrdr->mpi_progress_render;
+ break;
+ default: FATAL("Unreachable code.\n"); break;
+ }
+
+ FOR_EACH(iproc, 0, htrdr->mpi_nprocs) {
+ total += progress[iproc];
+ }
+ total = total / htrdr->mpi_nprocs;
+ ASSERT(total <= 100);
+ return (int8_t)total;
+}
+
diff --git a/src/htrdr.h b/src/htrdr.h
@@ -65,6 +65,10 @@ struct htrdr {
int mpi_nprocs; /* Overall #processes in the MPI group */
char* mpi_err_str; /* Temp buffer used to store MPI error string */
+ /* Process progress percentage */
+ int8_t* mpi_progress_octree;
+ int8_t* mpi_progress_render;
+
struct logger logger;
struct mem_allocator* allocator;
struct mem_allocator* lifo_allocators; /* Per thread lifo allocator */
diff --git a/src/htrdr_c.h b/src/htrdr_c.h
@@ -18,6 +18,11 @@
#include <rsys/rsys.h>
+enum htrdr_mpi_progress {
+ HTRDR_MPI_PROGRESS_BUILD_OCTREE,
+ HTRDR_MPI_PROGRESS_RENDERING
+};
+
struct htrdr;
#define SW_WAVELENGTH_MIN 380 /* In nanometer */
@@ -96,7 +101,7 @@ is_file_updated
extern LOCAL_SYM res_T
update_file_stamp
- (struct htrdr* htrdr,
+ (struct htrdr* htrdr,
const char* filename);
extern LOCAL_SYM res_T
@@ -104,5 +109,20 @@ create_directory
(struct htrdr* htrdt,
const char* path);
+extern LOCAL_SYM void
+fetch_mpi_progress
+ (struct htrdr* htrdr,
+ const enum htrdr_mpi_progress progress);
+
+extern LOCAL_SYM void
+print_mpi_progress
+ (struct htrdr* htrdr,
+ const enum htrdr_mpi_progress progress);
+
+extern int8_t
+total_mpi_progress
+ (const struct htrdr* htrdr,
+ const enum htrdr_mpi_progress progress);
+
#endif /* HTRDR_C_H */
diff --git a/src/htrdr_draw_radiance_sw.c b/src/htrdr_draw_radiance_sw.c
@@ -208,36 +208,6 @@ draw_tile
return RES_OK;
}
-/* Return the overall percentage */
-static size_t
-fetch_process_progress(struct htrdr* htrdr, size_t* progress)
-{
- size_t overall_pcent;
- int iproc;
- ASSERT(htrdr && progress && htrdr->mpi_rank == 0);
-
- overall_pcent = progress[0];
- FOR_EACH(iproc, 1, htrdr->mpi_nprocs) {
- size_t proc_pcent = progress[iproc];
-
- /* Flush the last sent percentage of the process `iproc' */
- for(;;) {
- int flag;
-
- CHK(MPI_Iprobe(iproc, 0/*tag*/, MPI_COMM_WORLD, &flag,
- MPI_STATUS_IGNORE) == MPI_SUCCESS);
- if(flag == 0) break; /* No more message */
-
- CHK(MPI_Recv(&proc_pcent, sizeof(size_t), MPI_CHAR, iproc, 0/*tag*/,
- MPI_COMM_WORLD, MPI_STATUS_IGNORE) == MPI_SUCCESS);
- }
-
- progress[iproc] = proc_pcent;
- overall_pcent += progress[iproc];
- }
- return overall_pcent / (size_t)htrdr->mpi_nprocs;
-}
-
/*******************************************************************************
* Local functions
******************************************************************************/
@@ -251,13 +221,11 @@ htrdr_draw_radiance_sw
struct ssp_rng_proxy* rng_proxy = NULL;
struct ssp_rng** rngs = NULL;
size_t ntiles_x, ntiles_y, ntiles, ntiles_adjusted;
- size_t* progress = NULL;
size_t i;
int32_t mcode; /* Morton code of the tile */
struct htrdr_buffer_layout layout;
double pix_sz[2]; /* Pixel size in the normalized image plane */
size_t spp;
- size_t overall_pcent = 0;
ATOMIC nsolved_tiles = 0;
ATOMIC res = RES_OK;
ASSERT(htrdr && cam && buf);
@@ -311,20 +279,6 @@ htrdr_draw_radiance_sw
}
}
- if(htrdr->mpi_rank == 0) {
- progress = MEM_CALLOC
- (htrdr->allocator, (size_t)htrdr->mpi_nprocs, sizeof(size_t));
- } else {
- progress = MEM_CALLOC
- (htrdr->allocator, 1, sizeof(size_t));
- }
- if(!progress) {
- htrdr_log_err(htrdr,
- "%s: could not allocate the process progress counter.\n", FUNC_NAME);
- res = RES_MEM_ERR;
- goto error;
- }
-
ntiles_x = (layout.width + (TILE_SIZE-1)/*ceil*/)/TILE_SIZE;
ntiles_y = (layout.height+ (TILE_SIZE-1)/*ceil*/)/TILE_SIZE;
ntiles_adjusted = round_up_pow2(MMAX(ntiles_x, ntiles_y));
@@ -334,8 +288,10 @@ htrdr_draw_radiance_sw
pix_sz[0] = 1.0 / (double)layout.width;
pix_sz[1] = 1.0 / (double)layout.height;
- htrdr_fprintf(htrdr, stderr, "Rendering: %3i%%", 0);
- htrdr_fflush(htrdr, stderr);
+ if(htrdr->mpi_rank == 0) {
+ fetch_mpi_progress(htrdr, HTRDR_MPI_PROGRESS_RENDERING);
+ print_mpi_progress(htrdr, HTRDR_MPI_PROGRESS_RENDERING);
+ }
omp_set_num_threads((int)htrdr->nthreads);
#pragma omp parallel for schedule(static, 1/*chunck size*/)
@@ -344,9 +300,9 @@ htrdr_draw_radiance_sw
struct ssp_rng* rng = rngs[ithread];
size_t tile_org[2];
size_t tile_sz[2];
- size_t pcent;
size_t n;
res_T res_local = RES_OK;
+ int8_t pcent;
/* Decode the morton code to retrieve the tile index */
tile_org[0] = morton2D_decode((uint32_t)(mcode>>0));
@@ -370,21 +326,19 @@ htrdr_draw_radiance_sw
}
n = (size_t)ATOMIC_INCR(&nsolved_tiles);
- pcent = n * 100 / ntiles;
+ pcent = (int8_t)(n * 100 / ntiles);
#pragma omp critical
- if((size_t)pcent > progress[0]) {
- progress[0] = pcent;
-
+ if(pcent > htrdr->mpi_progress_render[0]) {
+ htrdr->mpi_progress_render[0] = pcent;
if(htrdr->mpi_rank != 0) {
/* Send the progress percentage of the process to the master process */
- CHK(MPI_Send(&pcent, sizeof(size_t), MPI_CHAR, 0/*dst*/, 0/*tag*/,
- MPI_COMM_WORLD) == MPI_SUCCESS);
+ CHK(MPI_Send(&pcent, sizeof(pcent), MPI_CHAR, 0/*dst*/,
+ HTRDR_MPI_PROGRESS_RENDERING/*tag*/, MPI_COMM_WORLD) == MPI_SUCCESS);
} else {
- overall_pcent = fetch_process_progress(htrdr, progress);
- htrdr_fprintf(htrdr, stderr, "%c[2K\rRendering: %3lu%%",
- 27, (unsigned long)overall_pcent);
- htrdr_fflush(htrdr, stderr);
+ fetch_mpi_progress(htrdr, HTRDR_MPI_PROGRESS_RENDERING);
+ htrdr_fprintf(htrdr, stderr, "\033[%dA", htrdr->mpi_nprocs);
+ print_mpi_progress(htrdr, HTRDR_MPI_PROGRESS_RENDERING);
}
}
@@ -392,14 +346,12 @@ htrdr_draw_radiance_sw
}
if(htrdr->mpi_rank == 0) {
- while(overall_pcent != 100) {
- overall_pcent = fetch_process_progress(htrdr, progress);
- htrdr_fprintf(htrdr, stderr, "%c[2K\rRendering: %3lu%%",
- 27, (unsigned long)overall_pcent);
- htrdr_fflush(htrdr, stderr);
+ while(total_mpi_progress(htrdr, HTRDR_MPI_PROGRESS_RENDERING) != 100) {
+ fetch_mpi_progress(htrdr, HTRDR_MPI_PROGRESS_RENDERING);
+ htrdr_fprintf(htrdr, stderr, "\033[%dA", htrdr->mpi_nprocs);
+ print_mpi_progress(htrdr, HTRDR_MPI_PROGRESS_RENDERING);
sleep(1);
}
- htrdr_fprintf(htrdr, stderr, "\n");
}
/* Gather accum buffers from the group of processes */
@@ -408,7 +360,6 @@ htrdr_draw_radiance_sw
exit:
if(rng_proxy) SSP(rng_proxy_ref_put(rng_proxy));
- if(progress) MEM_RM(htrdr->allocator, progress);
if(rngs) {
FOR_EACH(i, 0, htrdr->nthreads) {
if(rngs[i]) SSP(rng_ref_put(rngs[i]));
diff --git a/src/htrdr_sky.c b/src/htrdr_sky.c
@@ -38,6 +38,7 @@
#include <fcntl.h> /* open */
#include <libgen.h>
#include <math.h>
+#include <mpi.h>
#include <omp.h>
#include <string.h>
#include <sys/stat.h> /* mkdir */
@@ -1076,7 +1077,6 @@ setup_clouds
{
struct darray_specdata specdata;
size_t nvoxs[3];
- size_t progress = 0;
double vxsz[3];
double low[3];
double upp[3];
@@ -1205,8 +1205,10 @@ setup_clouds
omp_set_num_threads(1);
} else {
omp_set_num_threads((int)sky->htrdr->nthreads);
- htrdr_fprintf(sky->htrdr, stderr, "Building octrees: %3u%%", 0);
- htrdr_fflush(sky->htrdr, stderr);
+ if(sky->htrdr->mpi_rank == 0) {
+ fetch_mpi_progress(sky->htrdr, HTRDR_MPI_PROGRESS_BUILD_OCTREE);
+ print_mpi_progress(sky->htrdr, HTRDR_MPI_PROGRESS_BUILD_OCTREE);
+ }
}
#pragma omp parallel for
for(ispecdata=0;
@@ -1217,7 +1219,7 @@ setup_clouds
const size_t iband = darray_specdata_data_get(&specdata)[ispecdata].iband;
const size_t iquad = darray_specdata_data_get(&specdata)[ispecdata].iquad;
const size_t id = iband - sky->sw_bands_range[0];
- size_t pcent;
+ int8_t pcent;
size_t n;
res_T res_local = RES_OK;
@@ -1266,18 +1268,32 @@ setup_clouds
if(!sky->htrdr->cache_grids) {
/* Update the progress message */
n = (size_t)ATOMIC_INCR(&nbuilt_octrees);
- pcent = n * 100 / darray_specdata_size_get(&specdata);
+ pcent = (int8_t)(n * 100 / darray_specdata_size_get(&specdata));
#pragma omp critical
- if(pcent > progress) {
- progress = pcent;
- htrdr_fprintf(sky->htrdr, stderr,
- "%c[2K\rBuilding octrees: %3u%%", 27, (unsigned)pcent);
- htrdr_fflush(sky->htrdr, stderr);
+ if(pcent > sky->htrdr->mpi_progress_octree[0]) {
+ sky->htrdr->mpi_progress_octree[0] = pcent;
+ if(sky->htrdr->mpi_rank != 0) {
+ /* Send the progress percentage of the process to the master process */
+ CHK(MPI_Send(&pcent, sizeof(pcent), MPI_CHAR, 0/*dst*/,
+ HTRDR_MPI_PROGRESS_BUILD_OCTREE, MPI_COMM_WORLD) == MPI_SUCCESS);
+ } else {
+ fetch_mpi_progress(sky->htrdr, HTRDR_MPI_PROGRESS_BUILD_OCTREE);
+ htrdr_fprintf(sky->htrdr, stderr, "\033[%dA", sky->htrdr->mpi_nprocs);
+ print_mpi_progress(sky->htrdr, HTRDR_MPI_PROGRESS_BUILD_OCTREE);
+ }
}
}
}
- if(!sky->htrdr->cache_grids) htrdr_fprintf(sky->htrdr, stderr, "\n");
+
+ if(!sky->htrdr->cache_grids && sky->htrdr->mpi_rank == 0) {
+ while(total_mpi_progress(sky->htrdr, HTRDR_MPI_PROGRESS_BUILD_OCTREE) != 100) {
+ fetch_mpi_progress(sky->htrdr, HTRDR_MPI_PROGRESS_BUILD_OCTREE);
+ htrdr_fprintf(sky->htrdr, stderr, "\033[%dA", sky->htrdr->mpi_nprocs);
+ print_mpi_progress(sky->htrdr, HTRDR_MPI_PROGRESS_BUILD_OCTREE);
+ sleep(1);
+ }
+ }
exit:
darray_specdata_release(&specdata);
