rnatm

commit d56003e4786eb41732c673967e072a03bf1fc5b4
parent 334a8fa2e463624fe0aeccb963b5a0ad97b7103b
Author: Vincent Forest <vincent.forest@meso-star.com>
Date:   Thu, 28 Jul 2022 17:59:38 +0200

Preparing to build multiple octrees

Diffstat:
M
src/rnatm_octree.c
 | 
104
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------

1 file changed, 79 insertions(+), 25 deletions(-)
diff --git a/src/rnatm_octree.c b/src/rnatm_octree.c
@@ -349,7 +349,7 @@ error:
 }
 
 static res_T
-voxelize_atmosphere(struct rnatm* atm, struct pool* pool)
+voxelize_atmosphere(struct rnatm* atm, struct pool* pools[])
 {
   struct darray_size_t_list per_thread_tetra_list;
   double atm_low[3];
@@ -376,7 +376,7 @@ voxelize_atmosphere(struct rnatm* atm, struct pool* pool)
   vxsz[2] = (atm_upp[2] - atm_low[2]) / (double)atm->grid_definition[2];
 
   /* Number of partitions required to cover the entire atmosphere grid */
-  part_def = pool_get_partition_definition(pool);
+  part_def = pool_get_partition_definition(pools[0]);
   nparts[0] = (atm->grid_definition[0] + (part_def-1)/*ceil*/) / part_def;
   nparts[1] = (atm->grid_definition[1] + (part_def-1)/*ceil*/) / part_def;
   nparts[2] = (atm->grid_definition[2] + (part_def-1)/*ceil*/) / part_def;
@@ -411,7 +411,7 @@ voxelize_atmosphere(struct rnatm* atm, struct pool* pool)
     if(ATOMIC_GET(&res) != RES_OK) continue;
 
     /* Recover current partition */
-    part = pool_next_partition(pool);
+    part = pool_next_partition(pools[0]);
     if(!part) { ATOMIC_SET(&res, RES_UNKNOWN_ERR); continue; };
 
     /* Is the partition out of bounds of the atmosphere grid*/
@@ -581,7 +581,7 @@ static res_T
 build_octrees
   (struct rnatm* atm,
    const struct rnatm_create_args* args,
-   struct pool* pool)
+   struct pool* pools[])
 {
   struct build_octrees_context ctx = BUILD_OCTREES_CONTEXT_NULL;
   struct svx_voxel_desc vx_desc = SVX_VOXEL_DESC_NULL;
@@ -589,13 +589,13 @@ build_octrees
   double low[3], upp[3];
   size_t def[3];
   res_T res = RES_OK;
-  ASSERT(atm && args && pool);
+  ASSERT(atm && args && pools);
 
   res = svx_device_create(atm->logger, &atm->svx_allocator, atm->verbose, &svx);
   if(res != RES_OK) goto error;
 
   /* Setup the build context */
-  ctx.pool = pool;
+  ctx.pool = pools[0];
   ctx.tau_threshold = args->optical_thickness;
   ctx.part = NULL;
 
@@ -632,32 +632,86 @@ error:
   goto exit;
 }
 
+static void
+delete_per_thread_pools(struct rnatm* atm, struct pool* pools[])
+{
+  size_t ipool;
+  ASSERT(atm && pools);
+  FOR_EACH(ipool, 0, atm->nthreads) {
+    if(pools[ipool]) pool_ref_put(pools[ipool]);
+  }
+  MEM_RM(atm->allocator, pools);
+}
+
+static INLINE void
+invalidate_per_thread_pools(struct rnatm* atm, struct pool* pools[])
+{
+  size_t ipool;
+  ASSERT(atm && pools);
+  FOR_EACH(ipool, 0, atm->nthreads) pool_invalidate(pools[ipool]);
+}
+
 static res_T
-create_octrees(struct rnatm* atm, const struct rnatm_create_args* args)
+create_per_thread_pools(struct rnatm* atm, struct pool** out_pools[])
 {
-  /* Empirical constant that defines the number of voxel partitions to
+  /* Empirical constant that defines the number of non empty partitions to
    * pre-allocate per thread to avoid contension between the thread building the
    * octrees from the partitions and the threads that fill these partitions */
   const size_t NPARTITIONS_PER_THREAD = 64;
 
+  /* Empirical constant that defines the total number of partitions to
+   * pre-allocate per thread. This constant is necessarily greater than or equal
+   * to NPARTITIONS_PER_THREAD, the difference representing the number of
+   * completely empty partitions. Such partitions help reduce thread contention
+   * with a sparse grid without significantly increasing memory usage */
+  const size_t OVERALL_NPARTITIONS_PER_THREAD = NPARTITIONS_PER_THREAD * 64;
+
+  /* Definition of a partition on the 3 axis */
+  const size_t PARTITION_DEFINITION = 8;
+
   struct pool_create_args pool_args = POOL_CREATE_ARGS_DEFAULT;
-  struct pool* pool = NULL;
+  struct pool** pools = NULL;
+  size_t ipool = 0;
+  res_T res = RES_OK;
+  ASSERT(atm && out_pools);
 
-  ATOMIC res = RES_OK;
-  ASSERT(atm);
+  pools = MEM_CALLOC(atm->allocator, atm->nthreads, sizeof(*pools));
+  if(!pools) { res = RES_MEM_ERR; goto error; }
 
-  /* Create the vortex partition pool */
-  pool_args.npreallocated_partitions =  NPARTITIONS_PER_THREAD * atm->nthreads;
-  pool_args.npartitions = pool_args.npreallocated_partitions * 64;
-  pool_args.partition_definition = 8;
+  /* Create the per thread partition pools */
+  pool_args.npartitions = atm->nthreads * OVERALL_NPARTITIONS_PER_THREAD;
+  pool_args.npreallocated_partitions =  atm->nthreads * NPARTITIONS_PER_THREAD;
+  pool_args.partition_definition = PARTITION_DEFINITION;
   pool_args.allocator = atm->allocator;
-  res = pool_create(&pool_args, &pool);
-  if(res != RES_OK) {
-    log_err(atm, "failed to create the voxel partition pool -- %s\n",
-      res_to_cstr((res_T)res));
-    goto error;
+  FOR_EACH(ipool, 0, atm->nthreads) {
+    res = pool_create(&pool_args, pools+ipool);
+    if(res != RES_OK) goto error;
   }
 
+exit:
+  *out_pools = pools;
+  return res;
+error:
+  log_err(atm, "failed to allocate the per thread partition pool -- %s\n",
+    res_to_cstr(res));
+  if(pools) {
+    delete_per_thread_pools(atm, pools);
+    pools = NULL;
+  }
+  goto exit;
+}
+
+static res_T
+create_octrees(struct rnatm* atm, const struct rnatm_create_args* args)
+{
+  struct pool** pools = NULL;
+
+  ATOMIC res = RES_OK;
+  ASSERT(atm);
+
+  res = create_per_thread_pools(atm, &pools);
+  if(res != RES_OK) goto error;
+
   log_info(atm, "partitionning of radiative properties (grid definition: %ux%ux%u)\n",
     SPLIT3(atm->grid_definition));
 
@@ -669,21 +723,21 @@ create_octrees(struct rnatm* atm, const struct rnatm_create_args* args)
   {
     #pragma omp section
     {
-      const res_T res_local = voxelize_atmosphere(atm, pool);
+      const res_T res_local = voxelize_atmosphere(atm, pools);
       if(res_local != RES_OK) {
         log_err(atm, "atmosphere voxelization error -- %s\n",
           res_to_cstr((res_T)res));
-        pool_invalidate(pool);
+        invalidate_per_thread_pools(atm, pools);
         ATOMIC_SET(&res, res_local);
       }
     }
 
     #pragma omp section
     {
-      const res_T res_local = build_octrees(atm, args, pool);
+      const res_T res_local = build_octrees(atm, args, pools);
       if(res_local != RES_OK) {
         log_err(atm, "error building octrees -- %s\n", res_to_cstr((res_T)res));
-        pool_invalidate(pool);
+        invalidate_per_thread_pools(atm, pools);
         ATOMIC_SET(&res, res_local);
       }
     }
@@ -691,7 +745,7 @@ create_octrees(struct rnatm* atm, const struct rnatm_create_args* args)
   if(res != RES_OK) goto error;
 
 exit:
-  if(pool) pool_ref_put(pool);
+  if(pools) delete_per_thread_pools(atm, pools);
   return (res_T)res;
 error:
   goto exit;