commit 68fe23667ceceb5aa056e21d8a7a73b9379d75eb
parent b86f53e5d1ca111de535029a180c217702cb53ed
Author: Vincent Forest <vincent.forest@meso-star.com>
Date: Wed, 7 Sep 2022 16:51:35 +0200
Write the descriptor of the octree storage
Diffstat:
3 files changed, 240 insertions(+), 96 deletions(-)
diff --git a/src/rnatm_octree.c b/src/rnatm_octree.c
@@ -24,6 +24,7 @@
#include "rnatm_c.h"
#include "rnatm_log.h"
#include "rnatm_voxel_partition.h"
+#include "rnatm_octrees_storage.h"
#include <star/sars.h>
#include <star/sck.h>
@@ -1156,31 +1157,24 @@ vx_challenge_merge
}
static res_T
-reserve_storage_header(struct rnatm* atm, fpos_t* header)
+create_storage_toc(struct rnatm* atm, fpos_t* toc)
{
- size_t noctrees = 0;
- size_t header_size = 0;
int err = 0;
res_T res = RES_OK;
-
- ASSERT(atm && header);
+ ASSERT(atm && toc);
if(!atm->octrees_storage) goto exit;
- noctrees = darray_accel_struct_size_get(&atm->accel_structs);
-
- err = fgetpos(atm->octrees_storage, header);
- if(err != 0) { res = RES_IO_ERR; goto error; }
-
- /* Compute the header size */
- header_size =
- sizeof(size_t)/*#octrees*/
- + sizeof(fpos_t)*noctrees; /* Table of offsets */
- ASSERT(header_size < LONG_MAX);
+ err = fgetpos(atm->octrees_storage, toc);
+ if(err != 0) {
+ log_err(atm, "error retrieving toc position of octrees storage -- %s\n",
+ strerror(errno));
+ res = RES_IO_ERR;
+ goto error;
+ }
- /* Reserve the space for the header */
- err = fseek(atm->octrees_storage, (long)header_size, SEEK_CUR);
- if(err != 0) { res = RES_IO_ERR; goto error; }
+ res = reserve_octrees_storage_toc(atm, atm->octrees_storage);
+ if(res != RES_OK) goto error;
exit:
return res;
@@ -1191,12 +1185,11 @@ error:
}
static res_T
-store_octrees
+offload_octrees
(struct rnatm* atm,
const size_t ioctrees[2]) /* Octrees to write. Limits are inclusive */
{
size_t ioctree = 0;
- int err = 0;
res_T res = RES_OK;
ASSERT(atm && ioctrees && ioctrees[0] <= ioctrees[1]);
ASSERT(ioctrees[1] < darray_accel_struct_size_get(&atm->accel_structs));
@@ -1204,29 +1197,11 @@ store_octrees
/* No storage: nothing to do */
if(!atm->octrees_storage) goto exit;
- FOR_EACH(ioctree, ioctrees[0], ioctrees[1]+1) {
- struct accel_struct* accel_struct = NULL;
- accel_struct = darray_accel_struct_data_get(&atm->accel_structs) + ioctree;
-
- /* Save the current file offset */
- err = fgetpos(atm->octrees_storage, &accel_struct->fpos);
- if(err != 0) {
- log_err(atm, "error retrieving octree storage position -- %s\n",
- strerror(errno));
- res = RES_IO_ERR;
- goto error;
- }
-
- /* Serialize the octree */
- res = svx_tree_write(accel_struct->octree, atm->octrees_storage);
- if(res != RES_OK) {
- log_err(atm, "error serializing octree %lu -- %s\n",
- (unsigned long)ioctree, res_to_cstr(res));
- goto error;
- }
+ res = store_octrees(atm, ioctrees, atm->octrees_storage);
+ if(res != RES_OK) goto error;
- /* Free the octree memory */
- unload_octree(atm, ioctree);
+ FOR_EACH(ioctree, ioctrees[0], ioctrees[1]+1) {
+ unload_octree(atm, ioctree); /* Free the octree memory */
}
exit:
@@ -1236,43 +1211,26 @@ error:
}
static res_T
-finalize_storage(struct rnatm* atm, const fpos_t* header)
+finalize_storage(struct rnatm* atm, const fpos_t* toc)
{
- size_t naccel_structs = 0;
- size_t iaccel_struct = 0;
int err = 0;
res_T res = RES_OK;
- ASSERT(atm && header);
+ ASSERT(atm && toc);
/* No storage: nothing to do */
if(!atm->octrees_storage) goto exit;
- err = fsetpos(atm->octrees_storage, header);
+ err = fsetpos(atm->octrees_storage, toc);
if(err != 0) {
- log_err(atm, "error positioning on octree storage header -- %s\n",
+ log_err(atm, "error positioning on octree storage toc -- %s\n",
strerror(errno));
res = RES_IO_ERR;
goto error;
}
- naccel_structs = darray_accel_struct_size_get(&atm->accel_structs);
-
- /* Write the number of voxelized octrees and their corresponding offset */
- #define WRITE(Var) { \
- if(fwrite((Var), sizeof(*(Var)), 1, atm->octrees_storage) != 1) { \
- log_err(atm, "error serializing octree storage header\n"); \
- res = RES_IO_ERR; \
- goto error; \
- } \
- } (void)0
-
- WRITE(&naccel_structs);
- FOR_EACH(iaccel_struct, 0, naccel_structs) {
- const struct accel_struct* accel_struct =
- darray_accel_struct_cdata_get(&atm->accel_structs) + iaccel_struct;
- WRITE(&accel_struct->fpos);
- }
- #undef WRITE
+ /* Update the table of content */
+ res = write_octrees_storage_toc(atm, atm->octrees_storage);
+ if(res != RES_OK) goto exit;
exit:
return res;
@@ -1287,7 +1245,7 @@ build_octrees
struct pool* pool,
struct build_sync* sync)
{
- fpos_t octrees_header;
+ fpos_t storage_toc;
struct accel_struct* accel_structs = NULL;
double low[3], upp[3];
size_t def[3];
@@ -1317,7 +1275,7 @@ build_octrees
voxel_width = pool_get_voxel_width(pool);
/* Setup the disk storage of the octrees */
- res = reserve_storage_header(atm, &octrees_header);
+ res = create_storage_toc(atm, &storage_toc);
if(res != RES_OK) goto error;
/* Build the octrees. Each thread consumes an element of the voxels generated
@@ -1375,12 +1333,12 @@ build_octrees
/* Offload builded octrees */
ioctrees[0] = istruct;
ioctrees[1] = istruct + batch_size - 1;
- res = store_octrees(atm, ioctrees);
+ res = offload_octrees(atm, ioctrees);
if(res != RES_OK) goto error;
}
/* Finalize the file where octrees are offloaded */
- res = finalize_storage(atm, &octrees_header);
+ res = finalize_storage(atm, &storage_toc);
if(res != RES_OK) goto error;
exit:
@@ -1512,6 +1470,7 @@ error:
res_T
setup_octrees(struct rnatm* atm, const struct rnatm_create_args* args)
{
+ struct octrees_storage_desc desc;
char buf[128];
size_t bands[2];
struct time t0, t1;
@@ -1529,6 +1488,14 @@ setup_octrees(struct rnatm* atm, const struct rnatm_create_args* args)
res = compute_grid_definition(atm, args);
if(res != RES_OK) goto error;
+
+ if(atm->octrees_storage) {
+ res = setup_octrees_storage_desc(atm, &desc);
+ if(res != RES_OK) goto error;
+ res = write_octrees_storage_desc(atm, &desc, atm->octrees_storage);
+ if(res != RES_OK) goto error;
+ }
+
res = find_band_range(atm, atm->spectral_range, bands);
if(res != RES_OK) goto error;
res = create_octrees(atm, args, bands);
diff --git a/src/rnatm_octrees_storage.c b/src/rnatm_octrees_storage.c
@@ -25,37 +25,59 @@
#include <star/sars.h>
#include <star/sck.h>
#include <star/suvm.h>
+#include <star/svx.h>
+
#include <rsys/cstr.h>
/*******************************************************************************
* Helper functions
******************************************************************************/
+static int
+cmp_hash256(const void* a, const void* b)
+{
+ const char* hash0 = *(const hash256_T*)a;
+ const char* hash1 = *(const hash256_T*)b;
+ size_t i;
+ FOR_EACH(i, 0, sizeof(hash256_T)) {
+ if(hash0[i] < hash1[i]) {
+ return -1;
+ } else if(hash0[i] > hash1[i]) {
+ return +1;
+ }
+ }
+ return 0;
+}
+
static res_T
-register_gas_signature
+compute_gas_signature
(const struct gas* gas,
const size_t ibands[2], /* Limits are inclusive */
- struct sha256_ctx* ctx)
+ hash256_T hash)
{
- hash256_T hash;
+ struct sha256_ctx ctx;
struct suvm_mesh_desc mesh;
size_t iband;
res_T res = RES_OK;
- ASSERT(gas && ibands && ctx && ibands[0] <= ibands[1]);
+ ASSERT(gas && ibands && hash && ibands[0] <= ibands[1]);
+
+ sha256_ctx_init(&ctx);
/* Hash the mesh */
res = suvm_volume_get_mesh_desc(gas->volume, &mesh);
if(res != RES_OK) goto error;
res = suvm_mesh_desc_compute_hash(&mesh, hash);
if(res != RES_OK) goto error;
- sha256_ctx_update(ctx, hash, sizeof(hash256_T));
+ sha256_ctx_update(&ctx, hash, sizeof(hash256_T));
/* Hash the radiative properties to consider */
FOR_EACH(iband, ibands[0], ibands[1]+1) {
res = sck_band_compute_hash(gas->ck, iband, hash);
if(res != RES_OK) goto error;
- sha256_ctx_update(ctx, hash, sizeof(hash256_T));
+ sha256_ctx_update(&ctx, hash, sizeof(hash256_T));
}
+ sha256_ctx_finalize(&ctx, hash);
+
exit:
return res;
error:
@@ -63,31 +85,35 @@ error:
}
static res_T
-register_aerosol_signature
+compute_aerosol_signature
(const struct aerosol* aerosol,
const size_t ibands[2], /* Limits are inclusive */
- struct sha256_ctx* ctx)
+ hash256_T hash)
{
- hash256_T hash;
+ struct sha256_ctx ctx;
struct suvm_mesh_desc mesh;
size_t iband;
res_T res = RES_OK;
- ASSERT(aerosol && ibands && ctx && ibands[0] <= ibands[1]);
+ ASSERT(aerosol && ibands && hash && ibands[0] <= ibands[1]);
+
+ sha256_ctx_init(&ctx);
/* Hash the mesh */
res = suvm_volume_get_mesh_desc(aerosol->volume, &mesh);
if(res != RES_OK) goto error;
res = suvm_mesh_desc_compute_hash(&mesh, hash);
if(res != RES_OK) goto error;
- sha256_ctx_update(ctx, hash, sizeof(hash256_T));
+ sha256_ctx_update(&ctx, hash, sizeof(hash256_T));
/* Hash the radiative properties to consider */
FOR_EACH(iband, ibands[0], ibands[1]+1) {
res = sars_band_compute_hash(aerosol->sars, iband, hash);
if(res != RES_OK) goto error;
- sha256_ctx_update(ctx, hash, sizeof(hash256_T));
+ sha256_ctx_update(&ctx, hash, sizeof(hash256_T));
}
+ sha256_ctx_finalize(&ctx, hash);
+
exit:
return res;
error:
@@ -97,38 +123,62 @@ error:
static res_T
compute_octrees_signature(const struct rnatm* atm, hash256_T signature)
{
- struct sha256_ctx ctx;
+ hash256_T gas_signature;
+ hash256_T* aerosol_signatures = NULL;
+ size_t naerosols;
size_t ibands[2];
size_t i;
res_T res = RES_OK;
ASSERT(atm && signature);
- sha256_ctx_init(&ctx);
-
/* Compute the gas signature for the considered spectral range */
res = sck_find_bands(atm->gas.ck, atm->spectral_range, ibands);
if(res != RES_OK) goto error;
- res = register_gas_signature(&atm->gas, ibands, &ctx);
+ res = compute_gas_signature(&atm->gas, ibands, gas_signature);
if(res != RES_OK) goto error;
- /* Compute the aerosol signatures for the considered spectral range */
- FOR_EACH(i, 0, darray_aerosol_size_get(&atm->aerosols)) {
- const struct aerosol* aerosol = darray_aerosol_cdata_get(&atm->aerosols)+i;
- res = sars_find_bands(aerosol->sars, atm->spectral_range, ibands);
- if(res != RES_OK) goto error;
- res = register_aerosol_signature(aerosol, ibands, &ctx);
- if(res != RES_OK) goto error;
- }
+ naerosols = darray_aerosol_size_get(&atm->aerosols);
+
+ if(!naerosols) {
+ memcpy(signature, gas_signature, sizeof(hash256_T));
+ } else {
+ struct sha256_ctx ctx;
+
+ aerosol_signatures = MEM_CALLOC(atm->allocator, naerosols, sizeof(hash256_T));
+ if(!aerosol_signatures) { res = RES_MEM_ERR; goto error; }
- sha256_ctx_finalize(&ctx, signature);
+ FOR_EACH(i, 0, naerosols) {
+ const struct aerosol* aerosol = darray_aerosol_cdata_get(&atm->aerosols)+i;
+ res = sars_find_bands(aerosol->sars, atm->spectral_range, ibands);
+ if(res != RES_OK) goto error;
+ res = compute_aerosol_signature(aerosol, ibands, aerosol_signatures[i]);
+ if(res != RES_OK) goto error;
+ }
+
+ /* Pay attention to the order of how aerosol hashes are registered.
+ * Therefore, the same set of aerosols will have the same signature (i.e. the
+ * same hash), regardless of the order in which the aerosols were listed in
+ * the input arguments */
+ qsort(aerosol_signatures, naerosols, sizeof(hash256_T), cmp_hash256);
+
+ /* Build the signature for the dataset */
+ sha256_ctx_init(&ctx);
+ sha256_ctx_update(&ctx, gas_signature, sizeof(hash256_T));
+ FOR_EACH(i, 0, naerosols) {
+ sha256_ctx_update(&ctx, aerosol_signatures[i], sizeof(hash256_T));
+ }
+ sha256_ctx_finalize(&ctx, signature);
+ }
exit:
+ if(aerosol_signatures) MEM_RM(atm->allocator, aerosol_signatures);
return res;
error:
log_err(atm, "signature calculation error -- %s\n", res_to_cstr(res));
goto exit;
}
+
/*******************************************************************************
* Local functions
******************************************************************************/
@@ -148,7 +198,7 @@ setup_octrees_storage_desc
desc->grid_definition[1] = atm->grid_definition[1];
desc->grid_definition[2] = atm->grid_definition[2];
- log_info(atm, "sign octree data\n");
+ log_info(atm, "sign octree data\n");
res = compute_octrees_signature(atm, desc->signature);
if(res != RES_OK) goto error;
@@ -221,3 +271,109 @@ error:
log_err(atm, "unable to read tree storage descriptor\n");
goto exit;
}
+
+res_T
+reserve_octrees_storage_toc(const struct rnatm* atm, FILE* stream)
+{
+ size_t noctrees = 0;
+ size_t sizeof_toc = 0;
+ int err;
+ res_T res = RES_OK;
+ ASSERT(atm && stream);
+
+ noctrees = darray_accel_struct_size_get(&atm->accel_structs);
+
+ /* Compute the header size */
+ sizeof_toc =
+ sizeof(size_t)/*#octrees*/
+ + sizeof(fpos_t)*noctrees; /* Table of content */
+ ASSERT(sizeof_toc < LONG_MAX);
+
+ /* Reserve the space for the table of contents */
+ err = fseek(stream, (long)sizeof_toc, SEEK_CUR);
+ if(err != 0) { res = RES_IO_ERR; goto error; }
+
+exit:
+ return res;
+error:
+ log_err(atm, "error reserving octree storage toc -- %s\n",
+ strerror(errno));
+ goto exit;
+}
+
+res_T
+write_octrees_storage_toc(const struct rnatm* atm, FILE* stream)
+{
+ size_t noctrees = 0;
+ size_t ioctree = 0;
+ res_T res = RES_OK;
+ ASSERT(atm && stream);
+
+ noctrees = darray_accel_struct_size_get(&atm->accel_structs);
+
+ /* Write the number of voxelized octrees and their corresponding offset */
+ #define WRITE(Var) { \
+ if(fwrite((Var), sizeof(*(Var)), 1, stream) != 1) { \
+ res = RES_IO_ERR; \
+ goto error; \
+ } \
+ } (void)0
+ WRITE(&noctrees);
+ FOR_EACH(ioctree, 0, noctrees) {
+ const struct accel_struct* accel_struct =
+ darray_accel_struct_cdata_get(&atm->accel_structs) + ioctree;
+ WRITE(&accel_struct->fpos);
+ }
+ #undef WRITE
+
+exit:
+ return res;
+error:
+ log_err(atm, "error writing octree table of content\n");
+ goto exit;
+}
+
+res_T
+read_octrees_storage_toc(const struct rnatm* atm, FILE* stream)
+{
+ /* TODO */
+ (void)atm, (void)stream;
+ FATAL("Not implemented\n");
+ return RES_BAD_OP;
+}
+
+extern LOCAL_SYM res_T
+store_octrees(struct rnatm* atm, const size_t ioctrees[2], FILE* stream)
+{
+ size_t ioctree;
+ int err;
+ res_T res = RES_OK;
+ ASSERT(atm && ioctrees && stream && ioctrees[0] <= ioctrees[1]);
+
+ FOR_EACH(ioctree, ioctrees[0], ioctrees[1]+1) {
+ struct accel_struct* accel_struct = NULL;
+ accel_struct = darray_accel_struct_data_get(&atm->accel_structs) + ioctree;
+
+ /* Save the current file offset */
+ err = fgetpos(stream, &accel_struct->fpos);
+ if(err != 0) {
+ log_err(atm, "error retrieving octree storage position -- %s\n",
+ strerror(errno));
+ res = RES_IO_ERR;
+ goto error;
+ }
+
+ /* Serialize the octree */
+ res = svx_tree_write(accel_struct->octree, stream);
+ if(res != RES_OK) {
+ log_err(atm, "error writing octree %lu -- %s\n",
+ (unsigned long)ioctree, res_to_cstr(res));
+ goto error;
+ }
+ }
+
+exit:
+ return res;
+error:
+ goto exit;
+}
diff --git a/src/rnatm_octrees_storage.h b/src/rnatm_octrees_storage.h
@@ -57,4 +57,25 @@ read_octrees_storage_desc
struct octrees_storage_desc* desc,
FILE* stream);
+extern LOCAL_SYM res_T
+reserve_octrees_storage_toc
+ (const struct rnatm* atm,
+ FILE* stream);
+
+extern LOCAL_SYM res_T
+write_octrees_storage_toc
+ (const struct rnatm* atm,
+ FILE* stream);
+
+extern LOCAL_SYM res_T
+read_octrees_storage_toc
+ (const struct rnatm* atm,
+ FILE* stream);
+
+extern LOCAL_SYM res_T
+store_octrees
+ (struct rnatm* atm,
+ const size_t ioctrees[2],
+ FILE* stream);
+
#endif /* RNATM_OCTREES_STORAGE_H */
