commit 525d8a8d49ec713429d2165ae62b686a2a84a9ab
parent 44b4a606903dfe5c77327dfb4a20aa82b1c1a339
Author: Vincent Forest <vincent.forest@meso-star.com>
Date: Fri, 13 Oct 2023 15:30:41 +0200
Make memory mapping optional
By default, data is now loaded into memory. Memory mapping becomes an
option of the load functions. As a consequence, this commit introduces
API breaks. But we're taking advantage of this update to group the
input arguments in a specific structure. Future maintenance of the
calling code should therefore be simplified when adding a new optional
argument if the said structure is initialized with its default value.
Diffstat:
| M | src/sck.c | | | 449 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------- |
| M | src/sck.h | | | 23 | ++++++++++++++++++++--- |
| M | src/sck_c.h | | | 47 | ++++++++++++++++++++++++++++++----------------- |
| M | src/test_sck_load.c | | | 61 | ++++++++++++++++++++++++++++++++++++++++++++----------------- |
4 files changed, 467 insertions(+), 113 deletions(-)
diff --git a/src/sck.c b/src/sck.c
@@ -23,6 +23,7 @@
#include "sck_log.h"
#include <rsys/algorithm.h>
+#include <rsys/cstr.h>
#include <unistd.h> /* sysconf support */
@@ -40,6 +41,20 @@ check_sck_create_args(const struct sck_create_args* args)
return args ? RES_OK : RES_BAD_ARG;
}
+static INLINE res_T
+check_sck_load_args(const struct sck_load_args* args)
+{
+ if(!args || !args->path) return RES_BAD_ARG;
+ return RES_OK;
+}
+
+static INLINE res_T
+check_sck_load_stream_args(const struct sck_load_stream_args* args)
+{
+ if(!args || !args->stream || !args->name) return RES_BAD_ARG;
+ return RES_OK;
+}
+
static void
reset_sck(struct sck* sck)
{
@@ -69,6 +84,8 @@ read_quad_pt
res_T res = RES_OK;
ASSERT(sck && quad_pt && stream_name);
+ quad_pt->band = darray_band_data_get(&sck->bands) + iband;
+
if(fread(&quad_pt->weight, sizeof(quad_pt->weight), 1, stream) != 1) {
log_err(sck,
"%s: band %lu: quadrature point %lu: could not read the weight.\n",
@@ -108,6 +125,7 @@ read_band
res_T res = RES_OK;
ASSERT(sck && band && stream_name);
+ band->sck = sck;
iband = (size_t)(band - darray_band_cdata_get(&sck->bands));
/* Read band definition */
@@ -196,26 +214,279 @@ error:
}
static res_T
-load_stream(struct sck* sck, FILE* stream, const char* stream_name)
+map_data
+ (struct sck* sck,
+ const char* name,
+ const int fd, /* File descriptor */
+ const size_t filesz, /* Overall filesize */
+ const char* data_name,
+ const off_t offset, /* Offset of the data into file */
+ const size_t map_len,
+ void** out_map) /* Lenght of the data to map */
{
+ void* map = NULL;
+ res_T res = RES_OK;
+ ASSERT(sck && name && filesz && data_name && map_len && out_map);
+ ASSERT(IS_ALIGNED((size_t)offset, (size_t)sck->pagesize));
+
+ if((size_t)offset + map_len > filesz) {
+ log_err(sck, "%s: the %s to map exceed the file size\n",
+ name, data_name);
+ res = RES_IO_ERR;
+ goto error;
+ }
+
+ map = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE|MAP_POPULATE, fd, offset);
+ if(map == MAP_FAILED) {
+ log_err(sck, "%s: could not map the %s -- %s\n",
+ name, data_name, strerror(errno));
+ res = RES_IO_ERR;
+ goto error;
+ }
+
+exit:
+ *out_map = map;
+ return res;
+error:
+ if(map == MAP_FAILED) map = NULL;
+ goto exit;
+}
+
+static res_T
+map_file(struct sck* sck, FILE* stream, const char* stream_name)
+{
+ size_t filesz;
size_t map_len;
size_t iband;
- uint64_t nbands;
- off_t offset = 0;
+ off_t offset;
res_T res = RES_OK;
ASSERT(sck && stream && stream_name);
+ /* Compute the length in bytes of the k to map for each band/quadrature point */
+ map_len = ALIGN_SIZE(sck->nnodes * sizeof(float), sck->pagesize);
+
+ /* Compute the offset toward the 1st list of radiative coefficients */
+ offset = ftell(stream);
+ offset = (off_t)ALIGN_SIZE((uint64_t)offset, sck->pagesize);
+
+ /* Retrieve the overall filesize */
+ fseek(stream, 0, SEEK_END);
+ filesz = (size_t)ftell(stream);
+
+ FOR_EACH(iband, 0, darray_band_size_get(&sck->bands)) {
+ struct band* band = NULL;
+ size_t iquad_pt;
+
+ band = darray_band_data_get(&sck->bands) + iband;
+ band->map_len = map_len;
+
+ /* Mapping per band scattering coefficients */
+ res = map_data(sck, stream_name, fileno(stream), filesz,
+ "scattering coefficients", offset, band->map_len,
+ (void**)&band->ks_list);
+ if(res != RES_OK) {
+ log_err(sck,
+ "%s: data mapping error for band %lu\n",
+ stream_name, (unsigned long)iband);
+ goto error;
+ }
+
+ offset = (off_t)((size_t)offset + map_len);
+
+ FOR_EACH(iquad_pt, 0, darray_quad_pt_size_get(&band->quad_pts)) {
+ struct quad_pt* quad_pt = NULL;
+
+ /* Mapping absorption coefficients per band and quadrature point */
+ quad_pt = darray_quad_pt_data_get(&band->quad_pts)+iquad_pt;
+ quad_pt->map_len = map_len;
+
+ res = map_data(sck, stream_name, fileno(stream), filesz, "correlated Ka",
+ offset, quad_pt->map_len, (void**)&quad_pt->ka_list);
+ if(res != RES_OK) {
+ log_err(sck,
+ "%s: data mapping error for band %lu quadrature point %lu\n",
+ stream_name, (unsigned long)iband, (unsigned long)iquad_pt);
+ goto error;
+ }
+
+ offset = (off_t)((size_t)offset + map_len);
+ }
+ }
+
+exit:
+ return res;
+error:
+ goto exit;
+}
+
+static res_T
+read_padding(FILE* stream, const size_t padding)
+{
+ char chunk[1024];
+ size_t remaining_nbytes = padding;
+
+ while(remaining_nbytes) {
+ const size_t nbytes = MMIN(sizeof(chunk), remaining_nbytes);
+ if(fread(chunk, 1, nbytes, stream) != nbytes) return RES_IO_ERR;
+ remaining_nbytes -= nbytes;
+ }
+ return RES_OK;
+}
+
+/* Return the size in bytes of the data layout and band descriptors */
+static size_t
+compute_sizeof_header(struct sck* sck)
+{
+ size_t sizeof_header = 0;
+ size_t iband = 0;
+ size_t nbands = 0;
+ ASSERT(sck);
+
+ sizeof_header =
+ sizeof(uint64_t) /* pagesize */
+ + sizeof(uint64_t) /* #bands */
+ + sizeof(uint64_t);/* #nodes */
+
+ nbands = darray_band_size_get(&sck->bands);
+ FOR_EACH(iband, 0, nbands) {
+ const struct band* band = darray_band_cdata_get(&sck->bands)+iband;
+ const size_t nquad_pts = darray_quad_pt_size_get(&band->quad_pts);
+ const size_t sizeof_band_desc =
+ sizeof(double) /* Lower bound */
+ + sizeof(double) /* Upper bound */
+ + sizeof(uint64_t) /* #quad_pts */
+ + sizeof(double) * nquad_pts; /* Weights */
+
+ sizeof_header += sizeof_band_desc;
+ }
+ return sizeof_header;
+}
+
+static res_T
+load_data
+ (struct sck* sck,
+ FILE* stream,
+ const char* stream_name,
+ const char* data_name,
+ float** out_data)
+{
+ float* data = NULL;
+ res_T res = RES_OK;
+ ASSERT(sck && stream && stream_name && data_name && out_data);
+
+ data = MEM_ALLOC(sck->allocator, sizeof(float)*sck->nnodes);
+ if(!data) {
+ res = RES_MEM_ERR;
+ log_err(sck, "%s: could not allocate the %s -- %s\n",
+ stream_name, data_name, res_to_cstr(res));
+ goto error;
+ }
+
+ if(fread(data, sizeof(float), sck->nnodes, stream) != sck->nnodes) {
+ res = RES_IO_ERR;
+ log_err(sck, "%s: could not read the %s -- %s\n",
+ stream_name, data_name, res_to_cstr(res));
+ goto error;
+ }
+
+exit:
+ *out_data = data;
+ return res;
+error:
+ if(data) { MEM_RM(sck->allocator, data); data = NULL; }
+ goto exit;
+}
+
+static res_T
+load_file(struct sck* sck, FILE* stream, const char* stream_name)
+{
+ size_t padding_bytes;
+ size_t sizeof_header;
+ size_t sizeof_k_list;
+ size_t iband;
+ size_t nbands;
+ res_T res = RES_OK;
+
+ sizeof_header = compute_sizeof_header(sck);
+ sizeof_k_list = sizeof(float)*sck->nnodes;
+
+ padding_bytes = ALIGN_SIZE(sizeof_header, sck->pagesize) - sizeof_header;
+ if((res = read_padding(stream, padding_bytes)) != RES_OK) goto error;
+
+ /* Calculate the padding between the lists of radiative coefficients. Note
+ * that this padding is the same between each list */
+ padding_bytes = ALIGN_SIZE(sizeof_k_list, sck->pagesize) - sizeof_k_list;
+
+ nbands = darray_band_size_get(&sck->bands);
+ FOR_EACH(iband, 0, nbands) {
+ struct band* band = NULL;
+ size_t iquad_pt = 0;
+
+ band = darray_band_data_get(&sck->bands) + iband;
+ ASSERT(!band->ks_list && band->sck == sck);
+
+ /* Loading per band scattering coefficients */
+ res = load_data
+ (sck, stream, stream_name, "scattering coefficients", &band->ks_list);
+ if(res != RES_OK) {
+ log_err(sck,
+ "%s: data loading error for band %lu\n",
+ stream_name, (unsigned long)iband);
+ goto error;
+ }
+
+ if((res = read_padding(stream, padding_bytes)) != RES_OK) goto error;
+
+ FOR_EACH(iquad_pt, 0, darray_quad_pt_size_get(&band->quad_pts)) {
+ struct quad_pt* quad_pt = NULL;
+
+ quad_pt = darray_quad_pt_data_get(&band->quad_pts) + iquad_pt;
+
+ /* Loading absorption coefficients per band and quadrature point */
+ res = load_data
+ (sck, stream, stream_name, "correlated Ka", &quad_pt->ka_list);
+ if(res != RES_OK) {
+ log_err(sck,
+ "%s: data loading error for band %lu quadrature point %lu\n",
+ stream_name, (unsigned long)iband, (unsigned long)iquad_pt);
+ goto error;
+ }
+
+ if((res = read_padding(stream, padding_bytes)) != RES_OK) goto error;
+ }
+ }
+
+exit:
+ return res;
+error:
+ goto exit;
+}
+
+static res_T
+load_stream(struct sck* sck, const struct sck_load_stream_args* args)
+{
+ size_t iband;
+ uint64_t nbands;
+ res_T res = RES_OK;
+ ASSERT(sck && check_sck_load_stream_args(args) == RES_OK);
+
reset_sck(sck);
/* Read file header */
+ if(fread(&sck->pagesize, sizeof(sck->pagesize), 1, args->stream) != 1) {
+ if(ferror(args->stream)) {
+ log_err(sck, "%s: could not read the pagesize.\n", args->name);
+ }
+ res = RES_IO_ERR;
+ goto error;
+ }
#define READ(Var, Name) { \
- if(fread((Var), sizeof(*(Var)), 1, stream) != 1) { \
- log_err(sck, "%s: could not read the %s.\n", stream_name, (Name)); \
+ if(fread((Var), sizeof(*(Var)), 1, args->stream) != 1) { \
+ log_err(sck, "%s: could not read the %s.\n", args->name, (Name)); \
res = RES_IO_ERR; \
goto error; \
} \
} (void)0
- READ(&sck->pagesize, "page size");
READ(&nbands, "number of bands");
READ(&sck->nnodes, "number of nodes");
#undef READ
@@ -225,21 +496,20 @@ load_stream(struct sck* sck, FILE* stream, const char* stream_name)
log_err(sck,
"%s: invalid page size %lu. The page size attribute must be aligned on "
"the page size of the operating system (%lu).\n",
- stream_name,
- (unsigned long)sck->pagesize,
- (unsigned long)sck->pagesize_os);
+ args->name, sck->pagesize, (unsigned long)sck->pagesize_os);
res = RES_BAD_ARG;
goto error;
}
+
if(!nbands) {
log_err(sck, "%s: invalid number of bands %lu.\n",
- stream_name, (unsigned long)nbands);
+ args->name, (unsigned long)nbands);
res = RES_BAD_ARG;
goto error;
}
if(!sck->nnodes) {
log_err(sck, "%s: invalid number of nodes %lu.\n",
- stream_name, (unsigned long)sck->nnodes);
+ args->name, (unsigned long)sck->nnodes);
res = RES_BAD_ARG;
goto error;
}
@@ -248,20 +518,20 @@ load_stream(struct sck* sck, FILE* stream, const char* stream_name)
res = darray_band_resize(&sck->bands, nbands);
if(res != RES_OK) {
log_err(sck, "%s: could not allocate the list of bands (#bands=%lu).\n",
- stream_name, (unsigned long)nbands);
+ args->name, (unsigned long)nbands);
goto error;
}
/* Read the band description */
FOR_EACH(iband, 0, nbands) {
struct band* band = darray_band_data_get(&sck->bands) + iband;
- res = read_band(sck, band, stream, stream_name);
+ res = read_band(sck, band, args->stream, args->name);
if(res != RES_OK) goto error;
if(iband > 0 && band[0].low < band[-1].upp) {
log_err(sck,
"%s: bands must be sorted in ascending order and must not "
"overlap (band %lu in [%g, %g[ nm; band %lu in [%g, %g[ nm).\n",
- stream_name,
+ args->name,
(unsigned long)(iband-1), band[-1].low, band[-1].upp,
(unsigned long)(iband), band[ 0].low, band[ 0].upp);
res = RES_BAD_ARG;
@@ -269,52 +539,12 @@ load_stream(struct sck* sck, FILE* stream, const char* stream_name)
}
}
- /* Compute the length in bytes of the k to map for each band/quadrature point */
- map_len = ALIGN_SIZE(sck->nnodes * sizeof(float), sck->pagesize);
-
- /* Compute the offset toward the 1st list of radiative coefficients */
- offset = ftell(stream);
- offset = (off_t)ALIGN_SIZE((uint64_t)offset, sck->pagesize);
-
- FOR_EACH(iband, 0, nbands) {
- struct band* band = NULL;
- size_t iquad_pt;
-
- /* Map the per band scattering coefficient */
- band = darray_band_data_get(&sck->bands) + iband;
- band->map_len = map_len;
- band->ks_list = mmap(NULL, band->map_len, PROT_READ,
- MAP_PRIVATE|MAP_POPULATE, fileno(stream), offset);
- if(band->ks_list == MAP_FAILED) {
- log_err(sck,
- "%s: band %lu: could not map the scattering coefficients -- %s.\n",
- stream_name, (unsigned long)iband, strerror(errno));
- res = RES_IO_ERR;
- goto error;
- }
- offset = (off_t)((size_t)offset + map_len);
-
- FOR_EACH(iquad_pt, 0, darray_quad_pt_size_get(&band->quad_pts)) {
- struct quad_pt* quad_pt = NULL;
-
- /* Map the per band absorption coefficient */
- quad_pt = darray_quad_pt_data_get(&band->quad_pts)+iquad_pt;
- quad_pt->map_len = map_len;
- quad_pt->ka_list = mmap(NULL, quad_pt->map_len, PROT_READ,
- MAP_PRIVATE|MAP_POPULATE, fileno(stream), offset);
- if(quad_pt->ka_list == MAP_FAILED) {
- log_err(sck,
- "%s: band %lu: quadrature point %lu: coult not map the correlated K "
- "-- %s.\n",
- stream_name,
- (unsigned long)iband,
- (unsigned long)iquad_pt,
- strerror(errno));
- res = RES_IO_ERR;
- goto error;
- }
- offset = (off_t)((size_t)offset + map_len);
- }
+ if(args->memory_mapping) {
+ res = map_file(sck, args->stream, args->name);
+ if(res != RES_OK) goto error;
+ } else {
+ res = load_file(sck, args->stream, args->name);
+ if(res != RES_OK) goto error;
}
exit:
@@ -456,24 +686,27 @@ sck_ref_put(struct sck* sck)
}
res_T
-sck_load(struct sck* sck, const char* path)
+sck_load(struct sck* sck, const struct sck_load_args* args)
{
+ struct sck_load_stream_args stream_args = SCK_LOAD_STREAM_ARGS_NULL;
FILE* file = NULL;
res_T res = RES_OK;
- if(!sck || !path) {
- res = RES_BAD_ARG;
- goto error;
- }
+ if(!sck) { res = RES_BAD_ARG; goto error; }
+ res = check_sck_load_args(args);
+ if(res != RES_OK) goto error;
- file = fopen(path, "r");
+ file = fopen(args->path, "r");
if(!file) {
- log_err(sck, "%s: error opening file `%s'.\n", FUNC_NAME, path);
+ log_err(sck, "%s: error opening file `%s'.\n", FUNC_NAME, args->path);
res = RES_IO_ERR;
goto error;
}
- res = load_stream(sck, file, path);
+ stream_args.stream = file;
+ stream_args.name = args->path;
+ stream_args.memory_mapping = args->memory_mapping;
+ res = load_stream(sck, &stream_args);
if(res != RES_OK) goto error;
exit:
@@ -484,13 +717,13 @@ error:
}
res_T
-sck_load_stream
- (struct sck* sck,
- FILE* stream,
- const char* stream_name)
+sck_load_stream(struct sck* sck, const struct sck_load_stream_args* args)
{
- if(!sck || !stream) return RES_BAD_ARG;
- return load_stream(sck, stream, stream_name ? stream_name : "<stream>");
+ res_T res = RES_OK;
+ if(!sck) return RES_BAD_ARG;
+ res = check_sck_load_stream_args(args);
+ if(res != RES_OK) return res;
+ return load_stream(sck, args);
}
size_t
@@ -779,16 +1012,80 @@ band_release(struct band* band)
ASSERT(band);
darray_quad_pt_release(&band->quad_pts);
darray_double_release(&band->quad_pts_cumul);
- if(band->ks_list && band->ks_list != MAP_FAILED) {
+
+ if(!band->ks_list) return;
+
+ if(!band->map_len) {
+ MEM_RM(band->sck->allocator, band->ks_list);
+ } else if(band->ks_list != MAP_FAILED) {
munmap(band->ks_list, band->map_len);
}
}
+res_T
+band_copy(struct band* dst, const struct band* src)
+{
+ res_T res = RES_OK;
+ ASSERT(dst && src);
+
+ dst->sck = src->sck;
+ dst->low = src->low;
+ dst->upp = src->upp;
+
+ dst->map_len = src->map_len;
+ dst->ks_list = NULL;
+
+ if(src->map_len) {
+ /* The ks are mapped: copy the pointer */
+ dst->ks_list = src->ks_list;
+
+ } else if(src->ks_list != NULL) {
+ /* The ks are loaded: duplicate table contents */
+ const size_t memsz = sizeof(*dst->ks_list)*src->sck->nnodes;
+ dst->ks_list = MEM_ALLOC(dst->sck->allocator, memsz);
+ if(!dst->ks_list) return RES_MEM_ERR;
+ memcpy(dst->ks_list, src->ks_list, memsz);
+ }
+
+ res = darray_quad_pt_copy(&dst->quad_pts, &src->quad_pts);
+ if(res != RES_OK) return res;
+ res = darray_double_copy(&dst->quad_pts_cumul, &src->quad_pts_cumul);
+ if(res != RES_OK) return res;
+ return RES_OK;
+}
+
void
quad_pt_release(struct quad_pt* quad_pt)
{
ASSERT(quad_pt);
- if(quad_pt->ka_list && quad_pt->ka_list != MAP_FAILED) {
+ if(!quad_pt->ka_list) return;
+
+ if(!quad_pt->map_len) {
+ MEM_RM(quad_pt->band->sck->allocator, quad_pt->ka_list);
+ } else if(quad_pt->ka_list != MAP_FAILED) {
munmap(quad_pt->ka_list, quad_pt->map_len);
}
}
+
+res_T
+quad_pt_copy(struct quad_pt* dst, const struct quad_pt* src)
+{
+ ASSERT(dst && src);
+
+ dst->band = src->band;
+ dst->map_len = src->map_len;
+ dst->weight = src->weight;
+ dst->ka_list = NULL;
+ if(src->map_len) {
+ /* The ka are mapped: copy the pointer */
+ dst->ka_list = src->ka_list;
+
+ } else if(src->ka_list != NULL) {
+ /* The ka are loaded: duplicate table contents */
+ const size_t memsz = sizeof(*dst->ka_list)*src->band->sck->nnodes;
+ dst->ka_list = MEM_ALLOC(dst->band->sck->allocator, memsz);
+ if(!dst->ka_list) return RES_MEM_ERR;
+ memcpy(dst->ka_list, src->ka_list, memsz);
+ }
+ return RES_OK;
+}
diff --git a/src/sck.h b/src/sck.h
@@ -51,6 +51,24 @@ struct sck_create_args {
static const struct sck_create_args SCK_CREATE_ARGS_DEFAULT =
SCK_CREATE_ARGS_DEFAULT__;
+struct sck_load_args {
+ const char* path;
+ int memory_mapping; /* Use memory mapping instead of normal loading */
+};
+#define SCK_LOAD_ARGS_NULL__ {NULL, 0}
+static const struct sck_load_args SCK_LOAD_ARGS_NULL = SCK_LOAD_ARGS_NULL__;
+
+struct sck_load_stream_args {
+ FILE* stream;
+ const char* name; /* Stream name */
+ /* Use memory mapping instead of normal loading. Note that memory mapping
+ * cannot be used on some stream like stdin */
+ int memory_mapping;
+};
+#define SCK_LOAD_STREAM_ARGS_NULL__ {NULL, "stream", 0}
+static const struct sck_load_stream_args SCK_LOAD_STREAM_ARGS_NULL =
+ SCK_LOAD_STREAM_ARGS_NULL__;
+
struct sck_band {
double lower; /* Lower band wavelength in nm (inclusive) */
double upper; /* Upper band wavelength in nm (exclusive) */
@@ -98,13 +116,12 @@ sck_ref_put
SCK_API res_T
sck_load
(struct sck* sck,
- const char* path);
+ const struct sck_load_args* args);
SCK_API res_T
sck_load_stream
(struct sck* sck,
- FILE* stream,
- const char* stream_name); /* Can be NULL */
+ const struct sck_load_stream_args* args);
SCK_API size_t
sck_get_bands_count
diff --git a/src/sck_c.h b/src/sck_c.h
@@ -24,6 +24,7 @@
struct mem_allocator;
struct quad_pt {
+ struct band* band; /* Band to which the quadrature point belongs */
float* ka_list; /* Per node ka */
size_t map_len;
double weight;
@@ -34,6 +35,7 @@ quad_pt_init(struct mem_allocator* allocator, struct quad_pt* quad)
{
ASSERT(quad);
(void)allocator;
+ quad->band = NULL;
quad->ka_list = NULL;
quad->map_len = 0;
quad->weight = 0;
@@ -43,15 +45,36 @@ extern LOCAL_SYM void
quad_pt_release
(struct quad_pt* quad);
+extern LOCAL_SYM res_T
+quad_pt_copy
+ (struct quad_pt* dst,
+ const struct quad_pt* src);
+
+static INLINE res_T
+quad_pt_copy_and_release(struct quad_pt* dst, struct quad_pt* src)
+{
+ ASSERT(dst && src);
+ dst->band = src->band;
+ dst->ka_list = src->ka_list;
+ dst->map_len = src->map_len;
+ dst->weight = src->weight;
+ return RES_OK;
+}
+
/* Define the dynamic array of quadrature points */
#define DARRAY_NAME quad_pt
#define DARRAY_DATA struct quad_pt
+#define DARRAY_FUNCTOR_INIT quad_pt_init
+#define DARRAY_FUNCTOR_RELEASE quad_pt_release
+#define DARRAY_FUNCTOR_COPY quad_pt_copy
+#define DARRAY_FUNCTOR_COPY_AND_RELEASE quad_pt_copy_and_release
#include <rsys/dynamic_array.h>
struct band {
+ struct sck* sck;
double low; /* Lower bound in nm (inclusive) */
double upp; /* Upper bound in nm (exclusive) */
- size_t map_len;
+ size_t map_len; /* 0 <=> band's data are not mapped, i.e. they are loaded */
float* ks_list; /* Per node ks */
struct darray_quad_pt quad_pts;
struct darray_double quad_pts_cumul;
@@ -61,6 +84,7 @@ static INLINE void
band_init(struct mem_allocator* allocator, struct band* band)
{
ASSERT(band);
+ band->sck = NULL;
band->low = DBL_MAX;
band->upp =-DBL_MAX;
band->map_len = 0;
@@ -73,22 +97,10 @@ extern LOCAL_SYM void
band_release
(struct band* band);
-static INLINE res_T
-band_copy(struct band* dst, const struct band* src)
-{
- res_T res = RES_OK;
- ASSERT(dst && src);
-
- dst->low = src->low;
- dst->upp = src->upp;
- dst->map_len = src->map_len;
- dst->ks_list = src->ks_list;
- res = darray_quad_pt_copy(&dst->quad_pts, &src->quad_pts);
- if(res != RES_OK) return res;
- res = darray_double_copy(&dst->quad_pts_cumul, &src->quad_pts_cumul);
- if(res != RES_OK) return res;
- return RES_OK;
-}
+extern LOCAL_SYM res_T
+band_copy
+ (struct band* dst,
+ const struct band* src);
static INLINE res_T
band_copy_and_release(struct band* dst, struct band* src)
@@ -96,6 +108,7 @@ band_copy_and_release(struct band* dst, struct band* src)
res_T res = RES_OK;
ASSERT(dst && src);
+ dst->sck = src->sck;
dst->low = src->low;
dst->upp = src->upp;
dst->map_len = src->map_len;
diff --git a/src/test_sck_load.c b/src/test_sck_load.c
@@ -217,6 +217,8 @@ test_load(struct sck* sck)
hash256_T band_hash1;
hash256_T pt_hash0;
hash256_T pt_hash1;
+ struct sck_load_args args = SCK_LOAD_ARGS_NULL;
+ struct sck_load_stream_args stream_args = SCK_LOAD_STREAM_ARGS_NULL;
struct sck_band band;
FILE* fp = NULL;
@@ -229,9 +231,11 @@ test_load(struct sck* sck)
write_sck(fp, pagesize, nbands, nnodes);
rewind(fp);
- CHK(sck_load_stream(NULL, fp, filename) == RES_BAD_ARG);
- CHK(sck_load_stream(sck, NULL, filename) == RES_BAD_ARG);
- CHK(sck_load_stream(sck, fp, NULL) == RES_OK);
+ stream_args.stream = fp;
+ stream_args.name = filename;
+ CHK(sck_load_stream(NULL, &stream_args) == RES_BAD_ARG);
+ CHK(sck_load_stream(sck, NULL) == RES_BAD_ARG);
+ CHK(sck_load_stream(sck, &stream_args) == RES_OK);
CHK(sck_compute_hash(sck, NULL) == RES_BAD_ARG);
CHK(sck_compute_hash(NULL, hash0) == RES_BAD_ARG);
@@ -254,12 +258,18 @@ test_load(struct sck* sck)
check_sck_load(sck, nbands, nnodes);
rewind(fp);
- CHK(sck_load_stream(sck, fp, filename) == RES_OK);
+ stream_args.name = NULL;
+ CHK(sck_load_stream(sck, &stream_args) == RES_BAD_ARG);
+ stream_args.name = SCK_LOAD_STREAM_ARGS_NULL.name;
+ stream_args.memory_mapping = 1;
+ CHK(sck_load_stream(sck, &stream_args) == RES_OK);
- CHK(sck_load(NULL, filename) == RES_BAD_ARG);
+ args.path = "nop";
+ CHK(sck_load(NULL, &args) == RES_BAD_ARG);
CHK(sck_load(sck, NULL) == RES_BAD_ARG);
- CHK(sck_load(sck, "nop") == RES_IO_ERR);
- CHK(sck_load(sck, filename) == RES_OK);
+ CHK(sck_load(sck, &args) == RES_IO_ERR);
+ args.path = filename;
+ CHK(sck_load(sck, &args) == RES_OK);
check_sck_load(sck, nbands, nnodes);
CHK(sck_compute_hash(sck, hash1) == RES_OK);
@@ -269,7 +279,9 @@ test_load(struct sck* sck)
write_sck(fp, pagesize, nbands+1, nnodes);
rewind(fp);
- CHK(sck_load_stream(sck, fp, filename) == RES_OK);
+ stream_args.stream = fp;
+ stream_args.name = filename;
+ CHK(sck_load_stream(sck, &stream_args) == RES_OK);
CHK(sck_compute_hash(sck, hash1) == RES_OK);
CHK(!hash256_eq(hash0, hash1));
@@ -286,36 +298,42 @@ test_load(struct sck* sck)
static void
test_load_fail(struct sck* sck)
{
+ struct sck_load_stream_args stream_args = SCK_LOAD_STREAM_ARGS_NULL;
FILE* fp = NULL;
double low;
double upp;
+
/* The pagesize is less than the operating system page size*/
CHK(fp = tmpfile());
write_sck(fp, 2048, 1, 1);
rewind(fp);
- CHK(sck_load_stream(sck, fp, NULL) == RES_BAD_ARG);
+ stream_args.stream = fp;
+ CHK(sck_load_stream(sck, &stream_args) == RES_BAD_ARG);
CHK(fclose(fp) == 0);
/* The pagesize is not a power of two */
CHK(fp = tmpfile());
write_sck(fp, 4100, 1, 1);
rewind(fp);
- CHK(sck_load_stream(sck, fp, NULL) == RES_BAD_ARG);
+ stream_args.stream = fp;
+ CHK(sck_load_stream(sck, &stream_args) == RES_BAD_ARG);
CHK(fclose(fp) == 0);
/* Wrong #bands */
CHK(fp = tmpfile());
write_sck(fp, 4096, 0, 1);
rewind(fp);
- CHK(sck_load_stream(sck, fp, NULL) == RES_BAD_ARG);
+ stream_args.stream = fp;
+ CHK(sck_load_stream(sck, &stream_args) == RES_BAD_ARG);
CHK(fclose(fp) == 0);
/* Wrong #nodes */
CHK(fp = tmpfile());
write_sck(fp, 4096, 1, 0);
rewind(fp);
- CHK(sck_load_stream(sck, fp, NULL) == RES_BAD_ARG);
+ stream_args.stream = fp;
+ CHK(sck_load_stream(sck, &stream_args) == RES_BAD_ARG);
CHK(fclose(fp) == 0);
/* Wrong band boundaries */
@@ -327,7 +345,8 @@ test_load_fail(struct sck* sck)
CHK(fwrite(&low, sizeof(low), 1, fp) == 1);
CHK(fwrite(&upp, sizeof(upp), 1, fp) == 1);
rewind(fp);
- CHK(sck_load_stream(sck, fp, NULL) == RES_BAD_ARG);
+ stream_args.stream = fp;
+ CHK(sck_load_stream(sck, &stream_args) == RES_BAD_ARG);
CHK(fclose(fp) == 0);
/* Unsorted bands */
@@ -342,7 +361,8 @@ test_load_fail(struct sck* sck)
CHK(fwrite(&low, sizeof(low), 1, fp) == 1);
CHK(fwrite(&upp, sizeof(upp), 1, fp) == 1);
rewind(fp);
- CHK(sck_load_stream(sck, fp, NULL) == RES_BAD_ARG);
+ stream_args.stream = fp;
+ CHK(sck_load_stream(sck, &stream_args) == RES_BAD_ARG);
CHK(fclose(fp) == 0);
/* Bands overlap */
@@ -357,7 +377,8 @@ test_load_fail(struct sck* sck)
CHK(fwrite(&low, sizeof(low), 1, fp) == 1);
CHK(fwrite(&upp, sizeof(upp), 1, fp) == 1);
rewind(fp);
- CHK(sck_load_stream(sck, fp, NULL) == RES_BAD_ARG);
+ stream_args.stream = fp;
+ CHK(sck_load_stream(sck, &stream_args) == RES_BAD_ARG);
CHK(fclose(fp) == 0);
}
@@ -368,13 +389,16 @@ test_load_files(struct sck* sck, int argc, char** argv)
CHK(sck);
FOR_EACH(i, 1, argc) {
hash256_T hash;
+ struct sck_load_args args = SCK_LOAD_ARGS_NULL;
size_t nnodes;
size_t nbands;
size_t iband;
printf("Load %s\n", argv[1]);
- CHK(sck_load(sck, argv[i]) == RES_OK);
+ args.path = argv[i];
+ args.memory_mapping = 1;
+ CHK(sck_load(sck, &args) == RES_OK);
nbands = sck_get_bands_count(sck);
nnodes = sck_get_nodes_count(sck);
CHK(nbands);
@@ -420,6 +444,7 @@ test_load_files(struct sck* sck, int argc, char** argv)
static void
test_find(struct sck* sck)
{
+ struct sck_load_stream_args stream_args = SCK_LOAD_STREAM_ARGS_NULL;
size_t ibands[2];
double range[2];
FILE* fp;
@@ -427,7 +452,9 @@ test_find(struct sck* sck)
CHK(fp = tmpfile());
write_sck(fp, 4096, 10, 1);
rewind(fp);
- CHK(sck_load_stream(sck, fp, NULL) == RES_OK);
+ stream_args.stream = fp;
+ stream_args.memory_mapping = 1;
+ CHK(sck_load_stream(sck, &stream_args) == RES_OK);
range[0] = 0;
range[1] = 10;
