commit c9af5295aace4ee097f90c75ce59bc60a1d0045b
parent 550f6520e39bcef4b7f2cd11e33e17861362f892
Author: Vincent Forest <vincent.forest@meso-star.com>
Date: Mon, 30 Apr 2018 10:36:33 +0200
Update the htcop fileformat
Add the "page size" header attribute and make the size and address of the
RVT and RCT data aligned onto this attribute.
Update the les2htcop tool and htcop library to support this update.
Diffstat:
4 files changed, 79 insertions(+), 13 deletions(-)
diff --git a/doc/htcop.txt b/doc/htcop.txt
@@ -2,10 +2,13 @@
<definition>
<lower-pos>
<voxel-size>
- <RVT>
- <RCT>
+ <padding>
+ <RVT> # align on pagesize
+ <padding>
+ <RCT> # align on pagesize
+ <padding>
-<header> ::= <is-Z-irregular>
+<header> ::= <pagesize> <is-Z-irregular>
<definition> ::= <#X> <#Y> <#Z> <#time>
<lower-pos> ::= <double3>
<voxel-size> ::= <double3> [DOUBLE ... ] # optionnal double for irregular Z
@@ -20,3 +23,7 @@
<#time> ::= INT32
<is-Z-irregular> ::= INT8
+
+<pagesize> ::= INT64
+
+<padding> ::= [ BYTES ... ] # Alignment address onto <pagesize>
diff --git a/src/htcop.c b/src/htcop.c
@@ -26,10 +26,12 @@
#include <errno.h>
#include <string.h>
#include <sys/mman.h>
+#include <unistd.h>
enum { X, Y, Z, TIME }; /* Helper constants */
struct htcop {
+ int64_t pagesize;
int8_t irregular_z;
int32_t definition[4];
double lower[3];
@@ -39,6 +41,8 @@ struct htcop {
double* RCT; /* Mapped memory */
double* RVT; /* Mapped memory */
+ size_t pagesize_os; /* Page size of the os */
+
int verbose; /* Verbosity level */
struct logger* logger;
struct mem_allocator* allocator;
@@ -89,6 +93,23 @@ load_stream(struct htcop* htcop, FILE* stream, const char* stream_name)
goto error; \
} \
} (void)0
+ READ(&htcop->pagesize, 1, "page size");
+ if(!IS_POW2(htcop->pagesize)) {
+ log_err(htcop, "%s: invalid page size `%li'. It must be a power of two.\n",
+ stream_name, (long)htcop->pagesize);
+ res = RES_BAD_ARG;
+ goto error;
+ }
+
+ if(!IS_ALIGNED(htcop->pagesize, htcop->pagesize_os)) {
+ log_err(htcop,
+"%s: invalid page size `%li'. The page size attribute must be aligned on the "
+"page size of the operating system , i.e. %lu.\n",
+ stream_name, (long)htcop->pagesize, (unsigned long)htcop->pagesize_os);
+ res = RES_BAD_ARG;
+ goto error;
+ }
+
READ(&htcop->irregular_z, 1, "'irregular Z' flag");
READ(htcop->definition, 4, "spatial and time definitions");
READ(htcop->lower, 3, "lower position");
@@ -112,6 +133,8 @@ load_stream(struct htcop* htcop, FILE* stream, const char* stream_name)
* (size_t)htcop->definition[Z]
* (size_t)htcop->definition[TIME]
* sizeof(double);
+ /* Align sizeof mapped data onto page size */
+ map_len = ALIGN_SIZE(map_len, (size_t)htcop->pagesize);
if(map_len == 0) {
fprintf(stderr, "%s: the spatial/time definition cannot be null.\n",
@@ -120,7 +143,8 @@ load_stream(struct htcop* htcop, FILE* stream, const char* stream_name)
goto error;
}
- offset = ftell(stream);
+ /* Ensure that offset is align on page size */
+ offset = ALIGN_SIZE(ftell(stream), htcop->pagesize);
#define MMAP(Var) { \
htcop->Var = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE|MAP_POPULATE, \
@@ -190,6 +214,7 @@ htcop_create
htcop->allocator = allocator;
htcop->logger = logger;
htcop->verbose = verbose;
+ htcop->pagesize_os = (size_t)sysconf(_SC_PAGESIZE);
darray_double_init(htcop->allocator, &htcop->vxsz_z);
exit:
diff --git a/src/les2htcop.c b/src/les2htcop.c
@@ -17,9 +17,10 @@
#include "les2htcop.h"
-#include <rsys/rsys.h>
+#include <rsys/cstr.h>
#include <rsys/math.h>
#include <rsys/mem_allocator.h>
+#include <rsys/rsys.h>
#include <alloca.h>
#include <netcdf.h>
@@ -34,12 +35,13 @@
struct args {
const char* output;
const char* input;
+ long pagesize;
int force_overwrite;
int check;
int no_output;
int quit; /* Quit the application */
};
-#define ARGS_DEFAULT__ {NULL,NULL,0,0,0,0}
+#define ARGS_DEFAULT__ {NULL,NULL,4096,0,0,0,0}
static const struct args ARGS_DEFAULT = ARGS_DEFAULT__;
static void
@@ -66,6 +68,12 @@ print_help(const char* cmd)
" -o OUTPUT write results to OUTPUT. If not defined, write results to\n"
" standard output.\n");
printf(
+" -p targeted page size in bytes. Must be a power of 2. The size\n"
+" of the converted LES data and their address into output are\n"
+" aligned according to this size. By default, the page size\n"
+" is %li Bytes.\n",
+ ARGS_DEFAULT.pagesize);
+ printf(
" -q do not write results to OUTPUT.\n");
printf(
" -v display version information and exit.\n");
@@ -90,7 +98,7 @@ args_init(struct args* args, const int argc, char** argv)
res_T res = RES_OK;
ASSERT(args && argc && argv);
- while((opt = getopt(argc, argv, "cfhi:o:qv")) != -1) {
+ while((opt = getopt(argc, argv, "cfhi:o:p:qv")) != -1) {
switch(opt) {
case 'c': args->check = 1; break;
case 'f': args->force_overwrite = 1; break;
@@ -101,6 +109,10 @@ args_init(struct args* args, const int argc, char** argv)
goto exit;
case 'i': args->input = optarg; break;
case 'o': args->output = optarg; break;
+ case 'p':
+ res = cstr_to_long(optarg, &args->pagesize);
+ if(res == RES_OK && !IS_POW2(args->pagesize)) res = RES_BAD_ARG;
+ break;
case 'q': args->no_output = 1; break;
case 'v':
printf("%s %d.%d.%d\n",
@@ -694,7 +706,10 @@ main(int argc, char** argv)
} \
} (void)0
if(!args.no_output) {
- WRITE(&grid.is_z_irregular, 1, "'irregular' Z flag");
+ const int64_t pagesize = (int64_t)args.pagesize;
+
+ WRITE(&pagesize, 1, "page size");
+ WRITE(&grid.is_z_irregular, 1, "'irregular' Z boolean");
WRITE(&grid.nx, 1, "X definition");
WRITE(&grid.ny, 1, "Y definition");
WRITE(&grid.ny, 1, "Z definition");
@@ -704,8 +719,13 @@ main(int argc, char** argv)
WRITE(&grid.vxsz_y, 1, "Y voxel size");
WRITE(grid.vxsz_z, grid.is_z_irregular?(size_t)grid.nz:1, "Z voxel size(s)");
+ fseek(stream, ALIGN_SIZE(ftell(stream), (off_t)pagesize), SEEK_SET);
write_data(nc, "RVT", stream);
+
+ fseek(stream, ALIGN_SIZE(ftell(stream), (off_t)pagesize), SEEK_SET);
write_data(nc, "RCT", stream);
+
+ fseek(stream, ALIGN_SIZE(ftell(stream), (off_t)pagesize), SEEK_SET);
}
#undef WRITE
diff --git a/src/test_htcop_load.c b/src/test_htcop_load.c
@@ -16,9 +16,12 @@
#include "htcop.h"
#include "test_htcop_utils.h"
+#include <unistd.h>
+
int
main(int argc, char** argv)
{
+ int64_t pagesize;
struct htcop* htcop = NULL;
FILE* stream = NULL;
int8_t i8;
@@ -29,37 +32,48 @@ main(int argc, char** argv)
stream = tmpfile();
- i8 = 0;
+ pagesize = (int64_t)sysconf(_SC_PAGESIZE); /* pagesize */
+ CHK(fwrite(&pagesize, sizeof(pagesize), 1, stream) == 1);
+ i8 = 0; /* irregular Z */
CHK(fwrite(&i8, sizeof(i8), 1, stream) == 1);
- i32[0] = 2;
+ i32[0] = 2; /* Definition */
i32[1] = 2;
i32[2] = 3;
i32[3] = 1;
CHK(fwrite(i32, sizeof(i32[0]), 4, stream) == 4);
- dbl[0] = 0;
+ dbl[0] = 0; /* Lower position */
dbl[1] = 0;
dbl[2] = 0;
CHK(fwrite(dbl, sizeof(dbl[0]), 3, stream) == 3);
- dbl[0] = 1;
+ dbl[0] = 1; /* Voxel size */
dbl[1] = 2;
dbl[2] = 3;
CHK(fwrite(dbl, sizeof(dbl[0]), 3, stream) == 3);
+
+ fseek(stream, ALIGN_SIZE(ftell(stream), pagesize), SEEK_SET); /* padding */
FOR_EACH(i, 0, i32[0]*i32[1]*i32[2]*i32[3]) { /* RVT */
dbl[0] = (double)i;
CHK(fwrite(dbl, sizeof(dbl[0]), 1, stream) == 1);
}
+ fseek(stream, ALIGN_SIZE(ftell(stream), pagesize), SEEK_SET); /* padding */
FOR_EACH(i, 0, i32[0]*i32[1]*i32[2]*i32[3]) { /* RCT */
dbl[0] = (double)i;
CHK(fwrite(dbl, sizeof(dbl[0]), 1, stream) == 1);
- }
+ }
+ fseek(stream, ALIGN_SIZE(ftell(stream), pagesize), SEEK_SET); /* padding */
rewind(stream);
CHK(htcop_create(NULL, &mem_default_allocator, 1, &htcop) == RES_OK);
+
+ CHK(htcop_load_stream(NULL, stream) == RES_BAD_ARG);
+ CHK(htcop_load_stream(htcop, NULL) == RES_BAD_ARG);
CHK(htcop_load_stream(htcop, stream) == RES_OK);
+
CHK(htcop_ref_put(htcop) == RES_OK);
CHK(mem_allocated_size() == 0);
return 0;
}
+
