commit 1137cd50d8ff5bc38cba497529182ec3fe80059f parent c15e2a841e9dc247465dfb75e8a640edce67910e Author: Vincent Forest <vincent.forest@meso-star.com> Date: Wed, 30 Apr 2025 11:32:41 +0200 Merge branch 'release_0.15' Diffstat:
41 files changed, 339 insertions(+), 250 deletions(-)
diff --git a/Makefile b/Makefile @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +# Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,6 +22,9 @@ LIBNAME_STATIC = libstar-sp.a LIBNAME_SHARED = libstar-sp.so LIBNAME = $(LIBNAME_$(LIB_TYPE)) +default: library +all: library tests + ################################################################################ # Star-SP building ################################################################################ @@ -36,7 +39,7 @@ SRC =\ OBJ = $(SRC:.c=.o) DEP = $(SRC:.c=.d) -build_library: .config $(DEP) +library: .config $(DEP) @$(MAKE) -fMakefile $$(for i in $(DEP); do echo -f $${i}; done) \ $$(if [ -n "$(LIBNAME)" ]; then \ echo "$(LIBNAME)"; \ @@ -47,7 +50,7 @@ build_library: .config $(DEP) $(DEP) $(OBJ): config.mk $(LIBNAME_SHARED): $(OBJ) - $(CXX) $(CXXFLAGS) $(AES_CFLAGS) $(DPDC_CFLAGS) -o $@ $(OBJ) $(LDFLAGS_SO) $(DPDC_LIBS) + $(CXX) $(CXXFLAGS) $(INCS) -o $@ $(OBJ) $(LDFLAGS_SO) $(LIBS) $(LIBNAME_STATIC): libstar-sp.o $(AR) -rc $@ $? @@ -58,21 +61,19 @@ libstar-sp.o: $(OBJ) $(OBJCOPY) $(OCPFLAGS) $@ .config: config.mk - @if ! $(PKG_CONFIG) --atleast-version $(RSYS_VERSION) rsys; then \ - echo "rsys $(RSYS_VERSION) not found" >&2; exit 1; fi - @if ! $(PKG_CONFIG) --atleast-version $(RANDOM123_VERSION) random123; then \ - echo "random123 $(RANDOM123_VERSION) not found" >&2; exit 1; fi + $(PKG_CONFIG) --atleast-version $(RANDOM123_VERSION) random123 + $(PKG_CONFIG) --atleast-version $(RSYS_VERSION) rsys @echo "config done" > .config .SUFFIXES: .c .d .o .c.d: - @$(CXX) $(CXXFLAGS) $(DPDC_CFLAGS) $(AES_CFLAGS) -MM -MT "$(@:.d=.o) $@" $< -MF $@ + @$(CXX) $(CXXFLAGS) $(INCS) -MM -MT "$(@:.d=.o) $@" $< -MF $@ .c.o: - $(CXX) $(CXXFLAGS) $(DPDC_CFLAGS) $(AES_CFLAGS) -DSSP_SHARED_BUILD -c $< -o $@ + $(CXX) $(CXXFLAGS) $(INCS) -DSSP_SHARED_BUILD -c $< -o $@ ################################################################################ -# Installation +# Miscellaneous ################################################################################ pkg: sed -e 's#@PREFIX@#$(PREFIX)#g' \ @@ -91,33 +92,27 @@ star-sp-local.pc: config.mk star-sp.pc.in -e 's#@RANDOM123_VERSION@#$(RANDOM123_VERSION)#g'\ star-sp.pc.in > $@ -install: build_library pkg - @$(SHELL) make.sh install "$(DESTDIR)$(PREFIX)/lib" $(LIBNAME) - @$(SHELL) make.sh install "$(DESTDIR)$(PREFIX)/lib/pkgconfig" star-sp.pc - @$(SHELL) make.sh install "$(DESTDIR)$(PREFIX)/include/star" src/ssp.h - @$(SHELL) make.sh install "$(DESTDIR)$(PREFIX)/share/doc/star-sp" COPYING README.md +install: library pkg + install() { mode="$$1"; prefix="$$2"; shift 2; \ + mkdir -p "$${prefix}"; \ + cp "$$@" "$${prefix}"; \ + chmod "$${mode}" "$$@"; \ + }; \ + install 755 "$(DESTDIR)$(LIBPREFIX)" $(LIBNAME); \ + install 644 "$(DESTDIR)$(LIBPREFIX)/pkgconfig" star-sp.pc; \ + install 644 "$(DESTDIR)$(INCPREFIX)/star" src/ssp.h; \ + install 644 "$(DESTDIR)$(PREFIX)/share/doc/star-sp" COPYING README.md uninstall: - rm -f $(DESTDIR)$(PREFIX)/lib/libstar-sp.so - rm -f $(DESTDIR)$(PREFIX)/lib/pkgconfig/star-sp.pc - rm -f $(DESTDIR)$(PREFIX)/share/doc/star-sp/COPYING - rm -f $(DESTDIR)$(PREFIX)/share/doc/star-sp/README.md - rm -f $(DESTDIR)$(PREFIX)/include/star/ssp.h - -################################################################################ -# Miscellaneous targets -################################################################################ -all: build_library build_tests + rm -f "$(DESTDIR)$(LIBPREFIX)/$(LIBNAME)" + rm -f "$(DESTDIR)$(INCPREFIX)/star/ssp.h" + rm -f "$(DESTDIR)$(LIBPREFIX)/pkgconfig/star-sp.pc" + rm -f "$(DESTDIR)$(PREFIX)/share/doc/star-sp/COPYING" + rm -f "$(DESTDIR)$(PREFIX)/share/doc/star-sp/README.md" clean: clean_test - rm -f $(OBJ) $(TEST_OBJ) $(LIBNAME) - rm -f .config .test libstar-sp.o star-sp.pc star-sp-local.pc - -distclean: clean - rm -f $(DEP) $(TEST_DEP) - -lint: - shellcheck -o all make.sh + rm -f $(DEP) $(OBJ) $(LIBNAME) + rm -f .config libstar-sp.o star-sp.pc star-sp-local.pc ################################################################################ # Tests @@ -139,28 +134,33 @@ TEST_SRC =\ src/test_ssp_rng_proxy.c TEST_OBJ = $(TEST_SRC:.c=.o) TEST_DEP = $(TEST_SRC:.c=.d) +TEST_TGT = $(TEST_SRC:.c=.t) PKG_CONFIG_LOCAL = PKG_CONFIG_PATH="./:$${PKG_CONFIG_PATH}" $(PKG_CONFIG) -SSP_CFLAGS = $$($(PKG_CONFIG_LOCAL) $(PCFLAGS) --cflags star-sp-local.pc) -SSP_LIBS = $$($(PKG_CONFIG_LOCAL) $(PCFLAGS) --libs star-sp-local.pc) - -build_tests: build_library $(TEST_DEP) .test - @$(MAKE) -fMakefile -f.test $$(for i in $(TEST_DEP); do echo -f"$${i}"; done) test_bin +INCS_TEST = $$($(PKG_CONFIG_LOCAL) $(PCFLAGS) --cflags rsys star-sp-local.pc) +LIBS_TEST = $$($(PKG_CONFIG_LOCAL) $(PCFLAGS) --libs rsys star-sp-local.pc) -test: build_tests - @$(SHELL) make.sh run_test "$(AES_CFLAGS)" $(TEST_SRC) +CFLAGS_TEST = $(CFLAGS) $(INCS_TEST) +LDFLAGS_TEST = $(LDFLAGS_EXE) $(LIBS_TEST) -lm -.test: Makefile - @$(SHELL) make.sh config_test $(TEST_SRC) > .test +tests: library $(TEST_DEP) $(TEST_TGT) + @$(MAKE) -fMakefile \ + $$(for i in $(TEST_DEP); do echo -f"$${i}"; done) \ + $$(for i in $(TEST_TGT); do echo -f"$${i}"; done) \ + test_list -clean_test: - @$(SHELL) make.sh clean_test $(TEST_SRC) +$(TEST_TGT): + @{ \ + exe="$$(basename "$@" ".t")"; \ + printf '%s: %s\n' "$${exe}" $(@:.t=.o); \ + printf 'test_list: %s\n' "$${exe}"; \ + } > $@ $(TEST_DEP): config.mk star-sp-local.pc - @$(CC) $(CFLAGS) $(SSP_CFLAGS) $(RSYS_CFLAGS) -MM -MT "$(@:.d=.o) $@" $(@:.d=.c) -MF $@ + @$(CC) $(CFLAGS_TEST) -MM -MT "$(@:.d=.o) $@" $(@:.d=.c) -MF $@ $(TEST_OBJ): config.mk star-sp-local.pc - $(CC) $(CFLAGS) $(SSP_CFLAGS) $(RSYS_CFLAGS) -c $(@:.o=.c) -o $@ + $(CC) $(CFLAGS_TEST) -c $(@:.o=.c) -o $@ test_ssp_ran_circle \ test_ssp_ran_discrete \ @@ -177,4 +177,36 @@ test_ssp_ran_uniform_disk \ test_ssp_rng \ test_ssp_rng_proxy \ : config.mk star-sp-local.pc $(LIBNAME) - $(CC) $(CFLAGS) -o $@ src/$@.o $(LDFLAGS_EXE) $(SSP_LIBS) $(RSYS_LIBS) -lm + $(CC) $(CFLAGS_TEST) -o $@ src/$@.o $(LDFLAGS_TEST) + +clean_test: + rm -f $(TEST_DEP) $(TEST_OBJ) $(TEST_TGT) + for i in $(TEST_SRC); do rm -f "$$(basename "$${i}" ".c")"; done + +test: tests + @err=0; \ + check() { name="$$1"; exe="$$2"; shift 2; \ + printf '%s %s' "$${name}" "$$@"; \ + if "./$${exe}" "$$@"> /dev/null 2>&1; then \ + printf '\n'; \ + else \ + printf ': error %s\n' "$$?"; \ + err=$$((err+1)); \ + fi \ + }; \ + for i in $(TEST_SRC); do \ + test="$$(basename "$${i}" ".c")"; \ + if [ "$${test}" != "test_ssp_rng" ]; then \ + check "$${test}" "$${test}"; \ + else \ + if [ -n "$(AES_CFLAGS)" ]; then \ + check test_ssp_rng_aes test_ssp_rng aes; \ + fi; \ + check test_ssp_rng_kiss test_ssp_rng kiss; \ + check test_ssp_rng_mt19937_64 test_ssp_rng mt19937_64; \ + check test_ssp_rng_ranlux48 test_ssp_rng ranlux48; \ + check test_ssp_rng_random_device test_ssp_rng random_device; \ + check test_ssp_rng_threefry test_ssp_rng threefry; \ + fi \ + done; \ + [ "$${err}" -eq 0 ] diff --git a/README.md b/README.md @@ -23,6 +23,18 @@ Edit config.mk as needed, then run: ## Release notes +### Version 0.15 + +- Keep RNG caches managed by proxy in memory to avoid write failures + when the temporary directory ran out of space. +- Reduce the size of the RNG cache managed by proxy to 4MB to limit the + memory footprint of the proxy. +- Write state cache in case of fatal error to help debug what went + wrong. +- Improve the building system. Simplify it by doing everything in one + place (Makefile). Provide additional macros to control installation + subdirectories. + ### Version 0.14 Replace CMake by Makefile as build system. The build procedure is @@ -179,7 +191,7 @@ since CMake 3.20, version 2 has become obsolete. ## License -Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) Star-SP is free software released under GPL v3+ license: GNU GPL version 3 or later. You are welcome to redistribute it under certain conditions; diff --git a/config.mk b/config.mk @@ -1,4 +1,4 @@ -VERSION = 0.14.0 +VERSION = 0.15.0 PREFIX = /usr/local LIB_TYPE = SHARED @@ -7,6 +7,11 @@ LIB_TYPE = SHARED BUILD_TYPE = RELEASE #BUILD_TYPE = DEBUG +BINPREFIX = $(PREFIX)/bin +LIBPREFIX = $(PREFIX)/lib +INCPREFIX = $(PREFIX)/include +MANPREFIX = $(PREFIX)/share/man + ################################################################################ # Tools ################################################################################ @@ -26,15 +31,9 @@ PCFLAGS_STATIC = --static PCFLAGS = $(PCFLAGS_$(LIB_TYPE)) RSYS_VERSION = 0.14 -RSYS_CFLAGS = $$($(PKG_CONFIG) $(PCFLAGS) --cflags rsys) -RSYS_LIBS = $$($(PKG_CONFIG) $(PCFLAGS) --libs rsys) - RANDOM123_VERSION = 1.14 -RANDOM123_CFLAGS = $$($(PKG_CONFIG) $(PCFLAGS) --cflags random123) -RANDOM123_LIBS = $$($(PKG_CONFIG) $(PCFLAGS) --libs random123) - -DPDC_CFLAGS = $(RSYS_CFLAGS) $(RANDOM123_CFLAGS) -DPDC_LIBS = $(RSYS_LIBS) $(RANDOM123_LIBS) +INCS = $$($(PKG_CONFIG) $(PCFLAGS) --cflags random123 rsys) +LIBS = $$($(PKG_CONFIG) $(PCFLAGS) --libs random123 rsys) ################################################################################ # Compilation options @@ -75,7 +74,7 @@ CFLAGS = $(CFLAGS_$(BUILD_TYPE)) -fPIE CXXFLAGS_RELEASE = -O2 -DNDEBUG $(CXXFLAGS_COMMON) CXXFLAGS_DEBUG = -g $(CXXFLAGS_COMMON) -CXXFLAGS = $(CXXFLAGS_$(BUILD_TYPE)) -fPIC +CXXFLAGS = $(CXXFLAGS_$(BUILD_TYPE)) $(AES_CFLAGS) -fPIC ################################################################################ # Linker options diff --git a/make.sh b/make.sh @@ -1,92 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -config_test() -{ - for i in "$@"; do - test=$(basename "${i}" ".c") - test_list="${test_list} ${test}" - printf "%s: %s\n" "${test}" "src/${test}.o" - done - printf "test_bin: %s\n" "${test_list}" -} - -check() -{ - name="$1" - prog="$2" - shift 2 - - printf "%s " "${name}" - if ./"${prog}" "$@" > /dev/null 2>&1; then - printf "\033[1;32mOK\033[m\n" - else - printf "\033[1;31mError\033[m\n" - fi -} - -run_test() -{ - aes_flags=$1 - shift 1 - - for i in "$@"; do - test=$(basename "${i}" ".c") - if ! [ "${test}" = "test_ssp_rng" ]; then - check "${test}" "${test}" - else - if [ -n "${aes_flags}" ]; then - check "${test}_aes" "${test}" aes - fi - check "${test}_kiss" "${test}" kiss - check "${test}_mt19937_64" "${test}" mt19937_64 - check "${test}_ranlux48" "${test}" ranlux48 - check "${test}_random_device" "${test}" random_device - check "${test}_threefry" "${test}" threefry - fi - done 2> /dev/null -} - -clean_test() -{ - for i in "$@"; do - rm -f "$(basename "${i}" ".c")" - done -} - -install() -{ - prefix=$1 - shift 1 - - mkdir -p "${prefix}" - - for i in "$@"; do - dst="${prefix}/${i##*/}" - - if cmp -s "${i}" "${dst}"; then - printf "Up to date %s\n" "${dst}" - else - printf "Installing %s\n" "${dst}" - cp "${i}" "${prefix}" - fi - done -} - -"$@" diff --git a/src/ssp.h b/src/ssp.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/ssp_ran.c b/src/ssp_ran.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/ssp_ranst_discrete.c b/src/ssp_ranst_discrete.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/ssp_ranst_gaussian.c b/src/ssp_ranst_gaussian.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/ssp_ranst_piecewise_linear.c b/src/ssp_ranst_piecewise_linear.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/ssp_rng.c b/src/ssp_rng.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/ssp_rng_c.h b/src/ssp_rng_c.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/ssp_rng_proxy.c b/src/ssp_rng_proxy.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,6 +13,8 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#define _POSIX_C_SOURCE 200809L /* fmemopen, getpid */ + #include "ssp_rng_c.h" #include <rsys/dynamic_array_char.h> @@ -22,27 +24,36 @@ #include <rsys/stretchy_array.h> #include <limits.h> +#include <unistd.h> /* getpid */ #define BUCKET_SIZE_DEFAULT 1000000 /* #RNs per bucket */ -/* Cache size to use. This is just a hint: the effective size of the cache is - * actually a multiple of the size of the RNG state it should store. */ +/* Cache capacity. Maximum size in bytes that the cache can store */ #if 1 - #define STATE_CACHE_HINT_MAX_SIZE (32*(1024*1024)) /* 32 MB */ + #define STATE_CACHE_CAPACITY (4*1024*1024) /* 4 MB */ #else - #define STATE_CACHE_HINT_MAX_SIZE 0 /* Disable the cache */ + #define STATE_CACHE_CAPACITY 0 /* Disable the cache */ #endif /* Cache of RNG states */ struct rng_state_cache { struct darray_char state; /* Save the next RNG state with 'no_wstream' */ struct darray_char state_scratch; /* Scracth state buffer */ - FILE* stream; /* Stream into which the RNG states are stored */ + + struct mem_allocator* allocator; + void* buffer; /* Memory in which RNG states are stored */ + FILE* stream; /* Stream to buffer storing RNG states */ + size_t state_pitch; /* #RNs between 2 cached states */ size_t nstates; /* #cached states */ + long read, write; /* Offset into the stream where to read/write RNG states */ + long end_of_cache; /* Stream offset that marks the end of cache */ + int no_wstream; /* Define if the RNG states are no more written to a stream */ int no_rstream; /* Define if the RNG states are no more read from a stream */ + + int is_init; /* Cache initialisation state */ }; CLBK(rng_proxy_cb_T, ARG1(const struct ssp_rng_proxy*)); @@ -161,41 +172,61 @@ error: /******************************************************************************* * Cache of RNG states ******************************************************************************/ +static void +rng_state_cache_release(struct rng_state_cache* cache) +{ + ASSERT(cache); + if(!cache->is_init) return; /* Nothing to release */ + + if(cache->stream) fclose(cache->stream); + if(cache->buffer) MEM_RM(cache->allocator, cache->buffer); + darray_char_release(&cache->state); + darray_char_release(&cache->state_scratch); +} + static res_T rng_state_cache_init (struct mem_allocator* allocator, const size_t state_pitch, /* #RNs between cached states */ struct rng_state_cache* cache) { + res_T res = RES_OK; ASSERT(cache); + memset(cache, 0, sizeof(*cache)); darray_char_init(allocator, &cache->state); darray_char_init(allocator, &cache->state_scratch); - cache->stream = tmpfile(); - if(!cache->stream) return RES_IO_ERR; - cache->read = cache->write = ftell(cache->stream); + cache->allocator = allocator; + cache->end_of_cache = STATE_CACHE_CAPACITY; cache->state_pitch = state_pitch; - return RES_OK; -} -static void -rng_state_cache_release(struct rng_state_cache* cache) -{ - ASSERT(cache); - if(cache->stream) fclose(cache->stream); - darray_char_release(&cache->state); - darray_char_release(&cache->state_scratch); + if(STATE_CACHE_CAPACITY != 0) { + cache->buffer = MEM_CALLOC(allocator, 1, STATE_CACHE_CAPACITY); + if(!cache->buffer) { res = RES_MEM_ERR; goto error; } + cache->stream = fmemopen(cache->buffer, STATE_CACHE_CAPACITY, "w+"); + if(!cache->stream) { res = RES_IO_ERR; goto error; } + cache->read = cache->write = ftell(cache->stream); + } + + cache->is_init = 1; + +exit: + return res; +error: + rng_state_cache_release(cache); + goto exit; } static res_T rng_state_cache_clear(struct rng_state_cache* cache) { - if(!cache->stream) { - cache->stream = tmpfile(); - if(!cache->stream) return RES_IO_ERR; - } else { + ASSERT(cache); + if(cache->stream) { rewind(cache->stream); cache->read = cache->write = ftell(cache->stream); + } else { + ASSERT(STATE_CACHE_CAPACITY == 0); + cache->read = cache->write = 0; } cache->nstates = 0; cache->no_wstream = 0; @@ -211,6 +242,72 @@ rng_state_cache_is_empty(struct rng_state_cache* cache) } static res_T +rng_state_cache_dump(struct rng_state_cache* cache) +{ + /* Output file into which cache stream is dumped */ + char name[128] = {0}; + pid_t process = 0; /* Process identifier */ + FILE* fp = NULL; + + /* Temporary memory to copy cache stream */ + void* mem = NULL; + + /* Miscellaneous */ + long offset = 0; + int n = 0; + res_T res = RES_OK; + + ASSERT(cache); + + /* No cache to dump */ + if(!cache->stream) goto exit; + + process = getpid(); + + #define TRY(Cond, Err) { \ + if(!(Cond)) { \ + fprintf(stderr, "%s:%d: %s\n", FUNC_NAME, __LINE__, strerror(Err)); \ + switch(Err) { \ + case EIO: res = RES_IO_ERR; break; \ + case ENOMEM: res = RES_MEM_ERR; break; \ + default: res = RES_UNKNOWN_ERR; break; \ + } \ + goto error; \ + } \ + } (void)0 + + /* Requests the offset at the end of the file, + * i.e. the size of the cache stream */ + TRY(fseek(cache->stream, 0, SEEK_END) == 0, errno); + TRY((offset = ftell(cache->stream)) >= 0, errno); + + /* Define the state cache filename */ + n = snprintf(name, sizeof(name), "rng_cache_r%lu_w%lu_s%ld_%d", + cache->read, cache->write, offset, process); + TRY(n >= 0, errno); + TRY((unsigned long)n < sizeof(name), ENOMEM); + + /* Create the output file */ + TRY((fp = fopen(name, "w")) != NULL, errno); + + rewind(cache->stream); + + /* Load cache stream in mem, and then dump it to the state cache */ + TRY((mem = mem_alloc(offset)) != NULL, ENOMEM); + TRY((fread(mem, offset, 1, cache->stream)) == 1, EIO); + TRY((fwrite(mem, offset, 1, fp)) == 1, EIO); + + #undef TRY + +exit: + if(mem) mem_rm(mem); + if(fp) CHK(fclose(fp) == 0); + return res; +error: + goto exit; +} + +static res_T rng_state_cache_read (struct rng_state_cache* cache, struct ssp_rng* rng, @@ -222,19 +319,25 @@ rng_state_cache_read mutex_lock(mutex); ASSERT(!rng_state_cache_is_empty(cache)); + /* The read file pointer has reached the end of the cache. + * Rewind it to the beginning of the cache */ + if(cache->read == cache->end_of_cache) { + cache->read = 0; + cache->end_of_cache = STATE_CACHE_CAPACITY; + } + if(!cache->no_rstream && cache->no_wstream && cache->read == cache->write && cache->nstates == 1/* A state is saved in 'cache->state' */) { - /* There is no more data cached into the stream. Close the stream and do - * not rely anymore on the proxy RNG to generate the RNG states */ - fclose(cache->stream); - cache->stream = NULL; + /* There is no more data cached into the stream. Do not rely anymore on the + * proxy RNG to generate the RNG states */ cache->no_rstream = 1; } /* Read the cached RNG state from the stream */ if(!cache->no_rstream) { + fseek(cache->stream, cache->read, SEEK_SET); res = ssp_rng_read(rng, cache->stream); if(res != RES_OK) { @@ -243,11 +346,6 @@ rng_state_cache_read } cache->read = ftell(cache->stream); - /* The fp reaches the end of the cached data */ - if(cache->read >= STATE_CACHE_HINT_MAX_SIZE) { - cache->read = 0; - } - /* Remove one cached states */ cache->nstates -= 1; @@ -288,43 +386,48 @@ error: static res_T rng_state_cache_write(struct rng_state_cache* cache, struct ssp_rng* rng) { + long remaining_space = 0; /* Remaining cache space */ + size_t len = 0; /* Length of the cache state in bytes */ res_T res = RES_OK; + ASSERT(cache && rng); if(cache->no_wstream) goto exit; /* Do not cache the submitted state */ - fseek(cache->stream, cache->write, SEEK_SET); - if(STATE_CACHE_HINT_MAX_SIZE > 0 - && (rng_state_cache_is_empty(cache) || cache->write > cache->read)) { - /* Directly write the RNG state into the cache stream */ - res = ssp_rng_write(rng, cache->stream); - if(res != RES_OK) goto error; - cache->write = ftell(cache->stream); + /* Store in memory the state to be cached */ + res = rng_write_cstr(rng, &cache->state, &len); + if(res != RES_OK) goto error; - /* The fp exceed the amount of cached data */ - if(cache->write >= STATE_CACHE_HINT_MAX_SIZE) { - cache->write = 0; - } + /* There are no spaces left at the end of the stream: rewind the writing */ + if(len > (size_t)(STATE_CACHE_CAPACITY - cache->write)) { + cache->end_of_cache = cache->write; /* Mark the end of cache */ + cache->write = 0; + } + /* Calculate remaining cache space */ + if(rng_state_cache_is_empty(cache) || cache->write > cache->read) { + remaining_space = STATE_CACHE_CAPACITY - cache->write; } else { - size_t len; - res = rng_write_cstr(rng, &cache->state, &len); - if(res != RES_OK) goto error; + remaining_space = cache->read - cache->write; + } - if(len > (size_t)(cache->read - cache->write)) { - /* No sufficient space into the cache stream to save the RNG state */ - cache->no_wstream = 1; - } else { - /* Write the RNG state into the cached stream */ - size_t sz; - sz = fwrite(darray_char_cdata_get(&cache->state), 1, len, cache->stream); - if(sz != len) { res = RES_IO_ERR; goto error; } - cache->write = ftell(cache->stream); - - /* The fp exceed the amount of cached data */ - if(cache->write >= STATE_CACHE_HINT_MAX_SIZE) { - cache->write = 0; - } + /* There is no sufficient space. Cache cannot be used */ + if(remaining_space < 0 || len > (size_t)remaining_space) { + cache->no_wstream = 1; + + /* There is sufficient space. Write the RNG state into the cached stream */ + } else { + size_t sz = 0; + + fseek(cache->stream, cache->write, SEEK_SET); + sz = fwrite(darray_char_cdata_get(&cache->state), 1, len, cache->stream); + if(sz != len) { res = RES_IO_ERR; goto error; } + cache->write = ftell(cache->stream); + + /* Flush write state to detect a write error */ + if(fflush(cache->stream) != 0) { + res = RES_IO_ERR; + goto error; } } @@ -497,7 +600,10 @@ rng_proxy_next_ran_pool /* Register a new state for *all* buckets */ FOR_EACH(ibucket, 0, sa_size(proxy->states)) { res = rng_state_cache_write(proxy->states + ibucket, proxy->rng); - if(res != RES_OK) FATAL("RNG proxy: cannot write to state cache\n"); + if(res != RES_OK) { + rng_state_cache_dump(proxy->states + ibucket); + FATAL("RNG proxy: cannot write to state cache\n"); + } ssp_rng_discard(proxy->rng, proxy->bucket_size); } /* Discard RNs to reach the next sequence */ @@ -513,7 +619,12 @@ rng_proxy_next_ran_pool (proxy->states + bucket_name, proxy->pools[bucket_name], proxy->mutex); - if(res != RES_OK) FATAL("RNG proxy: cannot read from state cache\n"); + if(res != RES_OK) { + mutex_lock(proxy->mutex); + rng_state_cache_dump(proxy->states + bucket_name); + mutex_unlock(proxy->mutex); + FATAL("RNG proxy: cannot read from state cache\n"); + } /* Update the sequence of the bucket RNG */ proxy->per_bucket_sequence_id[bucket_name] += 1; @@ -556,6 +667,14 @@ rng_proxy_setup sa_add(proxy->buckets, nbuckets); sa_add(proxy->per_bucket_sequence_id, nbuckets); + /* Clearing allocated memory. This operation is necessary to manage errors and + * identify which data has been initialised and which has not */ + memset(proxy->states, 0, sizeof(*proxy->states)*nbuckets); + memset(proxy->pools, 0, sizeof(*proxy->pools)*nbuckets); + memset(proxy->buckets, 0, sizeof(*proxy->buckets)*nbuckets); + memset(proxy->per_bucket_sequence_id, 0, + sizeof(*proxy->per_bucket_sequence_id)*nbuckets); + FOR_EACH(ibucket, 0, nbuckets) { res = rng_state_cache_init (proxy->allocator, sequence_pitch, proxy->states+ibucket); diff --git a/src/test_ssp_ran_circle.c b/src/test_ssp_ran_circle.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_circle.h b/src/test_ssp_ran_circle.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_discrete.c b/src/test_ssp_ran_discrete.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_discrete.h b/src/test_ssp_ran_discrete.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_gaussian.c b/src/test_ssp_ran_gaussian.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_gaussian.h b/src/test_ssp_ran_gaussian.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_hemisphere.c b/src/test_ssp_ran_hemisphere.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_hemisphere.h b/src/test_ssp_ran_hemisphere.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_hg.c b/src/test_ssp_ran_hg.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_hg.h b/src/test_ssp_ran_hg.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_piecewise_linear.c b/src/test_ssp_ran_piecewise_linear.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_piecewise_linear.h b/src/test_ssp_ran_piecewise_linear.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_sphere.c b/src/test_ssp_ran_sphere.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_sphere.h b/src/test_ssp_ran_sphere.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_sphere_cap.c b/src/test_ssp_ran_sphere_cap.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_sphere_cap.h b/src/test_ssp_ran_sphere_cap.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_spherical_zone.c b/src/test_ssp_ran_spherical_zone.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_spherical_zone.h b/src/test_ssp_ran_spherical_zone.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_tetrahedron.c b/src/test_ssp_ran_tetrahedron.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_tetrahedron.h b/src/test_ssp_ran_tetrahedron.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_triangle.c b/src/test_ssp_ran_triangle.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_triangle.h b/src/test_ssp_ran_triangle.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_uniform_disk.c b/src/test_ssp_ran_uniform_disk.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_ran_uniform_disk.h b/src/test_ssp_ran_uniform_disk.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_rng.c b/src/test_ssp_rng.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_rng.h b/src/test_ssp_rng.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_rng_proxy.c b/src/test_ssp_rng_proxy.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -476,7 +476,7 @@ test_read_with_cached_states(void) } /* Discard several RNs for the first RNG only to make under pressure the - * cache stream of 'rng1'. The cache stream limit is set to 32 MB and the + * cache stream of 'rng1'. The cache memory limit is 32 MB maximum and the * size of a Mersenne Twister RNG state is greater than 6 KB. Consquently, * ~5500 RNG states will exceed the cache stream, i.e. 5500*2 = 11000 * random generations (since there is 2 RNs per bucket). Above this limit, @@ -607,7 +607,7 @@ test_cache(void) CHK(ssp_rng_create(NULL, SSP_RNG_MT19937_64, &rng) == RES_OK); /* Generate several RNs for the first RNG only to make under pressure the - * cache stream of 'rng1'. The cache stream limit is set to 32 MB and the + * cache stream of 'rng1'. The cache memory limit is 32 MB maximum and the * size of a Mersenne Twister RNG state is greater than 6 KB. Consquently, * ~5500 RNG states will exceed the cache stream, i.e. 5500*2 = 11000 * random generations (since there is 2 RNs per bucket). Above this limit, @@ -628,6 +628,7 @@ test_cache(void) CHK(ssp_rng_get(rng1) == ssp_rng_get(rng)); if(i % 2) CHK(ssp_rng_discard(rng, 2) == RES_OK); } + CHK(ssp_rng_ref_put(rng) == RES_OK); CHK(ssp_rng_proxy_ref_put(proxy) == RES_OK); @@ -806,7 +807,7 @@ test_flush_sequence() /* Fill the cache of the rng1 */ CHK(ssp_rng_discard(rng0, 1013) == RES_OK); CHK(ssp_rng_proxy_get_sequence_id(proxy, &id) == RES_OK); - CHK(id == 63 + 203/*ceil(1023/(args.sequence_size*0.5))*/); + CHK(id == 63 + 203/*ceil(1013/(args.sequence_size*0.5))*/); /* Discard several sequences */ CHK(ssp_rng_proxy_flush_sequences(proxy, 314) == RES_OK); @@ -825,6 +826,24 @@ test_flush_sequence() CHK(id == 63 + 203 + 314); } + /* Generate several RNs for the first RNG only to make under pressure the + * cache stream of 'rng1'. The cache memory limit is 32 MB maximum and the + * size of a Mersenne Twister RNG state is greater than 6 KB. Consquently, + * ~5500 RNG states will exceed the cache stream, i.e. 5500*5 = 27500 + * random generations (since there is 5 RNs per bucket). Above this limit, + * 'rng1' will not rely anymore on the proxy RNG to manage its state. */ + CHK(ssp_rng_discard(rng0, 30000) == RES_OK); + + /* Discard enough random numbers from rng1 to consume all cached RNG states, + * and thus disable read cache usage */ + CHK(ssp_rng_discard(rng1, 30000) == RES_OK); + + CHK(ssp_rng_proxy_get_sequence_id(proxy, &id) == RES_OK); + CHK(id == 63+203+314+(30000*2)/args.sequence_size); + + /* Try to flush a sequence on a proxy whose a generator don't use cache */ + CHK(ssp_rng_proxy_flush_sequences(proxy, 1) == RES_OK); + CHK(ssp_rng_proxy_ref_put(proxy) == RES_OK); CHK(ssp_rng_ref_put(rng1) == RES_OK); CHK(ssp_rng_ref_put(rng0) == RES_OK); diff --git a/src/test_ssp_rng_proxy.h b/src/test_ssp_rng_proxy.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/test_ssp_utils.h b/src/test_ssp_utils.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 |Méso|Star> (contact@meso-star.com) +/* Copyright (C) 2015-2025 |Méso|Star> (contact@meso-star.com) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by