star-mesh

Define and load a binary data format for meshes
git clone git://git.meso-star.fr/star-mesh.git
Log | Files | Refs | README | LICENSE

commit cbf9aaae703b390303e4e46304df49060799a833
parent 9b5001e4084391e99e67cc27b0157d22546b5056
Author: Vincent Forest <vincent.forest@meso-star.com>
Date:   Fri, 27 Oct 2023 12:09:56 +0200

Merge branch 'release_0.1'

Diffstat:
M.gitignore | 14+++++++-------
AMakefile | 146+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
MREADME.md | 60+++++++++++++++++++++++++++++++++++++-----------------------
Dcmake/CMakeLists.txt | 118-------------------------------------------------------------------------------
Aconfig.mk | 77+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Ddoc/smsh.5.scd | 82-------------------------------------------------------------------------------
Amake.sh | 70++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Asmsh.5 | 102+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Asmsh.pc.in | 10++++++++++
Msrc/smsh.c | 254++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----------------
Msrc/smsh.h | 28+++++++++++++++++++++++-----
Msrc/test_smsh_load.c | 77+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------
12 files changed, 736 insertions(+), 302 deletions(-)

diff --git a/.gitignore b/.gitignore @@ -1,12 +1,12 @@ .gitignore -CMakeCache.txt -CMakeFiles -Makefile -tmp [Bb]uild* *.sw[po] -*.[ao] -*.orig +*.[aod] +*.so *~ +test* +!test*.[ch] +.config +.test tags - +*.pc diff --git a/Makefile b/Makefile @@ -0,0 +1,146 @@ +# Copyright (C) 2020-2023 |Méso|Star> (contact@meso-star.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +.POSIX: +.SUFFIXES: # Clean up default inference rules + +include config.mk + +LIBNAME_STATIC = libsmsh.a +LIBNAME_SHARED = libsmsh.so +LIBNAME = $(LIBNAME_$(LIB_TYPE)) + +################################################################################ +# Library building +################################################################################ +SRC = src/smsh.c src/smsh_log.c +OBJ = $(SRC:.c=.o) +DEP = $(SRC:.c=.d) + +build_library: .config $(DEP) + @$(MAKE) -fMakefile $$(for i in $(DEP); do echo -f $${i}; done) \ + $$(if [ -n "$(LIBNAME)" ]; then\ + echo "$(LIBNAME)";\ + else\ + echo "$(LIBNAME_SHARED)";\ + fi) + +$(DEP) $(OBJ): config.mk + +$(LIBNAME_SHARED): $(OBJ) + $(CC) $(CFLAGS_SO) $(RSYS_CFLAGS) -o $@ $(OBJ) $(LDFLAGS_SO) $(RSYS_LIBS) + +$(LIBNAME_STATIC): libsmsh.o + $(AR) -rc $@ $? + $(RANLIB) $@ + +libsmsh.o: $(OBJ) + $(LD) -r $(OBJ) -o $@ + $(OBJCOPY) $(OCPFLAGS) $@ + +.config: config.mk + @if ! $(PKG_CONFIG) --atleast-version $(RSYS_VERSION) rsys; then\ + echo "rsys $(RSYS_VERSION) not found" >&2; exit 1; fi + @echo "config done" > $@ + +.SUFFIXES: .c .d .o +.c.d: + @$(CC) $(CFLAGS_SO) $(RSYS_CFLAGS) -MM -MT "$(@:.d=.o) $@" $< -MF $@ + +.c.o: + $(CC) $(CFLAGS_SO) $(RSYS_CFLAGS) -DSMSH_SHARED_BUILD -c $< -o $@ + +################################################################################ +# Installation +################################################################################ +pkg: + sed -e 's#@PREFIX@#$(PREFIX)#g' \ + -e 's#@VERSION@#$(VERSION)#g' \ + -e 's#@RSYS_VERSION@#$(RSYS_VERSION)#g' \ + smsh.pc.in > smsh.pc + +smsh-local.pc: smsh.pc.in + sed -e '1d'\ + -e 's#^includedir=.*#includedir=./src/#'\ + -e 's#^libdir=.*#libdir=./#'\ + -e 's#@VERSION@#$(VERSION)#g'\ + -e 's#@RSYS_VERSION@#$(RSYS_VERSION)#g'\ + smsh.pc.in > $@ + +install: build_library pkg + @$(SHELL) make.sh install "$(DESTDIR)$(PREFIX)/lib" $(LIBNAME) + @$(SHELL) make.sh install "$(DESTDIR)$(PREFIX)/lib/pkgconfig" smsh.pc + @$(SHELL) make.sh install "$(DESTDIR)$(PREFIX)/include/star" src/smsh.h + @$(SHELL) make.sh install "$(DESTDIR)$(PREFIX)/share/doc/star-mesh" COPYING README.md + @$(SHELL) make.sh install "$(DESTDIR)$(PREFIX)/share/man/man5" smsh.5 + +uninstall: + rm -f "$(DESTDIR)$(PREFIX)/lib/$(LIBNAME)" + rm -f "$(DESTDIR)$(PREFIX)/lib/pkgconfig/smsh.pc" + rm -f "$(DESTDIR)$(PREFIX)/share/doc/star-mesh/COPYING" + rm -f "$(DESTDIR)$(PREFIX)/share/doc/star-mesh/README.md" + rm -f "$(DESTDIR)$(PREFIX)/include/star/smsh.h" + rm -f "$(DESTDIR)$(PREFIX)/share/man/man5/smsh.5" + +################################################################################ +# Miscellaneous targets +################################################################################ +all: build_library build_tests + +clean: clean_test + rm -f $(OBJ) $(TEST_OBJ) $(LIBNAME) + rm -f .config .test libsmsh.o smsh.pc smsh-local.pc + rm -f test_file.smsh + +distclean: clean + rm -f $(DEP) $(TEST_DEP) + +lint: + shellcheck -o all make.sh + mandoc -T lint -Wbase smsh.5 + +################################################################################ +# Tests +################################################################################ +TEST_SRC = src/test_smsh.c src/test_smsh_load.c +TEST_OBJ = $(TEST_SRC:.c=.o) +TEST_DEP = $(TEST_SRC:.c=.d) + +PKG_CONFIG_LOCAL = PKG_CONFIG_PATH="./:$${PKG_CONFIG_PATH}" $(PKG_CONFIG) +SMSH_CFLAGS = $$($(PKG_CONFIG_LOCAL) $(PCFLAGS) --cflags smsh-local.pc) +SMSH_LIBS = $$($(PKG_CONFIG_LOCAL) $(PCFLAGS) --libs smsh-local.pc) + +test: build_tests + @$(SHELL) make.sh run_test $(TEST_SRC) + +build_tests: build_library $(TEST_DEP) .test + @$(MAKE) -fMakefile -f.test $$(for i in $(TEST_DEP); do echo -f"$${i}"; done) test_bin + +.test: Makefile make.sh + @echo "Setup tests" + @$(SHELL) make.sh config_test $(TEST_SRC) > $@ + +clean_test: + @$(SHELL) make.sh clean_test $(TEST_SRC) + +$(TEST_DEP): config.mk smsh-local.pc + @$(CC) $(CFLAGS_EXE) $(RSYS_CFLAGS) $(SMSH_CFLAGS) \ + -MM -MT "$(@:.d=.o) $@" $(@:.d=.c) -MF $@ + +$(TEST_OBJ): config.mk smsh-local.pc + $(CC) $(CFLAGS_EXE) $(RSYS_CFLAGS) $(SMSH_CFLAGS) -c $(@:.o=.c) -o $@ + +test_smsh test_smsh_load: config.mk smsh-local.pc $(LIBNAME) + $(CC) $(CFLAGS_EXE) -o $@ src/$@.o $(LDFLAGS_EXE) $(SMSH_LIBS) $(RSYS_LIBS) diff --git a/README.md b/README.md @@ -1,28 +1,42 @@ # Star-Mesh -Star-Mesh is a C library that loads surfacic/volumetric meshes saved wrt the -Star-Mesh file format. - -## How to build - -Star-Mesh is compatible 64-bit POSIX systems. It relies on the -[CMake](http://www.cmake.org) and the -[RCMake](https://gitlab.com/vaplv/rcmake/) packages to build. It also depends -on the [RSys](https://gitlab.com/vaplv/rsys/) library. It optionally depends on -[scdoc](https://sr.ht/~sircmpwn/scdoc/) which, if available, is used to generate -the man page of the Star-Mesh file format. - -First ensure that CMake is installed on your system. Then install the RCMake -package as well as the aforementioned prerequisites. Finally generate the -project from the `cmake/CMakeLists.txt` file by appending to the -`CMAKE_PREFIX_PATH` variable the install directories of its dependencies. The -resulting project can be edited, built, tested and installed as any CMake -project. Refer to the [CMake](https://cmake.org/documentation) for further -informations on CMake. +Star-Mesh loads surfacic or volumetric meshes saved in Star-Mesh file +format. See smsh.5 for details. + +## Requirements + +- C compiler +- POSIX make +- pkg-config +- [RSys](https://gitlab.com/vaplv/rsys) +- [mandoc](https://mandoc.bsd.lv) + +## Installation + +Edit config.mk as needed, then run: + + make clean install + +## Release notes + +### Version 0.1 + +- Make memory mapping optional. By default, data is now loaded into + memory. Memory mapping becomes an option of the load functions, + (forbidden on stdin). As a consequence, this commit introduces API + breaks. +- Write the man page directly in mdoc's roff macros, instead of using + the intermediate scdoc source. +- Replace CMake by Makefile as build system. +- Update compiler and linker flags to increase the security and + robustness of generated binaries. +- Provide a pkg-config file to link the library as an external + dependency. ## License -Copyright (C) 2020-2023 [|Méso|Star>](https://www.meso-star.com) -(<contact@meso-star.com>). Star-Mesh is free software released under the -GPL v3+ license: GNU GPL version 3 or later. You are welcome to redistribute it -under certain conditions; refer to the COPYING file for details. +Copyright (C) 2020-2023 |Méso|Star> (contact@meso-star.com) + +Star-Mesh is free software released under the GPL v3+ license: GNU GPL +version 3 or later. You are welcome to redistribute it under certain +conditions; refer to the COPYING file for details. diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt @@ -1,118 +0,0 @@ -# Copyright (C) 2020-2023 |Méso|Star> (contact@meso-star.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -cmake_minimum_required(VERSION 3.1) -project(smsh C) -enable_testing() - -set(SMSH_SOURCE_DIR ${PROJECT_SOURCE_DIR}/../src) -option(NO_TEST "Do not build tests" OFF) - -################################################################################ -# Check dependencies -################################################################################ -find_package(RCMake 0.4 REQUIRED) -find_package(RSys 0.13 REQUIRED) - -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${RCMAKE_SOURCE_DIR}) -include(rcmake) -include(rcmake_runtime) - -include_directories(${RSys_INCLUDE_DIR}) - -################################################################################ -# Configure and define targets -################################################################################ -set(VERSION_MAJOR 0) -set(VERSION_MINOR 0) -set(VERSION_PATCH 0) -set(VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}) - -set(SMSH_FILES_SRC - smsh.c - smsh_log.c) -set(SMSH_FILES_INC - smsh_c.h - smsh_log.h) -set(SMSH_FILES_INC_API - smsh.h) - -set(SMSH_FILES_DOC COPYING README.md) - -# Prepend each file in the `SMSH_FILES_<SRC|INC>' list by `SMSH_SOURCE_DIR' -rcmake_prepend_path(SMSH_FILES_SRC ${SMSH_SOURCE_DIR}) -rcmake_prepend_path(SMSH_FILES_INC ${SMSH_SOURCE_DIR}) -rcmake_prepend_path(SMSH_FILES_INC_API ${SMSH_SOURCE_DIR}) -rcmake_prepend_path(SMSH_FILES_DOC ${PROJECT_SOURCE_DIR}/../) - -add_library(smsh SHARED ${SMSH_FILES_SRC} ${SMSH_FILES_INC} ${SMSH_FILES_INC_API}) -target_link_libraries(smsh RSys) - -set_target_properties(smsh PROPERTIES - DEFINE_SYMBOL SMSH_SHARED_BUILD - VERSION ${VERSION} - SOVERSION ${VERSION_MAJOR}) - -rcmake_setup_devel(smsh StarMesh ${VERSION} star/smsh_version.h) - -################################################################################ -# Add tests -################################################################################ -if(NOT NO_TEST) - function(build_test _name) - add_executable(${_name} ${SMSH_SOURCE_DIR}/${_name}.c) - target_link_libraries(${_name} smsh RSys ${ARGN}) - endfunction() - - function(new_test _name) - build_test(${_name} ${ARGN}) - add_test(${_name} ${_name}) - endfunction() - - new_test(test_smsh) - new_test(test_smsh_load) -endif() - -################################################################################ -# Man page -############################################################################### -find_program(SCDOC NAMES scdoc) -if(NOT SCDOC) - message(WARNING - "The `scdoc' program is missing. " - "The Star-Mesh man page cannot be generated.") -else() - set(_src ${PROJECT_SOURCE_DIR}/../doc/smsh.5.scd) - add_custom_command( - OUTPUT smsh.5 - COMMAND ${SCDOC} < ${_src} > smsh.5 - DEPENDS ${_src} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Buid ROFF man page smsh.5" - VERBATIM) - add_custom_target(man-roff ALL DEPENDS smsh.5) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/smsh.5 DESTINATION share/man/man5) -endif() - -################################################################################ -# Define output & install directories -################################################################################ -install(TARGETS smsh - ARCHIVE DESTINATION bin - LIBRARY DESTINATION lib - RUNTIME DESTINATION bin) -install(FILES ${SMSH_FILES_INC_API} DESTINATION include/star) -install(FILES ${SMSH_FILES_DOC} DESTINATION share/doc/star-mesh) - diff --git a/config.mk b/config.mk @@ -0,0 +1,77 @@ +VERSION = 0.1.0 +PREFIX = /usr/local + +LIB_TYPE = SHARED +#LIB_TYPE = STATIC + +BUILD_TYPE = RELEASE +#BUILD_TYPE = DEBUG + +################################################################################ +# Tools +################################################################################ +AR = ar +CC = cc +LD = ld +OBJCOPY = objcopy +PKG_CONFIG = pkg-config +RANLIB = ranlib + +################################################################################ +# Dependencies +################################################################################ +PCFLAGS_SHARED = +PCFLAGS_STATIC = --static +PCFLAGS = $(PCFLAGS_$(LIB_TYPE)) + +RSYS_VERSION=0.14 +RSYS_CFLAGS = $$($(PKG_CONFIG) $(PCFLAGS) --cflags rsys) +RSYS_LIBS = $$($(PKG_CONFIG) $(PCFLAGS) --libs rsys) + +################################################################################ +# Compilation options +################################################################################ +WFLAGS =\ + -Wall\ + -Wcast-align\ + -Wconversion\ + -Wextra\ + -Wmissing-declarations\ + -Wmissing-prototypes\ + -Wshadow + +CFLAGS_HARDENED =\ + -D_FORTIFY_SOURCES=2\ + -fcf-protection=full\ + -fstack-clash-protection\ + -fstack-protector-strong + +CFLAGS_COMMON =\ + -std=c89\ + -pedantic\ + -fvisibility=hidden\ + -fstrict-aliasing\ + $(CFLAGS_HARDENED)\ + $(WFLAGS) + +CFLAGS_DEBUG = -g $(CFLAGS_COMMON) +CFLAGS_RELEASE = -O2 -DNDEBUG $(CFLAGS_COMMON) +CFLAGS = $(CFLAGS_$(BUILD_TYPE)) + +CFLAGS_SO = $(CFLAGS) -fPIC +CFLAGS_EXE = $(CFLAGS) -fPIE + +################################################################################ +# Linker options +################################################################################ +LDFLAGS_HARDENED = -Wl,-z,relro,-z,now +LDFLAGS_DEBUG = $(LDFLAGS_HARDENED) +LDFLAGS_RELEASE = -s $(LDFLAGS_HARDENED) +LDFLAGS = $(LDFLAGS_$(BUILD_TYPE)) + +LDFLAGS_SO = $(LDFLAGS) -shared -Wl,--no-undefined +LDFLAGS_EXE = $(LDFLAGS) -pie + +OCPFLAGS_DEBUG = --localize-hidden +OCPFLAGS_RELEASE = --localize-hidden --strip-unneeded +OCPFLAGS = $(OCPFLAGS_$(BUILD_TYPE)) diff --git a/doc/smsh.5.scd b/doc/smsh.5.scd @@ -1,82 +0,0 @@ -smsh(5) - -; Copyright (C) 2020-2023 |Méso|Star> (contact@meso-star.com) -; -; This program is free software: you can redistribute it and/or modify -; it under the terms of the GNU General Public License as published by -; the Free Software Foundation, either version 3 of the License, or -; (at your option) any later version. -; -; This program is distributed in the hope that it will be useful, -; but WITHOUT ANY WARRANTY; without even the implied warranty of -; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -; GNU General Public License for more details. -; -; You should have received a copy of the GNU General Public License -; along with this program. If not, see <http://www.gnu.org/licenses/>. - -# NAME - -smsh - Star-Mesh file format - -# DESCRIPTION - -*smsh* is a binary file format that describes an indexed mesh (surface or -volume). Only the geometric data of the mesh is stored; no additional properties -are attached to its nodes or cells. - -A *smsh* file begins with a header that describes the layout of the data, -followed by the geometric data itself, i.e. the list of nodes and the list of -cells. - -The header consists of 5 integers. The first integer is a power of two (usually -4096) that defines the size of the memory page in bytes (_pagesize_) on which -the list of nodes and the list of cells are aligned. By aligning data to -_pagesize_, and depending on system requirements, memory mapping can be used to -automatically load/unload pages on demand (see *mmap*(2)). The remaining -integers store the number of nodes (_#nodes_) and the number of cells -(_#cells_), followed by the size of a node (_dimnode_) and the size of a cell -(_dimcell_), respectively. - -Fill bytes follow the file header to align nodes to _pagesize_. The nodes are -then listed with a list of _#nodes_ _dimnode_ double-precision floating-point -numbers per node, where _#nodes_ is the number of mesh nodes and _dimnode_ is -the number of floating-point numbers per node. Additional fill bytes are added -after the node list to align the list of upcoming cells to _pagesize_. The cells -are then listed using _dimcell_ 64-bit signed integers per node where each integer -indexes a node in the previously defined list of nodes (indexing starts at 0). -Finally, fill bytes are added to align the overall file size to _pagesize_. - -# BINARY FILE FORMAT - -Data are encoded with respect to the little endian bytes ordering, i.e. least -significant bytes are stored first. - -``` -<smsh> ::= <pagesize> <#nodes> <#cells> <dimnode> <dimcell> - <padding> - <nodes> - <padding> - <cells> - <padding> -<pagesize> ::= UINT64 -<#nodes> ::= UINT64 -<#cells> ::= UINT64 -<dimnode> ::= UINT32 -<dimcell> ::= UINT32 - -<padding> ::= [ BYTE ... ] - -<nodes> ::= <node-pos> - [ <node-pos> ... ] - -<cells> ::= <cell-ids> - [ <cell-ids> ... ] - -<node-pos> ::= DOUBLE ... -<cell-ids> ::= UINT64 ... -``` - -# SEE ALSO - -*mmap*(2) diff --git a/make.sh b/make.sh @@ -0,0 +1,70 @@ +#!/bin/sh + +# Copyright (C) 2020-2023 |Méso|Star> (contact@meso-star.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +set -e + +config_test() +{ + for i in "$@"; do + test=$(basename "${i}" ".c") + test_list="${test_list} ${test}" + printf "%s: %s\n" "${test}" "src/${test}.o" + done + printf "test_bin: %s\n" "${test_list}" +} + +run_test() +{ + for i in "$@"; do + test=$(basename "${i}" ".c") + + printf "%s " "${test}" + if "./${test}" > /dev/null 2>&1; then + printf "\033[1;32mOK\033[m\n" + else + printf "\033[1;31mError\033[m\n" + fi + done 2> /dev/null +} + +clean_test() +{ + for i in "$@"; do + rm -f "$(basename "${i}" ".c")" + done +} + +install() +{ + prefix=$1 + shift 1 + + mkdir -p "${prefix}" + + for i in "$@"; do + dst="${prefix}/${i##*/}" + + if cmp -s "${i}" "${dst}"; then + printf "Up to date %s\n" "${dst}" + else + printf "Installing %s\n" "${dst}" + cp "${i}" "${prefix}" + fi + done +} + +"$@" diff --git a/smsh.5 b/smsh.5 @@ -0,0 +1,102 @@ +.\" Copyright (C) 2020-2023 |Méso|Star> (contact@meso-star.com) +.\" +.\" This program is free software: you can redistribute it and/or modify +.\" it under the terms of the GNU General Public License as published by +.\" the Free Software Foundation, either version 3 of the License, or +.\" (at your option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, +.\" but WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +.\" GNU General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public License +.\" along with this program. If not, see <http://www.gnu.org/licenses/>. +.Dd July 26, 2023 +.Dt SMSH 5 +.Os +.Sh NAME +.Nm smsh +.Nd Star-Mesh file format +.Sh DESCRIPTION +.Nm +is a binary file format that describes an indexed mesh +.Pq surface or volume . +Only the geometric data of the mesh is stored; no additional properties are +attached to its nodes or cells. +.Pp +A +.Nm +file begins with a header that describes the layout of the data, followed by +the geometric data itself, i.e. the list of nodes and the list of cells. +.Pp +The header consists of 5 integers. +The first integer is a power of two +.Pq usually 4096 +that defines the size of the memory page in bytes +.Pq Va pagesize +on which the list of nodes and the list of cells are aligned. +By aligning data to +.Va pagesize , +and depending on system requirements, memory mapping can be used to +automatically load/unload pages on demand +.Pq see Xr mmap 2 . +The remaining integers store the number of nodes +.Pq Va #nodes +and the number of cells +.Pq Va #cells , +followed by the size of a node +.Pq Va dimnode +and the size of a cell +.Pq Va dimcell , +respectively. +.Pp +Fill bytes follow the file header to align nodes to +.Va pagesize . +The nodes are then listed with a list of +.Va #nodes dimnode +double-precision floating-point numbers per node, where +.Va #nodes +is the number +of mesh nodes and +.Va dimnode +is the number of floating-point numbers per node. +Additional fill bytes are added after the node list to align the list of +upcoming cells to +.Va pagesize . +The cells are then listed using +.Va dimcell +64-bit unsigned integers per node where each +integer indexes a node in the previously defined list of nodes +.Pq indexing starts at 0 . +Finally, fill bytes are added to align the overall file size to +.Va pagesize . +.Pp +Data are encoded with respect to the little endian bytes ordering, i.e. least +significant bytes are stored first. +.Pp +The file format is as follows: +.Bl -column (pagesize) (::=) () +.It Ao Va smsh Ac Ta ::= Ta Ao Va pagesize Ac Ao Va #nodes Ac Ao Va #cells Ac Ao Va dimnode Ac Ao Va dimcel Ac +.It Ta Ta Aq Va padding +.It Ta Ta Aq Va nodes +.It Ta Ta Aq Va padding +.It Ta Ta Aq Va cells +.It Ta Ta Aq Va padding +.It Ao Va pagesize Ac Ta ::= Ta Vt uint64_t +.It Ao Va #nodes Ac Ta ::= Ta Vt uint64_t +.It Ao Va #cells Ac Ta ::= Ta Vt uint64_t +.It Ao Va dimnode Ac Ta ::= Ta Vt uint32_t +.It Ao Va dimcell Ac Ta ::= Ta Vt uint32_t +.It \ Ta Ta +.It Ao Va padding Ac Ta ::= Ta Op Vt int8_t ... +# Ensure alignment on +.Va pagesize +.It \ Ta Ta +.It Ao Va nodes Ac Ta ::= Ta Ao Va node-pos Ac Va ... +.It Ao Va cells Ac Ta ::= Ta Ao Va cell-ids Ac Va ... +.It Ao Va node-pos Ac Ta ::= Ta Vt double ... +.It Ao Va cell-ids Ac Ta ::= Ta Vt uint64_t ... +.El +.Sh SEE ALSO +.Xr mmap 2 diff --git a/smsh.pc.in b/smsh.pc.in @@ -0,0 +1,10 @@ +prefix=@PREFIX@ +includedir=${prefix}/include +libdir=${prefix}/lib + +Requires: rsys >= @RSYS_VERSION@ +Name: Star-Mesh +Description: Star-Mesh Library +Version: @VERSION@ +Libs: -L${libdir} -lsmsh +CFlags: -I${includedir} diff --git a/src/smsh.c b/src/smsh.c @@ -21,15 +21,29 @@ #include "smsh_c.h" #include "smsh_log.h" +#include <rsys/cstr.h> #include <rsys/mem_allocator.h> #include <errno.h> #include <unistd.h> #include <sys/mman.h> /* mmap */ +#include <sys/stat.h> /* fstat */ /******************************************************************************* * Helper functions ******************************************************************************/ +static INLINE int +is_stdin(FILE* stream) +{ + struct stat stream_buf; + struct stat stdin_buf; + ASSERT(stream); + + CHK(fstat(fileno(stream), &stream_buf) == 0); + CHK(fstat(STDIN_FILENO, &stdin_buf) == 0); + return stream_buf.st_dev == stdin_buf.st_dev; +} + static INLINE res_T check_smsh_create_args(const struct smsh_create_args* args) { @@ -37,6 +51,28 @@ check_smsh_create_args(const struct smsh_create_args* args) return args ? RES_OK : RES_BAD_ARG; } +static INLINE res_T +check_smsh_load_args(const struct smsh_load_args* args) +{ + if(!args || !args->path) return RES_BAD_ARG; + return RES_OK; +} + +static INLINE res_T +check_smsh_load_stream_args + (struct smsh* smsh, + const struct smsh_load_stream_args* args) +{ + if(!args || !args->stream || !args->name) return RES_BAD_ARG; + if(args->memory_mapping && is_stdin(args->stream)) { + log_err(smsh, + "%s: unable to use memory mapping on data loaded from stdin\n", + args->name); + return RES_BAD_ARG; + } + return RES_OK; +} + static void reset_smsh(struct smsh* smsh) { @@ -44,10 +80,20 @@ reset_smsh(struct smsh* smsh) smsh->nnodes = 0; smsh->ncells = 0; smsh->pagesize = 0; - if(smsh->nodes && smsh->nodes != MAP_FAILED) - munmap(smsh->nodes, smsh->map_len_nodes); - if(smsh->cells && smsh->cells != MAP_FAILED) - munmap(smsh->cells, smsh->map_len_cells); + if(smsh->nodes) { + if(!smsh->map_len_nodes) { + MEM_RM(smsh->allocator, smsh->nodes); + } else if(smsh->nodes != MAP_FAILED) { + munmap(smsh->nodes, smsh->map_len_nodes); + } + } + if(smsh->cells) { + if(!smsh->map_len_cells) { + MEM_RM(smsh->allocator, smsh->cells); + } else if(smsh->nodes != MAP_FAILED) { + munmap(smsh->cells, smsh->map_len_cells); + } + } smsh->nodes = NULL; smsh->cells = NULL; smsh->map_len_nodes = 0; @@ -57,7 +103,7 @@ reset_smsh(struct smsh* smsh) static res_T map_data (struct smsh* smsh, - const char* stream_name, + const char* name, const int fd, /* File descriptor */ const size_t filesz, /* Overall filesize */ const char* data_name, @@ -67,12 +113,12 @@ map_data { void* map = NULL; res_T res = RES_OK; - ASSERT(smsh && stream_name && filesz && data_name && map_len && out_map); + ASSERT(smsh && name && filesz && data_name && map_len && out_map); ASSERT(IS_ALIGNED((size_t)offset, (size_t)smsh->pagesize)); if((size_t)offset + map_len > filesz) { log_err(smsh, "%s: the %s to load exceed the file size.\n", - stream_name, data_name); + name, data_name); res = RES_IO_ERR; goto error; } @@ -80,7 +126,7 @@ map_data map = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE|MAP_POPULATE, fd, offset); if(map == MAP_FAILED) { log_err(smsh, "%s: could not map the %s -- %s.\n", - stream_name, data_name, strerror(errno)); + name, data_name, strerror(errno)); res = RES_IO_ERR; goto error; } @@ -94,25 +140,140 @@ error: } static res_T -load_stream(struct smsh* smsh, FILE* stream, const char* stream_name) +map_file(struct smsh* smsh, FILE* stream, const char* name) { off_t pos_offset; off_t ids_offset; size_t filesz; res_T res = RES_OK; - ASSERT(smsh && stream && stream_name); + ASSERT(smsh && stream && name); + + /* Compute the length in bytes of the data to map */ + smsh->map_len_nodes = smsh->nnodes * sizeof(double) * smsh->dnode; + smsh->map_len_cells = smsh->ncells * sizeof(uint64_t) * smsh->dcell; + smsh->map_len_nodes = ALIGN_SIZE(smsh->map_len_nodes, (size_t)smsh->pagesize); + smsh->map_len_cells = ALIGN_SIZE(smsh->map_len_cells, (size_t)smsh->pagesize); + + /* Find the offsets of the positions/indices data into the stream */ + pos_offset = (off_t)ALIGN_SIZE((uint64_t)ftell(stream), smsh->pagesize); + ids_offset = (off_t)((size_t)pos_offset + smsh->map_len_nodes); + + /* Retrieve the overall filesize */ + fseek(stream, 0, SEEK_END); + filesz = (size_t)ftell(stream); + + /* Map the nodes */ + res = map_data(smsh, name, fileno(stream), filesz, "nodes", + pos_offset, smsh->map_len_nodes, (void**)&smsh->nodes); + if(res != RES_OK) goto error; + + /* Map the cells */ + res = map_data(smsh, name, fileno(stream), filesz, "cells", + ids_offset, smsh->map_len_cells, (void**)&smsh->cells); + if(res != RES_OK) goto error; + +exit: + return res; +error: + goto exit; +} + +static res_T +read_padding(FILE* stream, const size_t padding) +{ + char chunk[1024]; + size_t remaining_nbytes = padding; + + while(remaining_nbytes) { + const size_t nbytes = MMIN(sizeof(chunk), remaining_nbytes); + if(fread(chunk, 1, nbytes, stream) != nbytes) return RES_IO_ERR; + remaining_nbytes -= nbytes; + } + return RES_OK; +} + +static res_T +load_file(struct smsh* smsh, FILE* stream, const char* name) +{ + size_t ncoords; + size_t nindices; + size_t padding_bytes; + size_t sizeof_nodes; + size_t sizeof_cells; + size_t sizeof_header; + res_T res = RES_OK; + ASSERT(smsh && stream && name); + + ncoords = smsh->nnodes * smsh->dnode; + nindices = smsh->ncells * smsh->dcell; + sizeof_nodes = ncoords * sizeof(*smsh->nodes); + sizeof_cells = nindices * sizeof(*smsh->cells); + sizeof_header = + sizeof(smsh->pagesize) + + sizeof(smsh->nnodes) + + sizeof(smsh->ncells) + + sizeof(smsh->dnode) + + sizeof(smsh->dcell); + + /* Allocate the memory space where the loaded data will be stored */ + smsh->nodes = MEM_CALLOC(smsh->allocator, ncoords, sizeof(*smsh->nodes)); + if(!smsh->nodes) { res = RES_MEM_ERR; goto error; } + smsh->cells = MEM_CALLOC(smsh->allocator, nindices, sizeof(*smsh->cells)); + if(!smsh->cells) { res = RES_MEM_ERR; goto error; } + + padding_bytes = ALIGN_SIZE(sizeof_header, smsh->pagesize) - sizeof_header; + if((res = read_padding(stream, padding_bytes)) != RES_OK) goto error; + + /* Load the nodes */ + if(fread(smsh->nodes, sizeof(*smsh->nodes), ncoords, stream) != ncoords) { + res = RES_IO_ERR; + goto error; + } + + padding_bytes = ALIGN_SIZE(sizeof_nodes, smsh->pagesize) - sizeof_nodes; + if((res = read_padding(stream, padding_bytes)) != RES_OK) goto error; + + /* Load the cells */ + if(fread(smsh->cells, sizeof(*smsh->cells), nindices, stream) != nindices) { + res = RES_IO_ERR; + goto error; + } + + padding_bytes = ALIGN_SIZE(sizeof_cells, smsh->pagesize) - sizeof_cells; + if((res = read_padding(stream, padding_bytes)) != RES_OK) goto error; + +exit: + return res; +error: + log_err(smsh, "%s: error while loading data -- %s.\n", + name, res_to_cstr(res)); + goto exit; +} + +static res_T +load_stream(struct smsh* smsh, const struct smsh_load_stream_args* args) +{ + res_T res = RES_OK; + ASSERT(smsh && check_smsh_load_stream_args(smsh, args) == RES_OK); reset_smsh(smsh); /* Read file header */ + if(fread(&smsh->pagesize, sizeof(&smsh->pagesize), 1, args->stream) != 1) { + if(ferror(args->stream)) { + log_err(smsh, "%s: could not read the pagesize.\n", args->name); + } + res = RES_IO_ERR; + goto error; + } + #define READ(Var, N, Name) { \ - if(fread((Var), sizeof(*(Var)), (N), stream) != (N)) { \ - log_err(smsh, "%s: could not read the %s.\n", stream_name, (Name)); \ + if(fread((Var), sizeof(*(Var)), (N), args->stream) != (N)) { \ + log_err(smsh, "%s: could not read the %s.\n", args->name, (Name)); \ res = RES_IO_ERR; \ goto error; \ } \ } (void)0 - READ(&smsh->pagesize, 1, "page size"); READ(&smsh->nnodes, 1, "number of nodes"); READ(&smsh->ncells, 1, "number of cells"); READ(&smsh->dnode, 1, "node dimension"); @@ -123,34 +284,18 @@ load_stream(struct smsh* smsh, FILE* stream, const char* stream_name) log_err(smsh, "%s: invalid page size %li. The page size attribute must be aligned on " "the page size of the operating system (%lu).\n", - stream_name, smsh->pagesize, (unsigned long)smsh->pagesize_os); + args->name, smsh->pagesize, (unsigned long)smsh->pagesize_os); res = RES_BAD_ARG; goto error; } - /* Compute the length in bytes of the data to map */ - smsh->map_len_nodes = smsh->nnodes * sizeof(double) * smsh->dnode; - smsh->map_len_cells = smsh->ncells * sizeof(uint64_t) * smsh->dcell; - smsh->map_len_nodes = ALIGN_SIZE(smsh->map_len_nodes, (size_t)smsh->pagesize); - smsh->map_len_cells = ALIGN_SIZE(smsh->map_len_cells, (size_t)smsh->pagesize); - - /* Find the offsets of the positions/indices data into the stream */ - pos_offset = (off_t)ALIGN_SIZE((uint64_t)ftell(stream), smsh->pagesize); - ids_offset = (off_t)((size_t)pos_offset + smsh->map_len_nodes); - - /* Retrieve the overall filesize */ - fseek(stream, 0, SEEK_END); - filesz = (size_t)ftell(stream); - - /* Map the nodes */ - res = map_data(smsh, stream_name, fileno(stream), filesz, "nodes", - pos_offset, smsh->map_len_nodes, (void**)&smsh->nodes); - if(res != RES_OK) goto error; - - /* Map the cells */ - res = map_data(smsh, stream_name, fileno(stream), filesz, "cells", - ids_offset, smsh->map_len_cells, (void**)&smsh->cells); - if(res != RES_OK) goto error; + if(args->memory_mapping) { + res = map_file(smsh, args->stream, args->name); + if(res != RES_OK) goto error; + } else { + res = load_file(smsh, args->stream, args->name); + if(res != RES_OK) goto error; + } exit: return res; @@ -240,24 +385,27 @@ smsh_ref_put(struct smsh* smsh) res_T -smsh_load(struct smsh* smsh, const char* path) +smsh_load(struct smsh* smsh, const struct smsh_load_args* args) { + struct smsh_load_stream_args stream_args = SMSH_LOAD_STREAM_ARGS_NULL; FILE* file = NULL; res_T res = RES_OK; - if(!smsh || !path) { - res = RES_BAD_ARG; - goto error; - } + if(!smsh) { res = RES_BAD_ARG; goto error; } + res = check_smsh_load_args(args); + if(res != RES_OK) goto error; - file = fopen(path, "r"); + file = fopen(args->path, "r"); if(!file) { - log_err(smsh, "%s: error opening file `%s'.\n", FUNC_NAME, path); + log_err(smsh, "%s: error opening file `%s'.\n", FUNC_NAME, args->path); res = RES_IO_ERR; goto error; } - res = load_stream(smsh, file, path); + stream_args.stream = file; + stream_args.name = args->path; + stream_args.memory_mapping = args->memory_mapping; + res = load_stream(smsh, &stream_args); if(res != RES_OK) goto error; exit: @@ -268,13 +416,13 @@ error: } res_T -smsh_load_stream - (struct smsh* smsh, - FILE* stream, - const char* stream_name) +smsh_load_stream(struct smsh* smsh, const struct smsh_load_stream_args* args) { - if(!smsh || !stream) return RES_BAD_ARG; - return load_stream(smsh, stream, stream_name ? stream_name : "<stream>"); + res_T res = RES_OK; + if(!smsh) return RES_BAD_ARG; + res = check_smsh_load_stream_args(smsh, args); + if(res != RES_OK) return res; + return load_stream(smsh, args); } res_T @@ -301,8 +449,8 @@ smsh_desc_compute_hash(const struct smsh_desc* desc, hash256_T hash) sha256_ctx_update(&ctx, (const char*)(Var), sizeof(*Var)*(Nb)); sha256_ctx_init(&ctx); - HASH(desc->nodes, desc->nnodes); - HASH(desc->cells, desc->ncells); + HASH(desc->nodes, desc->nnodes*desc->dnode); + HASH(desc->cells, desc->ncells*desc->dcell); HASH(&desc->nnodes, 1); HASH(&desc->ncells, 1); HASH(&desc->dnode, 1); diff --git a/src/smsh.h b/src/smsh.h @@ -50,9 +50,28 @@ struct smsh_create_args { static const struct smsh_create_args SMSH_CREATE_ARGS_DEFAULT = SMSH_CREATE_ARGS_DEFAULT__; +struct smsh_load_args { + const char* path; + int memory_mapping; /* Use memory mapping instead of normal loading */ +}; +#define SMSH_LOAD_ARGS_NULL__ {NULL, 0} +static const struct smsh_load_args SMSH_LOAD_ARGS_NULL = + SMSH_LOAD_ARGS_NULL__; + +struct smsh_load_stream_args { + FILE* stream; + const char* name; /* Name of the stream */ + /* Use memory mapping instead of normal loading. Note that memory mapping + * cannot be used on some stream like stdin */ + int memory_mapping; +}; +#define SMSH_LOAD_STREAM_ARGS_NULL__ {NULL, "stream", 0} +static const struct smsh_load_stream_args SMSH_LOAD_STREAM_ARGS_NULL = + SMSH_LOAD_STREAM_ARGS_NULL__; + struct smsh_desc { - const double* nodes; /* List of double[3] */ - const uint64_t* cells; /* List of uint64_t[4] */ + const double* nodes; /* List of double[dnode] */ + const uint64_t* cells; /* List of uint64_t[dcell] */ size_t nnodes; size_t ncells; unsigned dnode; /* Dimension of a node */ @@ -85,13 +104,12 @@ smsh_ref_put SMSH_API res_T smsh_load (struct smsh* smsh, - const char* path); + const struct smsh_load_args* args); SMSH_API res_T smsh_load_stream (struct smsh* smsh, - FILE* stream, - const char* stream_name); /* NULL <=> use default stream name */ + const struct smsh_load_stream_args* args); SMSH_API res_T smsh_get_desc diff --git a/src/test_smsh_load.c b/src/test_smsh_load.c @@ -19,7 +19,9 @@ #include <rsys/math.h> #include <rsys/mem_allocator.h> #include <rsys/rsys.h> + #include <stdio.h> +#include <string.h> #include <unistd.h> /******************************************************************************* @@ -70,6 +72,8 @@ test_load_mesh(struct smsh* smsh, const uint32_t dnode, const uint32_t dcell) hash256_T hash0; hash256_T hash1; struct smsh_desc desc = SMSH_DESC_NULL; + struct smsh_load_args args = SMSH_LOAD_ARGS_NULL; + struct smsh_load_stream_args stream_args = SMSH_LOAD_STREAM_ARGS_NULL; FILE* fp = NULL; const char* filename = "test_file.smsh"; const uint64_t pagesize = 16384; @@ -116,9 +120,18 @@ test_load_mesh(struct smsh* smsh, const uint32_t dnode, const uint32_t dcell) CHK(fwrite(&byte, sizeof(byte), 1, fp) == 1); rewind(fp); - CHK(smsh_load_stream(NULL, fp, filename) == RES_BAD_ARG); - CHK(smsh_load_stream(smsh, NULL, filename) == RES_BAD_ARG); - CHK(smsh_load_stream(smsh, fp, NULL) == RES_OK); + + stream_args.stream = fp; + stream_args.name = filename; + CHK(smsh_load_stream(NULL, &stream_args) == RES_BAD_ARG); + CHK(smsh_load_stream(smsh, NULL) == RES_BAD_ARG); + stream_args.stream = NULL; + CHK(smsh_load_stream(smsh, &stream_args) == RES_BAD_ARG); + stream_args.stream = fp; + stream_args.name = NULL; + CHK(smsh_load_stream(smsh, &stream_args) == RES_BAD_ARG); + stream_args.name = filename; + CHK(smsh_load_stream(smsh, &stream_args) == RES_OK); CHK(smsh_get_desc(NULL, &desc) == RES_BAD_ARG); CHK(smsh_get_desc(smsh, NULL) == RES_BAD_ARG); CHK(smsh_get_desc(smsh, &desc) == RES_OK); @@ -129,25 +142,42 @@ test_load_mesh(struct smsh* smsh, const uint32_t dnode, const uint32_t dcell) CHK(smsh_desc_compute_hash(&desc, hash0) == RES_OK); rewind(fp); - CHK(smsh_load_stream(smsh, fp, filename) == RES_OK); + stream_args.name = SMSH_LOAD_STREAM_ARGS_NULL.name; + stream_args.memory_mapping = 1; + CHK(smsh_load_stream(smsh, &stream_args) == RES_OK); CHK(smsh_get_desc(smsh, &desc) == RES_OK); check_smsh_desc(&desc, nnodes, ncells, dnode, dcell); CHK(smsh_desc_compute_hash(&desc, hash1) == RES_OK); CHK(hash256_eq(hash0, hash1)); + CHK(fclose(fp) == 0); - CHK(smsh_load(NULL, filename) == RES_BAD_ARG); + args.path = filename; + CHK(smsh_load(NULL, &args) == RES_BAD_ARG); CHK(smsh_load(smsh, NULL) == RES_BAD_ARG); - CHK(smsh_load(smsh, "nop") == RES_IO_ERR); - CHK(smsh_load(smsh, filename) == RES_OK); + args.path = NULL; + CHK(smsh_load(smsh, &args) == RES_BAD_ARG); + args.path = "nop"; + CHK(smsh_load(smsh, &args) == RES_IO_ERR); + args.path = filename; + CHK(smsh_load(smsh, &args) == RES_OK); + CHK(smsh_get_desc(smsh, &desc) == RES_OK); check_smsh_desc(&desc, nnodes, ncells, dnode, dcell); + CHK(smsh_desc_compute_hash(&desc, hash1) == RES_OK); + CHK(hash256_eq(hash0, hash1)); - fclose(fp); + args.memory_mapping = 1; + CHK(smsh_load(smsh, &args) == RES_OK); + CHK(smsh_get_desc(smsh, &desc) == RES_OK); + check_smsh_desc(&desc, nnodes, ncells, dnode, dcell); + CHK(smsh_desc_compute_hash(&desc, hash1) == RES_OK); + CHK(hash256_eq(hash0, hash1)); } static void test_load_fail(struct smsh* smsh) { + struct smsh_load_stream_args args = SMSH_LOAD_STREAM_ARGS_NULL; const char byte = 0; FILE* fp = NULL; uint64_t pagesize; @@ -176,8 +206,9 @@ test_load_fail(struct smsh* smsh) CHK(fseek(fp, (long)ALIGN_SIZE((size_t)ftell(fp), pagesize)-1, SEEK_SET) == 0); CHK(fwrite(&byte, sizeof(byte), 1, fp) == 1); rewind(fp); - CHK(smsh_load_stream(smsh, fp, NULL) == RES_BAD_ARG); - fclose(fp); + args.stream = fp; + CHK(smsh_load_stream(smsh, &args) == RES_BAD_ARG); + CHK(fclose(fp) == 0); /* Wrong size */ fp = tmpfile(); @@ -199,8 +230,9 @@ test_load_fail(struct smsh* smsh) CHK(fseek(fp, (long)ALIGN_SIZE((size_t)ftell(fp), pagesize)-2, SEEK_SET) == 0); CHK(fwrite(&byte, sizeof(byte), 1, fp) == 1); rewind(fp); - CHK(smsh_load_stream(smsh, fp, NULL) == RES_IO_ERR); - fclose(fp); + args.stream = fp; + CHK(smsh_load_stream(smsh, &args) == RES_IO_ERR); + CHK(fclose(fp) == 0); } static void @@ -215,8 +247,25 @@ test_load_files(struct smsh* smsh, int argc, char** argv) size_t inode; size_t icell; - printf("Load %s\n", argv[i]); - CHK(smsh_load(smsh, argv[i]) == RES_OK); + /* Load from file */ + if(strcmp(argv[i], "-") != 0) { + struct smsh_load_args args = SMSH_LOAD_ARGS_NULL; + printf("Load %s\n", argv[i]); + args.path = argv[i]; + args.memory_mapping = 1; + CHK(smsh_load(smsh, &args) == RES_OK); + + /* Load from stdin */ + } else { + struct smsh_load_stream_args args = SMSH_LOAD_STREAM_ARGS_NULL; + printf("Load from stdin\n"); + args.stream = stdin; + args.name = "stdin"; + args.memory_mapping = 1; + CHK(smsh_load_stream(smsh, &args) == RES_BAD_ARG); + args.memory_mapping = 0; + CHK(smsh_load_stream(smsh, &args) == RES_OK); + } CHK(smsh_get_desc(smsh, &desc) == RES_OK); CHK(smsh_desc_compute_hash(&desc, hash) == RES_OK);