From 2f629db3d43619cfdda58a4fa7818015e5e865b4 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Thu, 27 Aug 2020 11:59:57 -0400 Subject: [PATCH] Refactor Zstd dump styles --- src/COMPRESS/dump_atom_zstd.cpp | 131 +++++------------ src/COMPRESS/dump_atom_zstd.h | 19 +-- src/COMPRESS/dump_custom_zstd.cpp | 133 +++++------------ src/COMPRESS/dump_custom_zstd.h | 18 +-- src/COMPRESS/zstd_file_writer.cpp | 158 +++++++++++++++++++++ src/COMPRESS/zstd_file_writer.h | 52 +++++++ src/file_writer.h | 53 +++++++ unittest/formats/test_dump_atom_gz.cpp | 44 ++++-- unittest/formats/test_dump_atom_zstd.cpp | 30 ++-- unittest/formats/test_dump_custom_gz.cpp | 12 +- unittest/formats/test_dump_custom_zstd.cpp | 12 +- 11 files changed, 400 insertions(+), 262 deletions(-) create mode 100644 src/COMPRESS/zstd_file_writer.cpp create mode 100644 src/COMPRESS/zstd_file_writer.h create mode 100644 src/file_writer.h diff --git a/src/COMPRESS/dump_atom_zstd.cpp b/src/COMPRESS/dump_atom_zstd.cpp index 427aede016..30862ab854 100644 --- a/src/COMPRESS/dump_atom_zstd.cpp +++ b/src/COMPRESS/dump_atom_zstd.cpp @@ -11,29 +11,24 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + #include "dump_atom_zstd.h" #include "domain.h" #include "error.h" #include "update.h" #include "force.h" -#include #include +#include using namespace LAMMPS_NS; DumpAtomZstd::DumpAtomZstd(LAMMPS *lmp, int narg, char **arg) : DumpAtom(lmp, narg, arg) { - cctx = nullptr; - zstdFp = nullptr; - fp = nullptr; - out_buffer_size = ZSTD_CStreamOutSize(); - out_buffer = new char[out_buffer_size]; - - checksum_flag = 1; - compression_level = 0; // = default - if (!compressed) error->all(FLERR,"Dump atom/zstd only writes compressed files"); } @@ -42,11 +37,6 @@ DumpAtomZstd::DumpAtomZstd(LAMMPS *lmp, int narg, char **arg) : DumpAtomZstd::~DumpAtomZstd() { - if(cctx && zstdFp) zstd_close(); - - delete [] out_buffer; - out_buffer = nullptr; - out_buffer_size = 0; } /* ---------------------------------------------------------------------- @@ -101,19 +91,15 @@ void DumpAtomZstd::openfile() if (filewriter) { if (append_flag) { - zstdFp = fopen(filecurrent,"ab"); - } else { - zstdFp = fopen(filecurrent,"wb"); + error->one(FLERR, "dump/zstd currently doesn't support append"); } - if (zstdFp == nullptr) error->one(FLERR,"Cannot open dump file"); - - cctx = ZSTD_createCCtx(); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compression_level); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, checksum_flag); - - if (cctx == nullptr) error->one(FLERR,"Cannot create Zstd context"); - } else zstdFp = nullptr; + try { + writer.open(filecurrent); + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); + } + } // delete string with timestep replaced @@ -151,7 +137,7 @@ void DumpAtomZstd::write_header(bigint ndump) } header += fmt::format("ITEM: ATOMS {}\n", columns); - zstd_write(header.c_str(), header.length()); + writer.write(header.c_str(), header.length()); } } @@ -159,14 +145,7 @@ void DumpAtomZstd::write_header(bigint ndump) void DumpAtomZstd::write_data(int n, double *mybuf) { - ZSTD_inBuffer input = { mybuf, (size_t)n, 0 }; - ZSTD_EndDirective mode = ZSTD_e_continue; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(input.pos < input.size); + writer.write(mybuf, n); } /* ---------------------------------------------------------------------- */ @@ -176,11 +155,10 @@ void DumpAtomZstd::write() DumpAtom::write(); if (filewriter) { if (multifile) { - zstd_close(); + writer.close(); } else { - if (flush_flag && zstdFp) { - zstd_flush(); - fflush(zstdFp); + if (flush_flag && writer.isopen()) { + writer.flush(); } } } @@ -192,67 +170,22 @@ int DumpAtomZstd::modify_param(int narg, char **arg) { int consumed = DumpAtom::modify_param(narg, arg); if(consumed == 0) { - if (strcmp(arg[0],"checksum") == 0) { - if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - if (strcmp(arg[1],"yes") == 0) checksum_flag = 1; - else if (strcmp(arg[1],"no") == 0) checksum_flag = 0; - else error->all(FLERR,"Illegal dump_modify command"); - return 2; - } else if (strcmp(arg[0],"compression_level") == 0) { - if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - compression_level = force->inumeric(FLERR,arg[1]); - int min_level = ZSTD_minCLevel(); - int max_level = ZSTD_maxCLevel(); - if (compression_level < min_level || compression_level > max_level) - error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); - return 2; + try { + if (strcmp(arg[0],"checksum") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + if (strcmp(arg[1],"yes") == 0) writer.setChecksum(true); + else if (strcmp(arg[1],"no") == 0) writer.setChecksum(false); + else error->all(FLERR,"Illegal dump_modify command"); + return 2; + } else if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + int compression_level = force->inumeric(FLERR,arg[1]); + writer.setCompressionLevel(compression_level); + return 2; + } + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); } } return consumed; } - -/* ---------------------------------------------------------------------- */ - -void DumpAtomZstd::zstd_write(const void * buffer, size_t length) -{ - ZSTD_inBuffer input = { buffer, length, 0 }; - ZSTD_EndDirective mode = ZSTD_e_continue; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(input.pos < input.size); -} - -void DumpAtomZstd::zstd_flush() { - size_t remaining; - ZSTD_inBuffer input = { nullptr, 0, 0 }; - ZSTD_EndDirective mode = ZSTD_e_flush; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(remaining); -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomZstd::zstd_close() -{ - size_t remaining; - ZSTD_inBuffer input = { nullptr, 0, 0 }; - ZSTD_EndDirective mode = ZSTD_e_end; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(remaining); - - ZSTD_freeCCtx(cctx); - cctx = nullptr; - if (zstdFp) fclose(zstdFp); - zstdFp = nullptr; -} diff --git a/src/COMPRESS/dump_atom_zstd.h b/src/COMPRESS/dump_atom_zstd.h index feb1cc912c..d1bfd9d00f 100644 --- a/src/COMPRESS/dump_atom_zstd.h +++ b/src/COMPRESS/dump_atom_zstd.h @@ -11,6 +11,10 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + #ifdef DUMP_CLASS DumpStyle(atom/zstd,DumpAtomZstd) @@ -21,8 +25,7 @@ DumpStyle(atom/zstd,DumpAtomZstd) #define LMP_DUMP_ATOM_ZSTD_H #include "dump_atom.h" -#include -#include +#include "zstd_file_writer.h" namespace LAMMPS_NS { @@ -32,13 +35,7 @@ class DumpAtomZstd : public DumpAtom { virtual ~DumpAtomZstd(); protected: - int compression_level; - int checksum_flag; - - ZSTD_CCtx * cctx; - FILE * zstdFp; - char * out_buffer; - size_t out_buffer_size; + ZstdFileWriter writer; virtual void openfile(); virtual void write_header(bigint); @@ -46,10 +43,6 @@ class DumpAtomZstd : public DumpAtom { virtual void write(); virtual int modify_param(int, char **); - - void zstd_write(const void * buffer, size_t length); - void zstd_flush(); - void zstd_close(); }; } diff --git a/src/COMPRESS/dump_custom_zstd.cpp b/src/COMPRESS/dump_custom_zstd.cpp index 16794bcafd..420425b050 100644 --- a/src/COMPRESS/dump_custom_zstd.cpp +++ b/src/COMPRESS/dump_custom_zstd.cpp @@ -11,6 +11,10 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + #include "dump_custom_zstd.h" #include "domain.h" #include "error.h" @@ -25,32 +29,16 @@ using namespace LAMMPS_NS; DumpCustomZstd::DumpCustomZstd(LAMMPS *lmp, int narg, char **arg) : DumpCustom(lmp, narg, arg) { - cctx = nullptr; - zstdFp = nullptr; - fp = nullptr; - out_buffer_size = ZSTD_CStreamOutSize(); - out_buffer = new char[out_buffer_size]; - - checksum_flag = 1; - compression_level = 0; // = default - if (!compressed) error->all(FLERR,"Dump custom/zstd only writes compressed files"); } - /* ---------------------------------------------------------------------- */ DumpCustomZstd::~DumpCustomZstd() { - if(cctx && zstdFp) zstd_close(); - - delete [] out_buffer; - out_buffer = nullptr; - out_buffer_size = 0; } - /* ---------------------------------------------------------------------- generic opening of a dump file ASCII or binary or gzipped @@ -103,25 +91,23 @@ void DumpCustomZstd::openfile() if (filewriter) { if (append_flag) { - zstdFp = fopen(filecurrent,"ab"); - } else { - zstdFp = fopen(filecurrent,"wb"); + error->one(FLERR, "dump/zstd currently doesn't support append"); } - if (zstdFp == nullptr) error->one(FLERR,"Cannot open dump file"); - - cctx = ZSTD_createCCtx(); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compression_level); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, checksum_flag); - - if (cctx == nullptr) error->one(FLERR,"Cannot create Zstd context"); - } else zstdFp = nullptr; + try { + writer.open(filecurrent); + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); + } + } // delete string with timestep replaced if (multifile) delete [] filecurrent; } +/* ---------------------------------------------------------------------- */ + void DumpCustomZstd::write_header(bigint ndump) { std::string header; @@ -151,7 +137,7 @@ void DumpCustomZstd::write_header(bigint ndump) } header += fmt::format("ITEM: ATOMS {}\n", columns); - zstd_write(header.c_str(), header.length()); + writer.write(header.c_str(), header.length()); } } @@ -159,14 +145,7 @@ void DumpCustomZstd::write_header(bigint ndump) void DumpCustomZstd::write_data(int n, double *mybuf) { - ZSTD_inBuffer input = { mybuf, (size_t)n, 0 }; - ZSTD_EndDirective mode = ZSTD_e_continue; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(input.pos < input.size); + writer.write(mybuf, n); } /* ---------------------------------------------------------------------- */ @@ -176,11 +155,10 @@ void DumpCustomZstd::write() DumpCustom::write(); if (filewriter) { if (multifile) { - zstd_close(); + writer.close(); } else { - if (flush_flag && zstdFp) { - zstd_flush(); - fflush(zstdFp); + if (flush_flag && writer.isopen()) { + writer.flush(); } } } @@ -192,67 +170,22 @@ int DumpCustomZstd::modify_param(int narg, char **arg) { int consumed = DumpCustom::modify_param(narg, arg); if(consumed == 0) { - if (strcmp(arg[0],"checksum") == 0) { - if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - if (strcmp(arg[1],"yes") == 0) checksum_flag = 1; - else if (strcmp(arg[1],"no") == 0) checksum_flag = 0; - else error->all(FLERR,"Illegal dump_modify command"); - return 2; - } else if (strcmp(arg[0],"compression_level") == 0) { - if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - compression_level = force->inumeric(FLERR,arg[1]); - int min_level = ZSTD_minCLevel(); - int max_level = ZSTD_maxCLevel(); - if (compression_level < min_level || compression_level > max_level) - error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); - return 2; + try { + if (strcmp(arg[0],"checksum") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + if (strcmp(arg[1],"yes") == 0) writer.setChecksum(true); + else if (strcmp(arg[1],"no") == 0) writer.setChecksum(false); + else error->all(FLERR,"Illegal dump_modify command"); + return 2; + } else if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + int compression_level = force->inumeric(FLERR,arg[1]); + writer.setCompressionLevel(compression_level); + return 2; + } + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); } } return consumed; } - -/* ---------------------------------------------------------------------- */ - -void DumpCustomZstd::zstd_write(const void * buffer, size_t length) -{ - ZSTD_inBuffer input = { buffer, length, 0 }; - ZSTD_EndDirective mode = ZSTD_e_continue; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(input.pos < input.size); -} - -void DumpCustomZstd::zstd_flush() { - size_t remaining; - ZSTD_inBuffer input = { nullptr, 0, 0 }; - ZSTD_EndDirective mode = ZSTD_e_flush; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(remaining); -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomZstd::zstd_close() -{ - size_t remaining; - ZSTD_inBuffer input = { nullptr, 0, 0 }; - ZSTD_EndDirective mode = ZSTD_e_end; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(remaining); - - ZSTD_freeCCtx(cctx); - cctx = nullptr; - if (zstdFp) fclose(zstdFp); - zstdFp = nullptr; -} diff --git a/src/COMPRESS/dump_custom_zstd.h b/src/COMPRESS/dump_custom_zstd.h index 4736567dec..fbda89d490 100644 --- a/src/COMPRESS/dump_custom_zstd.h +++ b/src/COMPRESS/dump_custom_zstd.h @@ -11,6 +11,10 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + #ifdef DUMP_CLASS DumpStyle(custom/zstd,DumpCustomZstd) @@ -21,7 +25,7 @@ DumpStyle(custom/zstd,DumpCustomZstd) #define LMP_DUMP_CUSTOM_ZSTD_H #include "dump_custom.h" -#include +#include "zstd_file_writer.h" #include namespace LAMMPS_NS { @@ -32,13 +36,7 @@ class DumpCustomZstd : public DumpCustom { virtual ~DumpCustomZstd(); protected: - int compression_level; - int checksum_flag; - - ZSTD_CCtx * cctx; - FILE * zstdFp; - char * out_buffer; - size_t out_buffer_size; + ZstdFileWriter writer; virtual void openfile(); virtual void write_header(bigint); @@ -46,10 +44,6 @@ class DumpCustomZstd : public DumpCustom { virtual void write(); virtual int modify_param(int, char **); - - void zstd_write(const void * buffer, size_t length); - void zstd_flush(); - void zstd_close(); }; } diff --git a/src/COMPRESS/zstd_file_writer.cpp b/src/COMPRESS/zstd_file_writer.cpp new file mode 100644 index 0000000000..eb63d4e767 --- /dev/null +++ b/src/COMPRESS/zstd_file_writer.cpp @@ -0,0 +1,158 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + +#include "zstd_file_writer.h" +#include +#include + +using namespace LAMMPS_NS; + +ZstdFileWriter::ZstdFileWriter() : FileWriter(), + fp(nullptr), + cctx(nullptr), + compression_level(0), + checksum_flag(1) +{ + out_buffer_size = ZSTD_CStreamOutSize(); + out_buffer = new char[out_buffer_size]; +} + +/* ---------------------------------------------------------------------- */ + +ZstdFileWriter::~ZstdFileWriter() +{ + close(); + + delete [] out_buffer; + out_buffer = nullptr; + out_buffer_size = 0; +} + +/* ---------------------------------------------------------------------- */ + +void ZstdFileWriter::open(const std::string & path) +{ + if(isopen()) return; + + fp = fopen(path.c_str(), "wb"); + + if (!fp) { + throw FileWriterException(fmt::format("Could not open file '{}'", path)); + } + + cctx = ZSTD_createCCtx(); + + if (!cctx) { + fclose(fp); + fp = nullptr; + throw FileWriterException("Could not create Zstd context"); + } + + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compression_level); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, checksum_flag); +} + +/* ---------------------------------------------------------------------- */ + +size_t ZstdFileWriter::write(const void * buffer, size_t length) +{ + if(!isopen()) return 0; + + ZSTD_inBuffer input = { buffer, length, 0 }; + ZSTD_EndDirective mode = ZSTD_e_continue; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, fp); + } while(input.pos < input.size); + + return length; +} + +/* ---------------------------------------------------------------------- */ + +void ZstdFileWriter::flush() +{ + if(!isopen()) return; + + size_t remaining; + ZSTD_inBuffer input = { nullptr, 0, 0 }; + ZSTD_EndDirective mode = ZSTD_e_flush; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, fp); + } while(remaining); + + fflush(fp); +} + +/* ---------------------------------------------------------------------- */ + +void ZstdFileWriter::close() +{ + if(!isopen()) return; + + size_t remaining; + ZSTD_inBuffer input = { nullptr, 0, 0 }; + ZSTD_EndDirective mode = ZSTD_e_end; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, fp); + } while(remaining); + + ZSTD_freeCCtx(cctx); + cctx = nullptr; + fclose(fp); + fp = nullptr; +} + +/* ---------------------------------------------------------------------- */ + +bool ZstdFileWriter::isopen() const +{ + return fp && cctx; +} + +/* ---------------------------------------------------------------------- */ + +void ZstdFileWriter::setCompressionLevel(int level) +{ + if (isopen()) + throw FileWriterException("Compression level can not be changed while file is open"); + + const int min_level = ZSTD_minCLevel(); + const int max_level = ZSTD_maxCLevel(); + + if(level < min_level || level > max_level) + throw FileWriterException(fmt::format("Compression level must in the range of [{}, {}]", min_level, max_level)); + + compression_level = level; +} + +/* ---------------------------------------------------------------------- */ + +void ZstdFileWriter::setChecksum(bool enabled) +{ + if (isopen()) + throw FileWriterException("Checksum flag can not be changed while file is open"); + checksum_flag = enabled ? 1 : 0; +} diff --git a/src/COMPRESS/zstd_file_writer.h b/src/COMPRESS/zstd_file_writer.h new file mode 100644 index 0000000000..8a974a8599 --- /dev/null +++ b/src/COMPRESS/zstd_file_writer.h @@ -0,0 +1,52 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + +#ifndef LMP_ZSTD_FILE_WRITER_H +#define LMP_ZSTD_FILE_WRITER_H + +#include "file_writer.h" +#include +#include +#include + +namespace LAMMPS_NS { + +class ZstdFileWriter : public FileWriter { + int compression_level; + int checksum_flag; + + ZSTD_CCtx * cctx; + FILE * fp; + char * out_buffer; + size_t out_buffer_size; +public: + ZstdFileWriter(); + virtual ~ZstdFileWriter(); + virtual void open(const std::string & path) override; + virtual void close() override; + virtual void flush() override; + virtual size_t write(const void * buffer, size_t length) override; + virtual bool isopen() const override; + + void setCompressionLevel(int level); + void setChecksum(bool enabled); +}; + + +} + +#endif diff --git a/src/file_writer.h b/src/file_writer.h new file mode 100644 index 0000000000..766ccc85e7 --- /dev/null +++ b/src/file_writer.h @@ -0,0 +1,53 @@ + +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + +#ifndef LMP_FILE_WRITER_H +#define LMP_FILE_WRITER_H + +#include + +namespace LAMMPS_NS { + +class FileWriter { +public: + FileWriter() = default; + virtual ~FileWriter() = default; + virtual void open(const std::string & path) = 0; + virtual void close() = 0; + virtual void flush() = 0; + virtual size_t write(const void * buffer, size_t length) = 0; + virtual bool isopen() const = 0; +}; + +class FileWriterException : public std::exception { + std::string message; +public: + FileWriterException(const std::string & msg) : message(msg) { + } + + ~FileWriterException() throw() { + } + + virtual const char * what() const throw() { + return message.c_str(); + } +}; + +} + +#endif diff --git a/unittest/formats/test_dump_atom_gz.cpp b/unittest/formats/test_dump_atom_gz.cpp index 438098ce4c..ab1f57cc1b 100644 --- a/unittest/formats/test_dump_atom_gz.cpp +++ b/unittest/formats/test_dump_atom_gz.cpp @@ -79,17 +79,19 @@ TEST_F(DumpAtomGZTest, compressed_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_run0.melt"; - auto compressed_file = "dump_compressed_run0.melt.gz"; + auto text_file = "dump_gz_text_run0.melt"; + auto compressed_file = "dump_gz_compressed_run0.melt.gz"; generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); auto converted_file = convert_compressed_to_text(compressed_file); - ASSERT_THAT(converted_file, Eq("dump_compressed_run0.melt")); + ASSERT_THAT(converted_file, Eq("dump_gz_compressed_run0.melt")); ASSERT_FILE_EXISTS(converted_file); ASSERT_FILE_EQUAL(text_file, converted_file); delete_file(text_file); @@ -101,11 +103,13 @@ TEST_F(DumpAtomGZTest, compressed_with_units_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_with_units_run0.melt"; - auto compressed_file = "dump_compressed_with_units_run0.melt.gz"; + auto text_file = "dump_gz_text_with_units_run0.melt"; + auto compressed_file = "dump_gz_compressed_with_units_run0.melt.gz"; generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "scale no units yes", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -122,11 +126,13 @@ TEST_F(DumpAtomGZTest, compressed_with_time_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_with_time_run0.melt"; - auto compressed_file = "dump_compressed_with_time_run0.melt.gz"; + auto text_file = "dump_gz_text_with_time_run0.melt"; + auto compressed_file = "dump_gz_compressed_with_time_run0.melt.gz"; generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "scale no time yes", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -143,12 +149,14 @@ TEST_F(DumpAtomGZTest, compressed_triclinic_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_run0.melt"; - auto compressed_file = "dump_compressed_tri_run0.melt.gz"; + auto text_file = "dump_gz_text_tri_run0.melt"; + auto compressed_file = "dump_gz_compressed_tri_run0.melt.gz"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -165,12 +173,14 @@ TEST_F(DumpAtomGZTest, compressed_triclinic_with_units_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_with_units_run0.melt"; - auto compressed_file = "dump_compressed_tri_with_units_run0.melt.gz"; + auto text_file = "dump_gz_text_tri_with_units_run0.melt"; + auto compressed_file = "dump_gz_compressed_tri_with_units_run0.melt.gz"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "scale no units yes", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -187,12 +197,14 @@ TEST_F(DumpAtomGZTest, compressed_triclinic_with_time_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_with_time_run0.melt"; - auto compressed_file = "dump_compressed_tri_with_time_run0.melt.gz"; + auto text_file = "dump_gz_text_tri_with_time_run0.melt"; + auto compressed_file = "dump_gz_compressed_tri_with_time_run0.melt.gz"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "scale no time yes", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -209,12 +221,14 @@ TEST_F(DumpAtomGZTest, compressed_triclinic_with_image_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_with_image_run0.melt"; - auto compressed_file = "dump_compressed_tri_with_image_run0.melt.gz"; + auto text_file = "dump_gz_text_tri_with_image_run0.melt"; + auto compressed_file = "dump_gz_compressed_tri_with_image_run0.melt.gz"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "image yes", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); diff --git a/unittest/formats/test_dump_atom_zstd.cpp b/unittest/formats/test_dump_atom_zstd.cpp index 470634a185..d588a803a6 100644 --- a/unittest/formats/test_dump_atom_zstd.cpp +++ b/unittest/formats/test_dump_atom_zstd.cpp @@ -79,8 +79,8 @@ TEST_F(DumpAtomZSTDTest, compressed_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_run0.melt"; - auto compressed_file = "dump_compressed_run0.melt.zst"; + auto text_file = "dump_zstd_text_run0.melt"; + auto compressed_file = "dump_zstd_compressed_run0.melt.zst"; generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "", 0); @@ -92,7 +92,7 @@ TEST_F(DumpAtomZSTDTest, compressed_run0) auto converted_file = convert_compressed_to_text(compressed_file); - ASSERT_THAT(converted_file, Eq("dump_compressed_run0.melt")); + ASSERT_THAT(converted_file, Eq("dump_zstd_compressed_run0.melt")); ASSERT_FILE_EXISTS(converted_file); ASSERT_FILE_EQUAL(text_file, converted_file); delete_file(text_file); @@ -104,8 +104,8 @@ TEST_F(DumpAtomZSTDTest, compressed_with_units_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_with_units_run0.melt"; - auto compressed_file = "dump_compressed_with_units_run0.melt.zst"; + auto text_file = "dump_zstd_text_with_units_run0.melt"; + auto compressed_file = "dump_zstd_compressed_with_units_run0.melt.zst"; generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "scale no units yes", 0); @@ -128,8 +128,8 @@ TEST_F(DumpAtomZSTDTest, compressed_with_time_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_with_time_run0.melt"; - auto compressed_file = "dump_compressed_with_time_run0.melt.zst"; + auto text_file = "dump_zstd_text_with_time_run0.melt"; + auto compressed_file = "dump_zstd_compressed_with_time_run0.melt.zst"; generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "scale no time yes", 0); @@ -152,8 +152,8 @@ TEST_F(DumpAtomZSTDTest, compressed_triclinic_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_run0.melt"; - auto compressed_file = "dump_compressed_tri_run0.melt.zst"; + auto text_file = "dump_zstd_text_tri_run0.melt"; + auto compressed_file = "dump_zstd_compressed_tri_run0.melt.zst"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "", 0); @@ -177,8 +177,8 @@ TEST_F(DumpAtomZSTDTest, compressed_triclinic_with_units_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_with_units_run0.melt"; - auto compressed_file = "dump_compressed_tri_with_units_run0.melt.zst"; + auto text_file = "dump_zstd_text_tri_with_units_run0.melt"; + auto compressed_file = "dump_zstd_compressed_tri_with_units_run0.melt.zst"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "scale no units yes", 0); @@ -202,8 +202,8 @@ TEST_F(DumpAtomZSTDTest, compressed_triclinic_with_time_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_with_time_run0.melt"; - auto compressed_file = "dump_compressed_tri_with_time_run0.melt.zst"; + auto text_file = "dump_zstd_text_tri_with_time_run0.melt"; + auto compressed_file = "dump_zstd_compressed_tri_with_time_run0.melt.zst"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "scale no time yes", 0); @@ -227,8 +227,8 @@ TEST_F(DumpAtomZSTDTest, compressed_triclinic_with_image_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_with_image_run0.melt"; - auto compressed_file = "dump_compressed_tri_with_image_run0.melt.zst"; + auto text_file = "dump_zstd_text_tri_with_image_run0.melt"; + auto compressed_file = "dump_zstd_compressed_tri_with_image_run0.melt.zst"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "image yes", 0); diff --git a/unittest/formats/test_dump_custom_gz.cpp b/unittest/formats/test_dump_custom_gz.cpp index c0be5f76c8..12e1eca289 100644 --- a/unittest/formats/test_dump_custom_gz.cpp +++ b/unittest/formats/test_dump_custom_gz.cpp @@ -73,12 +73,14 @@ TEST_F(DumpCustomGZTest, compressed_run1) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_custom_text_run1.melt"; - auto compressed_file = "dump_custom_compressed_run1.melt.gz"; + auto text_file = "dump_custom_gz_text_run1.melt"; + auto compressed_file = "dump_custom_gz_compressed_run1.melt.gz"; auto fields = "id type proc x y z ix iy iz xs ys zs xu yu zu xsu ysu zsu vx vy vz fx fy fz"; generate_text_and_compressed_dump(text_file, compressed_file, "custom/gz", fields, "units yes", 1); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -95,14 +97,16 @@ TEST_F(DumpCustomGZTest, compressed_triclinic_run1) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_custom_tri_text_run1.melt"; - auto compressed_file = "dump_custom_tri_compressed_run1.melt.gz"; + auto text_file = "dump_custom_gz_tri_text_run1.melt"; + auto compressed_file = "dump_custom_gz_tri_compressed_run1.melt.gz"; auto fields = "id type proc x y z xs ys zs xsu ysu zsu vx vy vz fx fy fz"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "custom/gz", fields, "units yes", 1); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); diff --git a/unittest/formats/test_dump_custom_zstd.cpp b/unittest/formats/test_dump_custom_zstd.cpp index 1b156b3f20..69576c9584 100644 --- a/unittest/formats/test_dump_custom_zstd.cpp +++ b/unittest/formats/test_dump_custom_zstd.cpp @@ -73,12 +73,14 @@ TEST_F(DumpCustomZstdTest, compressed_run1) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_custom_text_run1.melt"; - auto compressed_file = "dump_custom_compressed_run1.melt.zst"; + auto text_file = "dump_custom_zstd_text_run1.melt"; + auto compressed_file = "dump_custom_zstd_compressed_run1.melt.zst"; auto fields = "id type proc x y z ix iy iz xs ys zs xu yu zu xsu ysu zsu vx vy vz fx fy fz"; generate_text_and_compressed_dump(text_file, compressed_file, "custom/zstd", fields, "units yes", 1); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -95,14 +97,16 @@ TEST_F(DumpCustomZstdTest, compressed_triclinic_run1) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_custom_tri_text_run1.melt"; - auto compressed_file = "dump_custom_tri_compressed_run1.melt.zst"; + auto text_file = "dump_custom_zstd_tri_text_run1.melt"; + auto compressed_file = "dump_custom_zstd_tri_compressed_run1.melt.zst"; auto fields = "id type proc x y z xs ys zs xsu ysu zsu vx vy vz fx fy fz"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "custom/zstd", fields, "units yes", 1); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file);