Update dump local and local/gz

This commit is contained in:
Richard Berger 2021-04-09 15:42:28 -04:00
parent d19cd8fb11
commit cf41ea6faf
No known key found for this signature in database
GPG Key ID: A9E83994E0BA0CAB
5 changed files with 285 additions and 70 deletions

View File

@ -11,24 +11,18 @@
See the README file in the top-level LAMMPS directory.
------------------------------------------------------------------------- */
#include "dump_local_gz.h"
#include "domain.h"
#include "dump_local_gz.h"
#include "error.h"
#include "update.h"
#include <cstring>
using namespace LAMMPS_NS;
DumpLocalGZ::DumpLocalGZ(LAMMPS *lmp, int narg, char **arg) :
DumpLocal(lmp, narg, arg)
{
gzFp = nullptr;
compression_level = Z_BEST_COMPRESSION;
if (!compressed)
error->all(FLERR,"Dump local/gz only writes compressed files");
}
@ -38,12 +32,8 @@ DumpLocalGZ::DumpLocalGZ(LAMMPS *lmp, int narg, char **arg) :
DumpLocalGZ::~DumpLocalGZ()
{
if (gzFp) gzclose(gzFp);
gzFp = nullptr;
fp = nullptr;
}
/* ----------------------------------------------------------------------
generic opening of a dump file
ASCII or binary or gzipped
@ -93,17 +83,12 @@ void DumpLocalGZ::openfile()
// each proc with filewriter = 1 opens a file
if (filewriter) {
std::string mode;
if (append_flag) {
mode = fmt::format("ab{}", compression_level);
} else {
mode = fmt::format("wb{}", compression_level);
try {
writer.open(filecurrent, append_flag);
} catch (FileWriterException &e) {
error->one(FLERR, e.what());
}
gzFp = gzopen(filecurrent, mode.c_str());
if (gzFp == nullptr) error->one(FLERR,"Cannot open dump file");
} else gzFp = nullptr;
}
// delete string with timestep replaced
@ -112,29 +97,34 @@ void DumpLocalGZ::openfile()
void DumpLocalGZ::write_header(bigint ndump)
{
std::string header;
if ((multiproc) || (!multiproc && me == 0)) {
if (unit_flag && !unit_count) {
++unit_count;
gzprintf(gzFp,"ITEM: UNITS\n%s\n",update->unit_style);
header = fmt::format("ITEM: UNITS\n{}\n",update->unit_style);
}
if (time_flag) gzprintf(gzFp,"ITEM: TIME\n%.16g\n",compute_time());
gzprintf(gzFp,"ITEM: TIMESTEP\n");
gzprintf(gzFp,BIGINT_FORMAT "\n",update->ntimestep);
gzprintf(gzFp,"ITEM: NUMBER OF %s\n",label);
gzprintf(gzFp,BIGINT_FORMAT "\n",ndump);
if (domain->triclinic) {
gzprintf(gzFp,"ITEM: BOX BOUNDS xy xz yz %s\n",boundstr);
gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxxlo,boxxhi,boxxy);
gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxylo,boxyhi,boxxz);
gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxzlo,boxzhi,boxyz);
} else {
gzprintf(gzFp,"ITEM: BOX BOUNDS %s\n",boundstr);
gzprintf(gzFp,"%-1.16e %-1.16e\n",boxxlo,boxxhi);
gzprintf(gzFp,"%-1.16e %-1.16e\n",boxylo,boxyhi);
gzprintf(gzFp,"%-1.16e %-1.16e\n",boxzlo,boxzhi);
if (time_flag) {
header += fmt::format("ITEM: TIME\n{0:.16g}\n", compute_time());
}
gzprintf(gzFp,"ITEM: %s %s\n",label,columns);
header += fmt::format("ITEM: TIMESTEP\n{}\n", update->ntimestep);
header += fmt::format("ITEM: NUMBER OF {}\n{}\n", label, ndump);
if (domain->triclinic == 0) {
header += fmt::format("ITEM: BOX BOUNDS {}\n", boundstr);
header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxxlo, boxxhi);
header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxylo, boxyhi);
header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxzlo, boxzhi);
} else {
header += fmt::format("ITEM: BOX BOUNDS xy xz yz {}\n", boundstr);
header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxxlo, boxxhi, boxxy);
header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxylo, boxyhi, boxxz);
header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxzlo, boxzhi, boxyz);
}
header += fmt::format("ITEM: {} {}\n", label, columns);
writer.write(header.c_str(), header.length());
}
}
@ -143,19 +133,28 @@ void DumpLocalGZ::write_header(bigint ndump)
void DumpLocalGZ::write_data(int n, double *mybuf)
{
if (buffer_flag == 1) {
gzwrite(gzFp,mybuf,sizeof(char)*n);
writer.write(mybuf, sizeof(char)*n);
} else {
int i,j;
constexpr size_t VBUFFER_SIZE = 256;
char vbuffer[VBUFFER_SIZE];
int m = 0;
for (i = 0; i < n; i++) {
for (j = 0; j < size_one; j++) {
if (vtype[j] == INT)
gzprintf(gzFp,vformat[j],static_cast<int> (mybuf[m]));
else gzprintf(gzFp,vformat[j],mybuf[m]);
for (int i = 0; i < n; i++) {
for (int j = 0; j < size_one; j++) {
int written = 0;
if (vtype[j] == Dump::INT) {
written = snprintf(vbuffer, VBUFFER_SIZE, vformat[j], static_cast<int> (mybuf[m]));
} else {
written = snprintf(vbuffer, VBUFFER_SIZE, vformat[j], mybuf[m]);
}
if (written > 0) {
writer.write(vbuffer, written);
} else if (written < 0) {
error->one(FLERR, "Error while writing dump local/gz output");
}
m++;
}
gzprintf(gzFp,"\n");
writer.write("\n", 1);
}
}
}
@ -167,11 +166,11 @@ void DumpLocalGZ::write()
DumpLocal::write();
if (filewriter) {
if (multifile) {
gzclose(gzFp);
gzFp = nullptr;
writer.close();
} else {
if (flush_flag)
gzflush(gzFp,Z_SYNC_FLUSH);
if (flush_flag && writer.isopen()) {
writer.flush();
}
}
}
}
@ -182,14 +181,15 @@ int DumpLocalGZ::modify_param(int narg, char **arg)
{
int consumed = DumpLocal::modify_param(narg, arg);
if (consumed == 0) {
if (strcmp(arg[0],"compression_level") == 0) {
if (narg < 2) error->all(FLERR,"Illegal dump_modify command");
int min_level = Z_DEFAULT_COMPRESSION;
int max_level = Z_BEST_COMPRESSION;
compression_level = utils::inumeric(FLERR, arg[1], false, lmp);
if (compression_level < min_level || compression_level > max_level)
error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level));
return 2;
try {
if (strcmp(arg[0],"compression_level") == 0) {
if (narg < 2) error->all(FLERR,"Illegal dump_modify command");
int compression_level = utils::inumeric(FLERR, arg[1], false, lmp);
writer.setCompressionLevel(compression_level);
return 2;
}
} catch (FileWriterException &e) {
error->one(FLERR, fmt::format("Illegal dump_modify command: {}", e.what()));
}
}
return consumed;

View File

@ -21,7 +21,7 @@ DumpStyle(local/gz,DumpLocalGZ)
#define LMP_DUMP_LOCAL_GZ_H
#include "dump_local.h"
#include <zlib.h>
#include "gz_file_writer.h"
namespace LAMMPS_NS {
@ -31,8 +31,7 @@ class DumpLocalGZ : public DumpLocal {
virtual ~DumpLocalGZ();
protected:
int compression_level;
gzFile gzFp; // file pointer for the compressed output stream
GzFileWriter writer;
virtual void openfile();
virtual void write_header(bigint);

View File

@ -17,15 +17,13 @@
#ifdef LAMMPS_ZSTD
#include "dump_local_zstd.h"
#include "domain.h"
#include "dump_local_zstd.h"
#include "error.h"
#include "update.h"
#include <cstring>
using namespace LAMMPS_NS;
DumpLocalZstd::DumpLocalZstd(LAMMPS *lmp, int narg, char **arg) :
@ -42,7 +40,6 @@ DumpLocalZstd::~DumpLocalZstd()
{
}
/* ----------------------------------------------------------------------
generic opening of a dump file
ASCII or binary or gzipped
@ -145,7 +142,31 @@ void DumpLocalZstd::write_header(bigint ndump)
void DumpLocalZstd::write_data(int n, double *mybuf)
{
writer.write(mybuf, sizeof(char)*n);
if (buffer_flag == 1) {
writer.write(mybuf, sizeof(char)*n);
} else {
constexpr size_t VBUFFER_SIZE = 256;
char vbuffer[VBUFFER_SIZE];
int m = 0;
for (int i = 0; i < n; i++) {
for (int j = 0; j < size_one; j++) {
int written = 0;
if (vtype[j] == Dump::INT) {
written = snprintf(vbuffer, VBUFFER_SIZE, vformat[j], static_cast<int> (mybuf[m]));
} else {
written = snprintf(vbuffer, VBUFFER_SIZE, vformat[j], mybuf[m]);
}
if (written > 0) {
writer.write(vbuffer, written);
} else if (written < 0) {
error->one(FLERR, "Error while writing dump local/gz output");
}
m++;
}
writer.write("\n", 1);
}
}
}
/* ---------------------------------------------------------------------- */
@ -184,7 +205,7 @@ int DumpLocalZstd::modify_param(int narg, char **arg)
return 2;
}
} catch (FileWriterException &e) {
error->one(FLERR, e.what());
error->one(FLERR, fmt::format("Illegal dump_modify command: {}", e.what()));
}
}
return consumed;

View File

@ -93,6 +93,18 @@ TEST_F(DumpLocalTest, run0)
delete_file(dump_file);
}
TEST_F(DumpLocalTest, label_run0)
{
auto dump_file = "dump_local_label_run0.melt";
generate_dump(dump_file, "index c_comp[1]", "label ELEMENTS", 0);
ASSERT_FILE_EXISTS(dump_file);
auto lines = read_lines(dump_file);
ASSERT_THAT(lines[2], Eq("ITEM: NUMBER OF ELEMENTS"));
ASSERT_THAT(lines[8], Eq("ITEM: ELEMENTS index c_comp[1] "));
delete_file(dump_file);
}
TEST_F(DumpLocalTest, format_line_run0)
{
auto dump_file = "dump_local_format_line_run0.melt";
@ -170,6 +182,60 @@ TEST_F(DumpLocalTest, no_buffer_run0)
delete_file(dump_file);
}
TEST_F(DumpLocalTest, with_units_run0)
{
auto dump_file = "dump_with_units_run0.melt";
generate_dump(dump_file, "index c_comp[1]", "units yes", 0);
ASSERT_FILE_EXISTS(dump_file);
auto lines = read_lines(dump_file);
ASSERT_EQ(lines.size(), 875);
ASSERT_THAT(lines[0], Eq("ITEM: UNITS"));
ASSERT_THAT(lines[1], Eq("lj"));
ASSERT_THAT(lines[2], Eq("ITEM: TIMESTEP"));
ASSERT_EQ(std::stoi(lines[3]), 0);
ASSERT_THAT(lines[4], Eq("ITEM: NUMBER OF ENTRIES"));
ASSERT_EQ(std::stoi(lines[5]), 864);
}
TEST_F(DumpLocalTest, with_time_run0)
{
auto dump_file = "dump_with_time_run0.melt";
generate_dump(dump_file, "index c_comp[1]", "time yes", 0);
ASSERT_FILE_EXISTS(dump_file);
auto lines = read_lines(dump_file);
ASSERT_EQ(lines.size(), 875);
ASSERT_THAT(lines[0], Eq("ITEM: TIME"));
ASSERT_THAT(std::stof(lines[1]), 0.0);
ASSERT_THAT(lines[2], Eq("ITEM: TIMESTEP"));
ASSERT_EQ(std::stoi(lines[3]), 0);
ASSERT_THAT(lines[4], Eq("ITEM: NUMBER OF ENTRIES"));
ASSERT_EQ(std::stoi(lines[5]), 864);
}
TEST_F(DumpLocalTest, triclinic_run0)
{
auto dump_file = "dump_local_triclinic_run0.melt";
enable_triclinic();
generate_dump(dump_file, "index c_comp[1]", "", 0);
ASSERT_FILE_EXISTS(dump_file);
auto lines = read_lines(dump_file);
ASSERT_THAT(lines[4], Eq("ITEM: BOX BOUNDS xy xz yz pp pp pp"));
ASSERT_EQ(utils::split_words(lines[5]).size(), 3);
ASSERT_EQ(utils::split_words(lines[6]).size(), 3);
ASSERT_EQ(utils::split_words(lines[7]).size(), 3);
delete_file(dump_file);
}
int main(int argc, char **argv)
{
MPI_Init(&argc, &argv);

View File

@ -69,6 +69,135 @@ TEST_F(DumpLocalCompressTest, compressed_run0)
delete_file(converted_file_0);
}
TEST_F(DumpLocalCompressTest, compressed_no_buffer_run0)
{
if (!COMPRESS_BINARY) GTEST_SKIP();
auto base_name = "no_buffer_run*.melt.local";
auto base_name_0 = "no_buffer_run0.melt.local";
auto text_files = text_dump_filename(base_name);
auto compressed_files = compressed_dump_filename(base_name);
auto text_file_0 = text_dump_filename(base_name_0);
auto compressed_file_0 = compressed_dump_filename(base_name_0);
auto fields = "index c_comp[1]";
if(compression_style == "local/zstd") {
generate_text_and_compressed_dump(text_files, compressed_files, fields, fields, "buffer no", "buffer no checksum yes", 0);
} else {
generate_text_and_compressed_dump(text_files, compressed_files, fields, "buffer no", 0);
}
TearDown();
ASSERT_FILE_EXISTS(text_file_0);
ASSERT_FILE_EXISTS(compressed_file_0);
auto converted_file_0 = convert_compressed_to_text(compressed_file_0);
ASSERT_FILE_EXISTS(converted_file_0);
ASSERT_FILE_EQUAL(text_file_0, converted_file_0);
delete_file(text_file_0);
delete_file(compressed_file_0);
delete_file(converted_file_0);
}
TEST_F(DumpLocalCompressTest, compressed_with_time_run0)
{
if (!COMPRESS_BINARY) GTEST_SKIP();
auto base_name = "with_time_run*.melt.local";
auto base_name_0 = "with_time_run0.melt.local";
auto text_files = text_dump_filename(base_name);
auto compressed_files = compressed_dump_filename(base_name);
auto text_file_0 = text_dump_filename(base_name_0);
auto compressed_file_0 = compressed_dump_filename(base_name_0);
auto fields = "index c_comp[1]";
if(compression_style == "local/zstd") {
generate_text_and_compressed_dump(text_files, compressed_files, fields, fields, "time yes", "time yes checksum yes", 0);
} else {
generate_text_and_compressed_dump(text_files, compressed_files, fields, "time yes", 0);
}
TearDown();
ASSERT_FILE_EXISTS(text_file_0);
ASSERT_FILE_EXISTS(compressed_file_0);
auto converted_file_0 = convert_compressed_to_text(compressed_file_0);
ASSERT_FILE_EXISTS(converted_file_0);
ASSERT_FILE_EQUAL(text_file_0, converted_file_0);
delete_file(text_file_0);
delete_file(compressed_file_0);
delete_file(converted_file_0);
}
TEST_F(DumpLocalCompressTest, compressed_with_units_run0)
{
if (!COMPRESS_BINARY) GTEST_SKIP();
auto base_name = "with_units_run*.melt.local";
auto base_name_0 = "with_units_run0.melt.local";
auto text_files = text_dump_filename(base_name);
auto compressed_files = compressed_dump_filename(base_name);
auto text_file_0 = text_dump_filename(base_name_0);
auto compressed_file_0 = compressed_dump_filename(base_name_0);
auto fields = "index c_comp[1]";
if(compression_style == "local/zstd") {
generate_text_and_compressed_dump(text_files, compressed_files, fields, fields, "units yes", "units yes checksum yes", 0);
} else {
generate_text_and_compressed_dump(text_files, compressed_files, fields, "units yes", 0);
}
TearDown();
ASSERT_FILE_EXISTS(text_file_0);
ASSERT_FILE_EXISTS(compressed_file_0);
auto converted_file_0 = convert_compressed_to_text(compressed_file_0);
ASSERT_FILE_EXISTS(converted_file_0);
ASSERT_FILE_EQUAL(text_file_0, converted_file_0);
delete_file(text_file_0);
delete_file(compressed_file_0);
delete_file(converted_file_0);
}
TEST_F(DumpLocalCompressTest, compressed_triclinic_run0)
{
if (!COMPRESS_BINARY) GTEST_SKIP();
enable_triclinic();
auto base_name = "triclinic_run*.melt.local";
auto base_name_0 = "triclinic_run0.melt.local";
auto text_files = text_dump_filename(base_name);
auto compressed_files = compressed_dump_filename(base_name);
auto text_file_0 = text_dump_filename(base_name_0);
auto compressed_file_0 = compressed_dump_filename(base_name_0);
auto fields = "index c_comp[1]";
if(compression_style == "local/zstd") {
generate_text_and_compressed_dump(text_files, compressed_files, fields, fields, "", "checksum yes", 0);
} else {
generate_text_and_compressed_dump(text_files, compressed_files, fields, "", 0);
}
TearDown();
ASSERT_FILE_EXISTS(text_file_0);
ASSERT_FILE_EXISTS(compressed_file_0);
auto converted_file_0 = convert_compressed_to_text(compressed_file_0);
ASSERT_FILE_EXISTS(converted_file_0);
ASSERT_FILE_EQUAL(text_file_0, converted_file_0);
delete_file(text_file_0);
delete_file(compressed_file_0);
delete_file(converted_file_0);
}
TEST_F(DumpLocalCompressTest, compressed_multi_file_run1)
{
if (!COMPRESS_BINARY) GTEST_SKIP();
@ -209,7 +338,7 @@ TEST_F(DumpLocalCompressTest, compressed_modify_bad_param)
command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_dump_filename("modify_bad_param_run0_*.melt.local"), fields));
END_HIDE_OUTPUT();
TEST_FAILURE(".*ERROR: Illegal dump_modify command: compression level must in the range of.*",
TEST_FAILURE(".*ERROR on proc 0: Illegal dump_modify command: Compression level must in the range of.*",
command("dump_modify id1 compression_level 12");
);
}
@ -224,7 +353,7 @@ TEST_F(DumpLocalCompressTest, compressed_modify_multi_bad_param)
command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_dump_filename("modify_multi_bad_param_run0_*.melt.local"), fields));
END_HIDE_OUTPUT();
TEST_FAILURE(".*ERROR: Illegal dump_modify command: compression level must in the range of.*",
TEST_FAILURE(".*ERROR on proc 0: Illegal dump_modify command: Compression level must in the range of.*",
command("dump_modify id1 pad 3 compression_level 12");
);
}