diff --git a/include/HDF_IO.hh b/include/HDF_IO.hh index 7597e10..16946d8 100755 --- a/include/HDF_IO.hh +++ b/include/HDF_IO.hh @@ -887,17 +887,30 @@ inline void HDFWriteDatasetVector( const std::string Filename, const std::string } template< typename T > -inline void HDFCreateEmptyDataset( const std::string Filename, const std::string ObjName, const size_t num_particles) +inline void HDFCreateEmptyDataset( const std::string Filename, const std::string ObjName, const size_t num_particles, const bool filter = false) { hid_t HDF_FileID, HDF_DatasetID, HDF_DataspaceID, - HDF_Type; + HDF_Type, + HDF_Prop; hsize_t HDF_Dims; + HDF_Prop = H5Pcreate(H5P_DATASET_CREATE); + + if (filter) + { + // 1MB chunking + hsize_t HDF_Dims[1] = {1024 * 1024 / sizeof(T)}; + H5Pset_chunk(HDF_Prop, 1, HDF_Dims); + + // md5 checksum + H5Pset_fletcher32(HDF_Prop); + } + HDF_FileID = H5Fopen( Filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT ); HDF_Type = GetDataType(); @@ -905,25 +918,39 @@ inline void HDFCreateEmptyDataset( const std::string Filename, const std::string HDF_Dims = (hsize_t) (num_particles); HDF_DataspaceID = H5Screate_simple(1, &HDF_Dims, NULL); HDF_DatasetID = H5Dcreate( HDF_FileID, ObjName.c_str(), HDF_Type, - HDF_DataspaceID, H5P_DEFAULT ); + HDF_DataspaceID, HDF_Prop ); H5Dclose( HDF_DatasetID ); H5Sclose( HDF_DataspaceID ); + H5Pclose( HDF_Prop ); H5Fclose( HDF_FileID ); } template< typename T > -inline void HDFCreateEmptyDatasetVector( const std::string Filename, const std::string ObjName, const size_t num_particles) +inline void HDFCreateEmptyDatasetVector( const std::string Filename, const std::string ObjName, const size_t num_particles, const bool filter = false) { hid_t HDF_FileID, HDF_DatasetID, HDF_DataspaceID, - HDF_Type; + HDF_Type, + HDF_Prop; hsize_t HDF_Dims[2]; + HDF_Prop = H5Pcreate(H5P_DATASET_CREATE); + + if (filter) + { + // 1MB chunking + hsize_t HDF_Dims[2] = {1024 * 1024 / sizeof(T), 3}; + H5Pset_chunk(HDF_Prop, 2, HDF_Dims); + + // md5 checksum + H5Pset_fletcher32(HDF_Prop); + } + HDF_FileID = H5Fopen( Filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT ); HDF_Type = GetDataType(); @@ -933,11 +960,12 @@ inline void HDFCreateEmptyDatasetVector( const std::string Filename, const std:: HDF_DataspaceID = H5Screate_simple(2, HDF_Dims, NULL); HDF_DatasetID = H5Dcreate( HDF_FileID, ObjName.c_str(), HDF_Type, - HDF_DataspaceID, H5P_DEFAULT ); + HDF_DataspaceID, HDF_Prop ); H5Dclose( HDF_DatasetID ); H5Sclose( HDF_DataspaceID ); + H5Pclose( HDF_Prop ); + H5Fclose( HDF_FileID ); - } template< typename T > diff --git a/src/plugins/output_swift.cc b/src/plugins/output_swift.cc index 9f4baa8..a3b61a2 100644 --- a/src/plugins/output_swift.cc +++ b/src/plugins/output_swift.cc @@ -262,30 +262,30 @@ public: if (this->has_64bit_reals()) { - HDFCreateEmptyDatasetVector(fname_, std::string("PartType") + std::to_string(sid) + std::string("/Coordinates"), global_num_particles); - HDFCreateEmptyDatasetVector(fname_, std::string("PartType") + std::to_string(sid) + std::string("/Velocities"), global_num_particles); + HDFCreateEmptyDatasetVector(fname_, std::string("PartType") + std::to_string(sid) + std::string("/Coordinates"), global_num_particles, true); + HDFCreateEmptyDatasetVector(fname_, std::string("PartType") + std::to_string(sid) + std::string("/Velocities"), global_num_particles, true); } else { - HDFCreateEmptyDatasetVector(fname_, std::string("PartType") + std::to_string(sid) + std::string("/Coordinates"), global_num_particles); - HDFCreateEmptyDatasetVector(fname_, std::string("PartType") + std::to_string(sid) + std::string("/Velocities"), global_num_particles); + HDFCreateEmptyDatasetVector(fname_, std::string("PartType") + std::to_string(sid) + std::string("/Coordinates"), global_num_particles, true); + HDFCreateEmptyDatasetVector(fname_, std::string("PartType") + std::to_string(sid) + std::string("/Velocities"), global_num_particles, true); } if (this->has_64bit_ids()) - HDFCreateEmptyDataset(fname_, std::string("PartType") + std::to_string(sid) + std::string("/ParticleIDs"), global_num_particles); + HDFCreateEmptyDataset(fname_, std::string("PartType") + std::to_string(sid) + std::string("/ParticleIDs"), global_num_particles, true); else - HDFCreateEmptyDataset(fname_, std::string("PartType") + std::to_string(sid) + std::string("/ParticleIDs"), global_num_particles); + HDFCreateEmptyDataset(fname_, std::string("PartType") + std::to_string(sid) + std::string("/ParticleIDs"), global_num_particles, true); if (this->has_64bit_reals()) - HDFCreateEmptyDataset(fname_, std::string("PartType") + std::to_string(sid) + std::string("/Masses"), global_num_particles); + HDFCreateEmptyDataset(fname_, std::string("PartType") + std::to_string(sid) + std::string("/Masses"), global_num_particles, true); else - HDFCreateEmptyDataset(fname_, std::string("PartType") + std::to_string(sid) + std::string("/Masses"), global_num_particles); + HDFCreateEmptyDataset(fname_, std::string("PartType") + std::to_string(sid) + std::string("/Masses"), global_num_particles, true); if( bdobaryons_ && s == cosmo_species::baryon) { // note: despite this being a constant array we still need to handle it in a distributed way - HDFCreateEmptyDataset(fname_, std::string("PartType") + std::to_string(sid) + std::string("/InternalEnergy"), global_num_particles); - HDFCreateEmptyDataset(fname_, std::string("PartType") + std::to_string(sid) + std::string("/SmoothingLength"), global_num_particles); + HDFCreateEmptyDataset(fname_, std::string("PartType") + std::to_string(sid) + std::string("/InternalEnergy"), global_num_particles, true); + HDFCreateEmptyDataset(fname_, std::string("PartType") + std::to_string(sid) + std::string("/SmoothingLength"), global_num_particles, true); } music::ilog << "Created empty arrays for PartType" << std::to_string(sid) << " into file " << fname_ << "." << std::endl;