44#include <boost/intrusive_ptr.hpp>
46#include <boost/thread/thread.hpp>
47#include <boost/thread/mutex.hpp>
50#include "SparseFieldIO.h"
77template <
typename Data_T>
78struct ReadThreadingState
99 const size_t numVoxels;
100 const size_t numBlocks;
101 const size_t numOccupiedBlocks;
102 const bool isCompressed;
103 const std::vector<size_t> &blockIdxToDatasetIdx;
104 size_t nextBlockToRead;
106 boost::mutex readMutex;
111template <
typename Data_T>
115 ReadBlockOp(ReadThreadingState<Data_T> &
state,
const size_t threadId)
119 const uLong srcLen = m_state.numVoxels *
sizeof(Data_T);
125 m_state.numOccupiedBlocks,
126 m_state.isCompressed));
127 m_reader = m_readerPtr.get();
136 boost::mutex::scoped_lock lock(m_state.readMutex);
137 blockIdx = m_state.nextBlockToRead;
138 m_state.nextBlockToRead++;
141 while (blockIdx < m_state.numBlocks) {
142 if (m_state.blocks[blockIdx].isAllocated) {
143 const size_t datasetIdx = m_state.blockIdxToDatasetIdx[blockIdx];
144 m_reader->readBlock(
datasetIdx, m_state.blocks[blockIdx].data);
148 boost::mutex::scoped_lock lock(m_state.readMutex);
149 blockIdx = m_state.nextBlockToRead;
150 m_state.nextBlockToRead++;
156 ReadThreadingState<Data_T> &m_state;
157 std::vector<uint8_t> m_cache;
158 boost::shared_ptr<OgSparseDataReader<Data_T> > m_readerPtr;
164template <
typename Data_T>
177 nextBlockToCompress(0),
181 for (
size_t i = 0;
i < numBlocks; ++
i) {
182 if (blocks[
i].isAllocated) {
183 nextBlockToCompress =
i;
184 nextBlockToWrite =
i;
189 nextBlockToCompress = numBlocks;
190 nextBlockToWrite = numBlocks;
195 const size_t numVoxels;
196 const size_t numBlocks;
197 const std::vector<uint8_t> isAllocated;
198 size_t nextBlockToCompress;
199 size_t nextBlockToWrite;
201 boost::mutex compressMutex;
206template <
typename Data_T>
210 WriteBlockOp(ThreadingState<Data_T> &
state,
const size_t threadId)
213 const uLong srcLen = m_state.numVoxels *
sizeof(Data_T);
223 boost::mutex::scoped_lock lock(m_state.compressMutex);
224 blockIdx = m_state.nextBlockToCompress++;
226 while (m_state.nextBlockToCompress < m_state.numBlocks) {
227 if (m_state.blocks[m_state.nextBlockToCompress].isAllocated) {
230 m_state.nextBlockToCompress++;
234 while (blockIdx < m_state.numBlocks) {
235 if (m_state.blocks[blockIdx].isAllocated) {
238 reinterpret_cast<const uint8_t *
>(m_state.blocks[blockIdx].data);
240 const uLong srcLen = m_state.numVoxels *
sizeof(Data_T);
248 std::cout <<
"ERROR: Couldn't compress in SparseFieldIO." << std::endl
249 <<
" Level: " <<
level << std::endl
250 <<
" Status: " <<
status << std::endl
251 <<
" srcLen: " <<
srcLen << std::endl
253 <<
" cmpLen: " <<
cmpLen << std::endl;
257 while (m_state.nextBlockToWrite != blockIdx) {
259 boost::this_thread::sleep(boost::posix_time::microseconds(1));
262 m_state.data.addData(
cmpLen, &m_cache[0]);
264 m_state.nextBlockToWrite++;
265 while (m_state.nextBlockToWrite < m_state.numBlocks){
267 if (m_state.blocks[m_state.nextBlockToWrite].isAllocated) {
270 m_state.nextBlockToWrite++;
275 boost::mutex::scoped_lock lock(m_state.compressMutex);
276 blockIdx = m_state.nextBlockToCompress++;
278 while (m_state.nextBlockToCompress < m_state.numBlocks) {
279 if (m_state.blocks[m_state.nextBlockToCompress].isAllocated) {
282 m_state.nextBlockToCompress++;
289 ThreadingState<Data_T> &m_state;
290 std::vector<uint8_t> m_cache;
291 const size_t m_threadId;
302const int SparseFieldIO::k_versionNumber(1);
303const std::string SparseFieldIO::k_versionAttrName(
"version");
304const std::string SparseFieldIO::k_extentsStr(
"extents");
305const std::string SparseFieldIO::k_extentsMinStr(
"extents_min");
306const std::string SparseFieldIO::k_extentsMaxStr(
"extents_max");
307const std::string SparseFieldIO::k_dataWindowStr(
"data_window");
308const std::string SparseFieldIO::k_dataWindowMinStr(
"data_window_min");
309const std::string SparseFieldIO::k_dataWindowMaxStr(
"data_window_max");
310const std::string SparseFieldIO::k_componentsStr(
"components");
311const std::string SparseFieldIO::k_dataStr(
"data");
312const std::string SparseFieldIO::k_blockOrderStr(
"block_order");
313const std::string SparseFieldIO::k_numBlocksStr(
"num_blocks");
314const std::string SparseFieldIO::k_blockResStr(
"block_res");
315const std::string SparseFieldIO::k_bitsPerComponentStr(
"bits_per_component");
316const std::string SparseFieldIO::k_numOccupiedBlocksStr(
"num_occupied_blocks");
317const std::string SparseFieldIO::k_isCompressed(
"data_is_compressed");
323 const std::string &layerPath,
413 if (components == 1) {
416 field->setSize(extents,
dataW);
417 field->setBlockOrder(blockOrder);
422 field->setSize(extents,
dataW);
423 field->setBlockOrder(blockOrder);
428 field->setSize(extents,
dataW);
429 field->setBlockOrder(blockOrder);
433 }
else if (components == 3) {
436 field->setSize(extents,
dataW);
437 field->setBlockOrder(blockOrder);
442 field->setSize(extents,
dataW);
443 field->setBlockOrder(blockOrder);
448 field->setSize(extents,
dataW);
449 field->setBlockOrder(blockOrder);
462 const std::string &layerPath,
OgDataType typeEnum)
568 "SparseFieldIO::read()");
604 numBlocks, filename, layerPath);
607 numBlocks, filename, layerPath);
610 numBlocks, filename, layerPath);
613 numBlocks, filename, layerPath);
616 numBlocks, filename, layerPath);
619 numBlocks, filename, layerPath);
671 "SparseField template parameter");
722 "SparseField template parameter");
730template <
class Data_T>
732SparseFieldIO::readData(
const OgIGroup &location,
const Box3i &extents,
734 const size_t numBlocks,
const std::string &filename,
735 const std::string &layerPath)
742 result->setSize(extents,
dataW);
743 result->setBlockOrder(blockOrder);
747 const size_t numVoxels = (1 << (result->m_blockOrder * 3));
748 const int valuesPerBlock = (1 << (result->m_blockOrder * 3)) * components;
765 result->addReference(filename, layerPath, valuesPerBlock, numVoxels,
774 std::vector<size_t> blockIdxToDatasetIdx(numBlocks);
780 location.findDataset<
uint8_t>(
"block_is_allocated_data");
803 location.findDataset<Data_T>(
"block_empty_value_data");
809 for (
size_t i = 0;
i < numBlocks; ++
i) {
821 if (occupiedBlocks > 0) {
824 result->setupReferenceBlocks();
827 ReadThreadingState<Data_T>
state(location, blocks, numVoxels, numBlocks,
828 occupiedBlocks, isCompressed,
829 blockIdxToDatasetIdx);
849template <
class Data_T>
862 int valuesPerBlock = (1 << (field->
m_blockOrder * 3)) * components;
877 {
dw.min.x,
dw.min.y,
dw.min.z,
dw.max.x,
dw.max.y,
dw.max.z };
903 int numBlocks = blockRes.x * blockRes.y * blockRes.z;
932 for (
int i = 0;
i < numBlocks; ++
i) {
941 for (
int i = 0;
i < numBlocks; ++
i) {
942 emptyValue[
i] =
static_cast<Data_T
>(blocks[
i].
emptyValue);
948 int occupiedBlocks = 0;
949 for (
int i = 0;
i < numBlocks; ++
i) {
950 if (blocks[
i].isAllocated) {
960 if (occupiedBlocks > 0) {
999 "SparseFieldIO::writeInternal");
1008 for (
int i = 0;
i < numBlocks; ++
i) {
1009 if (blocks[
i].isAllocated) {
1013 count[1] = valuesPerBlock;
1018 "Couldn't select slab " +
1027 "Couldn't write slab " +
1043template <
class Data_T>
1047 using namespace Exc;
1055 const size_t numBlocks = blockRes.x * blockRes.y * blockRes.z;
1056 const size_t numVoxels = (1 << (field->
m_blockOrder * 3));
1082 std::vector<uint8_t> isAllocated(numBlocks);
1083 for (
size_t i = 0;
i < numBlocks; ++
i) {
1090 std::vector<Data_T> emptyValue(numBlocks);
1091 for (
size_t i = 0;
i < numBlocks; ++
i) {
1092 emptyValue[
i] =
static_cast<Data_T
>(blocks[
i].
emptyValue);
1098 int occupiedBlocks = 0;
1099 for (
size_t i = 0;
i < numBlocks; ++
i) {
1100 if (blocks[
i].isAllocated) {
1114 if (occupiedBlocks > 0) {
1116 ThreadingState<Data_T>
state(data, blocks, numVoxels, numBlocks,
1133template <
class Data_T>
1134bool SparseFieldIO::readData(
hid_t location,
1136 const std::string &filename,
1137 const std::string &layerPath,
1140 using namespace std;
1141 using namespace Exc;
1151 int valuesPerBlock = numVoxels * components;
1164 valuesPerBlock, numVoxels,
1177 for (
int i = 0;
i < numBlocks; ++
i) {
1190 for (
int i = 0;
i < numBlocks; ++
i) {
1197 if (occupiedBlocks > 0) {
1205 size_t b = 0,
bend =
b + numBlocks;
1218 if (blocks[
b].isAllocated) {
1219 mem +=
sizeof(Data_T)*numVoxels;
Contains the initIO function.
FIELD3D_API size_t numIOThreads()
Returns the number of I/O threads to use.
OgDataType
Enumerates the various uses for Ogawa-level groups.
Contains typedefs for the commonly used types in Field3D.
boost::intrusive_ptr< FieldBase > Ptr
const Box3i & extents() const
Returns the extents of the data. This signifies the relevant area that the data exists over....
const Box3i & dataWindow() const
Returns the data window. Any coordinate inside this window is safe to pass to value() in the Field su...
Scoped object - creates a dataset on creation and closes it on destruction.
Scoped object - creates a dataspace on creation and closes it on destruction.
This class gets used by SparseFieldIO and SparseFileManager to read the block data....
This Field subclass stores voxel data in block-allocated arrays.
Block * m_blocks
Array of blocks. Not using std::vector since SparseBlock is noncopyable.
void addReference(const std::string &filename, const std::string &layerPath, int valuesPerBlock, int numVoxels, int occupiedBlocks)
Internal function to create a Reference for the current field, for use in dynamic reading.
boost::intrusive_ptr< SparseField > Ptr
int m_blockOrder
Block order (size = 2^blockOrder)
void setupReferenceBlocks()
Internal function to setup the Reference's block pointers, for use with dynamic reading.
V3i m_blockRes
Block array resolution.
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
FIELD3D_API bool readAttribute(hid_t location, const std::string &attrName, std::string &value)
Reads a string attribute.
FIELD3D_API bool writeAttribute(hid_t location, const std::string &attrName, const std::string &value)
Writes a string attribute.
FIELD3D_API bool checkHdf5Gzip()
Checks whether gzip is available in the current hdf5 library.
Namespace for Exception objects.
Contains utility functions and classes for Hdf5 files.
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity.
Namespace for sparse field specifics.
#define FIELD3D_NAMESPACE_SOURCE_CLOSE
Storage for one individual block of a SparseField.
bool isAllocated
Whether the block is allocated or not.
void resize(int n)
Alloc data.
Data_T emptyValue
The value to use if the block isn't allocated. We allow setting this per block so that we for example...
Data_T * data
Pointer to data. Null if block is unallocated.