Field3D
SparseFile.h
Go to the documentation of this file.
1//----------------------------------------------------------------------------//
2
3/*
4 * Copyright (c) 2009 Sony Pictures Imageworks Inc
5 *
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 *
12 * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the
17 * distribution. Neither the name of Sony Pictures Imageworks nor the
18 * names of its contributors may be used to endorse or promote
19 * products derived from this software without specific prior written
20 * permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33 * OF THE POSSIBILITY OF SUCH DAMAGE.
34 */
35
36//----------------------------------------------------------------------------//
37
42//----------------------------------------------------------------------------//
43
44#ifndef _INCLUDED_Field3D_SparseFile_H_
45#define _INCLUDED_Field3D_SparseFile_H_
46
47//----------------------------------------------------------------------------//
48
49#include <boost/scoped_ptr.hpp>
50#include <deque>
51#include <list>
52#include <vector>
53
54#include <hdf5.h>
55
56#include "Exception.h"
57#include "Hdf5Util.h"
58#include "OgawaFwd.h"
59#include "SparseDataReader.h"
60#include "Traits.h"
61
62//----------------------------------------------------------------------------//
63
64#define F3D_SHORT_MUTEX_ARRAY 1
65#define F3D_MUTEX_ARRAY_SIZE 1000
66#define F3D_NO_BLOCKS_ARRAY 1
67
68#include "ns.h"
69
71
72//----------------------------------------------------------------------------//
73// Forward declarations
74//----------------------------------------------------------------------------//
75
76namespace Sparse {
77
78 template <typename Data_T>
79 struct SparseBlock;
80
81}
82
83template <typename Data_T>
84class SparseField;
85
86template <typename Data_T>
88
89//----------------------------------------------------------------------------//
90
91namespace SparseFile {
92
93//----------------------------------------------------------------------------//
94// Reference
95//----------------------------------------------------------------------------//
96
102//----------------------------------------------------------------------------//
103
104template <class Data_T>
106{
107public:
108
109 // Typedefs ------------------------------------------------------------------
110
111#define USE_SHPTR 0
112
113#if USE_SHPTR
114 typedef boost::shared_ptr<Reference> Ptr;
115#else
116 typedef Reference* Ptr;
117#endif
118
119#if F3D_NO_BLOCKS_ARRAY
121#else
122 typedef std::vector<Sparse::SparseBlock<Data_T>*> BlockPtrs;
123#endif
124
125 // Public data members -------------------------------------------------------
126
127 std::string filename;
128 std::string layerPath;
133
135 std::vector<int> fileBlockIndices;
138 std::vector<int> blockLoaded;
145 std::vector<bool> blockUsed;
148 std::vector<int> loadCounts;
152 std::vector<int> refCounts;
156 boost::mutex *blockMutex;
157#if F3D_SHORT_MUTEX_ARRAY
160#endif
161
162 // Ctors, dtor ---------------------------------------------------------------
163
165 ~Reference();
166
167 // Main methods --------------------------------------------------------------
168
171 static Ptr create(const std::string a_filename,
172 const std::string a_layerPath);
174 bool fileIsOpen();
176 void setNumBlocks(int numBlocks);
180 void openFile();
182 void closeFile();
186 void loadBlock(int blockIdx);
188 void unloadBlock(int blockIdx);
191 void incBlockRef(int blockIdx);
193 void decBlockRef(int blockIdx);
195 int blockSize(int blockIdx) const;
198 int totalLoads() const;
201 int numLoadedBlocks() const;
204 int totalLoadedBlocks() const;
207 float averageLoads() const;
211 long long int memSize() const;
212
213private:
214
215 typedef boost::mutex Mutex;
216
217 // Private constructors ---
218
220 Reference(const std::string filename, const std::string layerPath);
221
223 Reference(const Reference &o);
224
226 Reference & operator=(const Reference &o);
227
228 // Data members ---
229
232
236
240
242 boost::shared_ptr<OgSparseDataReader<Data_T> > m_ogReaderPtr;
252
254 mutable Mutex m_mutex;
255
258
259};
260
261//----------------------------------------------------------------------------//
262// FileReferences
263//----------------------------------------------------------------------------//
264
266{
267public:
268
269 /* A note on thread safety
270
271 FileReferences uses a std::deque to store the References, which means
272 that already-added references 'stay put' in memory, even as more elements
273 are added. For threading purposes, we only need to lock during append()
274 and numRefs(), but ref() is safe, since we never remove references.
275
276 */
277
278 // Ctors, dtor ---------------------------------------------------------------
279
281
282 // Main methods --------------------------------------------------------------
283
286 template <class Data_T>
288
291 template <class Data_T>
293
295 template <class Data_T>
296 size_t numRefs() const;
297
299 long long int memSize() const;
300
301private:
302
303 // Data members --------------------------------------------------------------
304
305 std::deque<Reference<half>::Ptr> m_hRefs;
306 std::deque<Reference<V3h>::Ptr> m_vhRefs;
307 std::deque<Reference<float>::Ptr> m_fRefs;
308 std::deque<Reference<V3f>::Ptr> m_vfRefs;
309 std::deque<Reference<double>::Ptr> m_dRefs;
310 std::deque<Reference<V3d>::Ptr> m_vdRefs;
311
312 // Mutexes -------------------------------------------------------------------
313
314 typedef boost::mutex Mutex;
315
316 mutable Mutex m_mutex;
317
318};
319
320//----------------------------------------------------------------------------//
321
331
332//----------------------------------------------------------------------------//
333
334} // namespace SparseFile
335
336//----------------------------------------------------------------------------//
337// SparseFileManager
338//----------------------------------------------------------------------------//
339
395//----------------------------------------------------------------------------//
396
398{
399
400public:
401
402 template <class Data_T>
403 friend class SparseField;
404
405 // typedefs ------------------------------------------------------------------
406
407 typedef std::list<SparseFile::CacheBlock> CacheList;
408
409 // Main methods --------------------------------------------------------------
410
412 static SparseFileManager &singleton();
413
416 void setLimitMemUse(bool enabled);
417
420 bool doLimitMemUse() const;
421
423 void setMaxMemUse(float maxMemUse);
424
427 void flushCache();
428
430 long long totalLoads();
431
433 long long numLoadedBlocks();
434
436 long long totalLoadedBlocks();
437
440 float cacheFractionLoaded();
441
443 float cacheLoadsPerBlock();
444
447 float cacheEfficiency();
448
450 void resetCacheStatistics();
451
453 long long int memSize() const;
454
455 //--------------------------------------------------------------------------//
456 // Utility functions
457
462 template <class Data_T>
463 void incBlockRef(int fileId, int blockIdx);
464
469 template <class Data_T>
470 void decBlockRef(int fileId, int blockIdx);
471
475 template <class Data_T>
476 void activateBlock(int fileId, int blockIdx);
477
478protected:
479
481 template <class Data_T>
482 SparseFile::Reference<Data_T>* reference(int index);
483
486 template <class Data_T>
487 int getNextId(const std::string filename, const std::string layerPath);
488
489 template <class Data_T>
490 void removeFieldFromCache(int refIdx);
491
492private:
493
496
498 static boost::scoped_ptr<SparseFileManager> ms_singleton;
499
501 void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx);
502
505 void deallocateBlocks(int64_t bytesNeeded);
506
509 template <class Data_T>
510 int64_t deallocateBlock(const SparseFile::CacheBlock &cb);
511
513 template <class Data_T>
514 void deallocateBlock(CacheList::iterator &it);
515
518
521
524
528
532
539
542 CacheList::iterator m_nextBlock;
543
546 mutable boost::mutex m_mutex;
547
548};
549
550//----------------------------------------------------------------------------//
551// Reference implementations
552//----------------------------------------------------------------------------//
553
554namespace SparseFile {
555
556//----------------------------------------------------------------------------//
557
558template <class Data_T>
560 const std::string a_layerPath)
561 : filename(a_filename), layerPath(a_layerPath),
562 valuesPerBlock(-1), numVoxels(-1), numBlocks(-1), occupiedBlocks(-1),
563 blockMutex(NULL), m_fileHandle(-1), m_reader(NULL), m_ogReader(NULL),
564 m_numActiveBlocks(0)
565{
566 /* Empty */
567}
568
569//----------------------------------------------------------------------------//
570
571template <class Data_T>
573{
574 closeFile();
575
576 if (m_reader) {
577 delete m_reader;
578 }
579
580 if (blockMutex)
581 delete [] blockMutex;
582}
583
584//----------------------------------------------------------------------------//
585
586template <class Data_T>
588{
589 m_ogReaderPtr.reset();
590 m_ogReader = NULL;
591 m_reader = NULL;
592 blockMutex = NULL;
593 *this = o;
594}
595
596//----------------------------------------------------------------------------//
597
598template <class Data_T>
601{
602 if (this == &o) {
603 return *this;
604 }
605
606 // Copy public member variables (where appropriate)
607 filename = o.filename;
608 layerPath = o.layerPath;
609 valuesPerBlock = o.valuesPerBlock;
610 numVoxels = o.numVoxels;
611 occupiedBlocks = o.occupiedBlocks;
612 fileBlockIndices = o.fileBlockIndices;
613 blockLoaded = o.blockLoaded;
614 blocks = o.blocks;
615 blockUsed = o.blockUsed;
616 loadCounts = o.loadCounts;
617 refCounts = o.refCounts;
618 if (blockMutex)
619 delete[] blockMutex;
620#if F3D_SHORT_MUTEX_ARRAY
621# if F3D_NO_BLOCKS_ARRAY
622 blockMutexSize = std::min(numBlocks, F3D_MUTEX_ARRAY_SIZE);
623# else
624 blockMutexSize = std::min(static_cast<int>(blocks.size()),
626# endif
627 blockMutex = new boost::mutex[blockMutexSize];
628#else
629 blockMutex = new boost::mutex[blocks.size()];
630#endif
631
632#if 0
633 // MW: Should this be copying the file handle? Never seems to happen,
634 // but it also seems bad. Ifdef'ing out for now.
635 // Copy private member variables (where appropriate)
636 m_fileHandle = o.m_fileHandle;
637 // Don't copy id, let hdf5 generate a new one.
638 if (m_fileHandle >= 0) {
639 m_layerGroup.open(m_fileHandle, layerPath.c_str());
640 }
641#else
642 m_fileHandle = -1;
643#endif
644
645 // Re-allocate reader
646 if (m_reader)
647 delete m_reader;
648 m_reader = NULL;
649
650 m_ogReaderPtr.reset();
651 m_ogReader = NULL;
652
653 return *this;
654}
655
656//----------------------------------------------------------------------------//
657
658template <class Data_T>
661 const std::string a_layerPath)
662{
663 return Ptr(new Reference(a_filename, a_layerPath));
664}
665
666//----------------------------------------------------------------------------//
667
668template <class Data_T>
670{
671 return m_fileHandle >= 0;
672}
673
674//----------------------------------------------------------------------------//
675
676template <class Data_T>
678{
679 boost::mutex::scoped_lock lock(m_mutex);
680
681 // Store number of blocks in member variable
682 numBlocks = a_numBlocks;
683
684 fileBlockIndices.resize(numBlocks);
685 blockLoaded.resize(numBlocks, 0);
686#if !F3D_NO_BLOCKS_ARRAY
687 blocks.resize(numBlocks, 0);
688#endif
689 blockUsed.resize(numBlocks, false);
690 loadCounts.resize(numBlocks, 0);
691 refCounts.resize(numBlocks, 0);
692 if (blockMutex)
693 delete[] blockMutex;
694#if F3D_SHORT_MUTEX_ARRAY
695# if F3D_NO_BLOCKS_ARRAY
696 blockMutexSize = std::min(numBlocks, F3D_MUTEX_ARRAY_SIZE);
697# else
698 blockMutexSize = std::min(blocks.size(),
699 static_cast<size_t>(F3D_MUTEX_ARRAY_SIZE));
700# endif
701 blockMutex = new boost::mutex[blockMutexSize];
702#else
703 blockMutex = new boost::mutex[numBlocks];
704#endif
705}
706
707//----------------------------------------------------------------------------//
708
709template <class Data_T>
711{
712 if (m_fileHandle >= 0) {
713 if (H5Fclose(m_fileHandle) < 0) {
714 Msg::print("In ~Reference(): Error closing file " + filename);
715 }
716 }
717}
718
719//----------------------------------------------------------------------------//
720
721template <class Data_T>
723{
724 // Deallocate the block
725#if F3D_NO_BLOCKS_ARRAY
726 blocks[blockIdx].clear();
727#else
728 blocks[blockIdx]->clear();
729#endif
730 // Mark block as unloaded
731 blockLoaded[blockIdx] = 0;
732 // Track count
733 m_numActiveBlocks--;
734#if 0
735 // If no active blocks, close the file. De-activate for now.
736 if (m_numActiveBlocks == 0) {
737 closeFile();
738 }
739#endif
740}
741
742//----------------------------------------------------------------------------//
743
744template <class Data_T>
746{
747#if F3D_SHORT_MUTEX_ARRAY
748 boost::mutex::scoped_lock lock(blockMutex[blockIdx % blockMutexSize]);
749#else
750 boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
751#endif
752 ++refCounts[blockIdx];
753}
754
755//----------------------------------------------------------------------------//
756
757template <class Data_T>
759{
760#if F3D_SHORT_MUTEX_ARRAY
761 boost::mutex::scoped_lock lock(blockMutex[blockIdx % blockMutexSize]);
762#else
763 boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
764#endif
765 --refCounts[blockIdx];
766}
767
768//----------------------------------------------------------------------------//
769
770template <class Data_T>
771int Reference<Data_T>::blockSize(int /* blockIdx */) const
772{
773 return numVoxels * sizeof(Data_T);
774}
775
776//----------------------------------------------------------------------------//
777
778template <class Data_T>
780{
781 std::vector<int>::const_iterator i = loadCounts.begin();
782 std::vector<int>::const_iterator end = loadCounts.end();
783 int numLoads = 0;
784 for (; i != end; ++i)
785 numLoads += *i;
786
787 return numLoads;
788}
789
790//----------------------------------------------------------------------------//
791
792template <class Data_T>
794{
795 std::vector<int>::const_iterator i = blockLoaded.begin();
796 std::vector<int>::const_iterator end = blockLoaded.end();
797 int numBlockCounter = 0;
798 for (; i != end; ++i)
799 if (*i)
801
802 return numBlockCounter;
803}
804
805//----------------------------------------------------------------------------//
806
807template <class Data_T>
809{
810 std::vector<int>::const_iterator i = loadCounts.begin();
811 std::vector<int>::const_iterator li = blockLoaded.begin();
812 std::vector<int>::const_iterator end = loadCounts.end();
813 int numBlockCounter = 0;
814
815 if (blockLoaded.size() == 0) {
816 for (; i != end; ++i)
817 if (*i)
819 } else {
820 assert(loadCounts.size() == blockLoaded.size());
821
822 for (; i != end; ++i, ++li)
823 if (*i || *li)
825 }
826
827 return numBlockCounter;
828}
829
830//----------------------------------------------------------------------------//
831
832template <class Data_T>
834{
835 std::vector<int>::const_iterator i = loadCounts.begin();
836 std::vector<int>::const_iterator end = loadCounts.end();
837 int numLoads = 0, numBlockCounter = 0;
838 for (; i != end; ++i) {
839 if (*i) {
840 numLoads += *i;
842 }
843 }
844
845 return (float)numLoads / std::max(1, numBlockCounter);
846}
847
848//----------------------------------------------------------------------------//
849
850template <class Data_T>
852{
853 std::vector<int>::iterator li = loadCounts.begin();
854 std::vector<int>::iterator lend = loadCounts.end();
855 for (; li != lend; ++li)
856 *li = 0;
857}
858
859//----------------------------------------------------------------------------//
860
861template <class Data_T>
862long long int
864{
865 boost::mutex::scoped_lock lock(m_mutex);
866
867 return sizeof(*this) +
868 fileBlockIndices.capacity() * sizeof(int) +
869 blockLoaded.capacity() * sizeof(int) +
871 blocks.capacity() * sizeof(Sparse::SparseBlock<Data_T>*) +
872#endif
873 blockUsed.capacity() * sizeof(bool) +
874 loadCounts.capacity() * sizeof(int) +
875 refCounts.capacity() * sizeof(int) +
877 blockMutexSize * sizeof(boost::mutex) +
878#else
879 numBlocks * sizeof(boost::mutex) +
880#endif
882}
883
884//----------------------------------------------------------------------------//
885
886} // namespace SparseFile
887
888//----------------------------------------------------------------------------//
889// Specializations for FileReferences
890//----------------------------------------------------------------------------//
891
892namespace SparseFile {
893
894//----------------------------------------------------------------------------//
895
897{
898#if !USE_SHPTR
899 for (size_t i = 0, end = m_hRefs.size(); i < end; ++i) {
900 delete m_hRefs[i];
901 }
902 for (size_t i = 0, end = m_fRefs.size(); i < end; ++i) {
903 delete m_fRefs[i];
904 }
905 for (size_t i = 0, end = m_dRefs.size(); i < end; ++i) {
906 delete m_dRefs[i];
907 }
908 for (size_t i = 0, end = m_vhRefs.size(); i < end; ++i) {
909 delete m_vhRefs[i];
910 }
911 for (size_t i = 0, end = m_vfRefs.size(); i < end; ++i) {
912 delete m_vfRefs[i];
913 }
914 for (size_t i = 0, end = m_vdRefs.size(); i < end; ++i) {
915 delete m_vdRefs[i];
916 }
917#endif
918}
919
920//----------------------------------------------------------------------------//
921
922template <>
923inline Reference<half>*
925{
926#if USE_SHPTR
927 return m_hRefs[idx].get();
928#else
929 return m_hRefs[idx];
930#endif
931}
932
933//----------------------------------------------------------------------------//
934
935template <>
936inline Reference<V3h>*
938{
939#if USE_SHPTR
940 return m_vhRefs[idx].get();
941#else
942 return m_vhRefs[idx];
943#endif
944}
945
946//----------------------------------------------------------------------------//
947
948template <>
949inline Reference<float>*
951{
952#if USE_SHPTR
953 return m_fRefs[idx].get();
954#else
955 return m_fRefs[idx];
956#endif
957}
958
959//----------------------------------------------------------------------------//
960
961template <>
962inline Reference<V3f>*
964{
965#if USE_SHPTR
966 return m_vfRefs[idx].get();
967#else
968 return m_vfRefs[idx];
969#endif
970}
971
972//----------------------------------------------------------------------------//
973
974template <>
975inline Reference<double>*
977{
978#if USE_SHPTR
979 return m_dRefs[idx].get();
980#else
981 return m_dRefs[idx];
982#endif
983}
984
985//----------------------------------------------------------------------------//
986
987template <>
988inline Reference<V3d>*
990{
991#if USE_SHPTR
992 return m_vdRefs[idx].get();
993#else
994 return m_vdRefs[idx];
995#endif
996}
997
998//----------------------------------------------------------------------------//
999
1000template <>
1001inline size_t FileReferences::append<half>(Reference<half>::Ptr ref)
1002{
1003 Mutex::scoped_lock lock(m_mutex);
1004
1005 m_hRefs.push_back(ref);
1006 return m_hRefs.size() - 1;
1007}
1008
1009//----------------------------------------------------------------------------//
1010
1011template <>
1012inline size_t FileReferences::append<V3h>(Reference<V3h>::Ptr ref)
1013{
1014 Mutex::scoped_lock lock(m_mutex);
1015
1016 m_vhRefs.push_back(ref);
1017 return m_vhRefs.size() - 1;
1018}
1019
1020//----------------------------------------------------------------------------//
1021
1022template <>
1023inline size_t FileReferences::append<float>(Reference<float>::Ptr ref)
1024{
1025 Mutex::scoped_lock lock(m_mutex);
1026
1027 m_fRefs.push_back(ref);
1028 return m_fRefs.size() - 1;
1029}
1030
1031//----------------------------------------------------------------------------//
1032
1033template <>
1034inline size_t FileReferences::append<V3f>(Reference<V3f>::Ptr ref)
1035{
1036 Mutex::scoped_lock lock(m_mutex);
1037
1038 m_vfRefs.push_back(ref);
1039 return m_vfRefs.size() - 1;
1040}
1041
1042//----------------------------------------------------------------------------//
1043
1044template <>
1045inline size_t FileReferences::append<double>(Reference<double>::Ptr ref)
1046{
1047 Mutex::scoped_lock lock(m_mutex);
1048
1049 m_dRefs.push_back(ref);
1050 return m_dRefs.size() - 1;
1051}
1052
1053//----------------------------------------------------------------------------//
1054
1055template <>
1056inline size_t FileReferences::append<V3d>(Reference<V3d>::Ptr ref)
1057{
1058 Mutex::scoped_lock lock(m_mutex);
1059
1060 m_vdRefs.push_back(ref);
1061 return m_vdRefs.size() - 1;
1062}
1063
1064//----------------------------------------------------------------------------//
1065
1066template <>
1067inline size_t FileReferences::numRefs<half>() const
1068{
1069 Mutex::scoped_lock lock(m_mutex);
1070
1071 return m_hRefs.size();
1072}
1073
1074//----------------------------------------------------------------------------//
1075
1076template <>
1077inline size_t FileReferences::numRefs<V3h>() const
1078{
1079 Mutex::scoped_lock lock(m_mutex);
1080
1081 return m_vhRefs.size();
1082}
1083
1084//----------------------------------------------------------------------------//
1085
1086template <>
1087inline size_t FileReferences::numRefs<float>() const
1088{
1089 Mutex::scoped_lock lock(m_mutex);
1090
1091 return m_fRefs.size();
1092}
1093
1094//----------------------------------------------------------------------------//
1095
1096template <>
1097inline size_t FileReferences::numRefs<V3f>() const
1098{
1099 Mutex::scoped_lock lock(m_mutex);
1100
1101 return m_vfRefs.size();
1102}
1103
1104//----------------------------------------------------------------------------//
1105
1106template <>
1107inline size_t FileReferences::numRefs<double>() const
1108{
1109 Mutex::scoped_lock lock(m_mutex);
1110
1111 return m_dRefs.size();
1112}
1113
1114//----------------------------------------------------------------------------//
1115
1116template <>
1117inline size_t FileReferences::numRefs<V3d>() const
1118{
1119 Mutex::scoped_lock lock(m_mutex);
1120
1121 return m_vdRefs.size();
1122}
1123
1124//----------------------------------------------------------------------------//
1125// Implementations for FileReferences
1126//----------------------------------------------------------------------------//
1127
1128#if 0
1129
1130template <class Data_T>
1132{
1133 assert(false && "Do not use memory limiting on sparse fields that aren't "
1134 "simple scalars or vectors!");
1136 "FileReferences::ref(): Do not use memory limiting on sparse "
1137 "fields that aren't simple scalars or vectors!");
1138 static Reference<Data_T> dummy("", "");
1139 return dummy;
1140}
1141
1142//----------------------------------------------------------------------------//
1143
1144template <class Data_T>
1146{
1147 assert(false && "Do not use memory limiting on sparse fields that aren't "
1148 "simple scalars or vectors!");
1150 "FileReferences::append(): Do not use memory limiting on sparse "
1151 "fields that aren't simple scalars or vectors!");
1152 return -1;
1153}
1154
1155//----------------------------------------------------------------------------//
1156
1157template <class Data_T>
1158size_t FileReferences::numRefs() const
1159{
1160 assert(false && "Do not use memory limiting on sparse fields that aren't "
1161 "simple scalars or vectors!");
1163 "FileReferences::numRefs(): "
1164 "Do not use memory limiting on sparse "
1165 "fields that aren't "
1166 "simple scalars or vectors!");
1167 return -1;
1168}
1169
1170#endif
1171
1172//----------------------------------------------------------------------------//
1173
1174} // namespace SparseFile
1175
1176//----------------------------------------------------------------------------//
1177// SparseFileManager implementations
1178//----------------------------------------------------------------------------//
1179
1180template <class Data_T>
1181int
1182SparseFileManager::getNextId(const std::string filename,
1183 const std::string layerPath)
1184{
1185 using namespace SparseFile;
1186
1187 // Must hold a mutex while appending to m_fileData
1188 boost::mutex::scoped_lock lock(m_mutex);
1189
1190 int id = m_fileData.append<Data_T>(Reference<Data_T>::create(filename,
1191 layerPath));
1192 return id;
1193}
1194
1195//----------------------------------------------------------------------------//
1196
1197template <class Data_T>
1198void
1200{
1201 boost::mutex::scoped_lock lock(m_mutex);
1202
1205
1206 CacheList::iterator it = m_blockCacheList.begin();
1207 CacheList::iterator end = m_blockCacheList.end();
1208 CacheList::iterator next;
1209
1210 int64_t bytesFreed = 0;
1211
1212 while (it != end) {
1213 if (it->blockType == blockType && it->refIdx == refIdx) {
1214 if (it == m_nextBlock) {
1215 ++m_nextBlock;
1216 }
1217 next = it;
1218 ++next;
1219 bytesFreed += reference->blockSize(it->blockIdx);
1220 m_blockCacheList.erase(it);
1221 it = next;
1222 } else {
1223 ++it;
1224 }
1225 }
1227
1228 std::vector<int>().swap(reference->fileBlockIndices);
1229#if F3D_NO_BLOCKS_ARRAY
1230 reference->fileBlockIndices.resize(reference->numBlocks, -1);
1231#else
1232 reference->fileBlockIndices.resize(reference->blocks.size(), -1);
1233 typedef typename SparseFile::Reference<Data_T>::BlockPtrs BlockPtrs;
1234 BlockPtrs().swap(reference->blocks);
1235#endif
1236 std::vector<int>().swap(reference->blockLoaded);
1237 std::vector<bool>().swap(reference->blockUsed);
1238 std::vector<int>().swap(reference->loadCounts);
1239 std::vector<int>().swap(reference->refCounts);
1240 delete[] reference->blockMutex;
1241 reference->blockMutex = NULL;
1242}
1243
1244//----------------------------------------------------------------------------//
1245
1246template <class Data_T>
1249{
1250 return m_fileData.ref<Data_T>(index);
1251}
1252
1253//----------------------------------------------------------------------------//
1254
1255template <class Data_T>
1256void
1258{
1260
1261 if (reference->fileBlockIndices[blockIdx] >= 0) {
1262 if (!reference->blockLoaded[blockIdx]) {
1263 int blockSize = reference->blockSize(blockIdx);
1264 if (m_limitMemUse) {
1265 // if we already have enough free memory, deallocateBlocks()
1266 // will just return
1267 deallocateBlocks(blockSize);
1268 }
1269
1270 if (!reference->fileIsOpen()) {
1271 reference->openFile();
1272 }
1273
1274 boost::mutex::scoped_lock lock_A(m_mutex);
1275#if F3D_SHORT_MUTEX_ARRAY
1276 boost::mutex::scoped_lock
1277 lock_B(reference->blockMutex[blockIdx % reference->blockMutexSize]);
1278#else
1279 boost::mutex::scoped_lock lock_B(reference->blockMutex[blockIdx]);
1280#endif
1281 // check to see if it was loaded between when the function
1282 // started and we got the lock on the block
1283 if (!reference->blockLoaded[blockIdx]) {
1284 reference->loadBlock(blockIdx);
1285 reference->loadCounts[blockIdx]++;
1287 m_memUse += blockSize;
1288 }
1289 }
1290 }
1291 reference->blockUsed[blockIdx] = true;
1292}
1293
1294//----------------------------------------------------------------------------//
1295
1296template <class Data_T>
1297void
1299{
1301
1302 if (reference->fileBlockIndices[blockIdx] >= 0) {
1303 reference->incBlockRef(blockIdx);
1304 }
1305}
1306
1307//----------------------------------------------------------------------------//
1308
1309template <class Data_T>
1310void
1312{
1314
1315 if (reference->fileBlockIndices[blockIdx] >= 0) {
1316 reference->decBlockRef(blockIdx);
1317 }
1318}
1319
1320//----------------------------------------------------------------------------//
1321
1323
1324//----------------------------------------------------------------------------//
1325
1326#endif
Contains Exception base class.
Contains various utility functions for Hdf5.
Contains forward declarations for Ogawa classes.
boost::shared_ptr< Alembic::Ogawa::IArchive > IArchivePtr
Definition OgawaFwd.h:87
boost::shared_ptr< OgIGroup > OgIGroupPtr
Definition OgawaFwd.h:71
Contains functions controlling the loading of sparse fields.
#define F3D_SHORT_MUTEX_ARRAY
Definition SparseFile.h:64
#define F3D_NO_BLOCKS_ARRAY
Definition SparseFile.h:66
#define F3D_MUTEX_ARRAY_SIZE
Definition SparseFile.h:65
#define FIELD3D_MTX_T
Definition StdMathLib.h:99
DataTypeEnum
Definition Traits.h:108
Scoped object - opens a group on creation and closes it on destruction.
Definition Hdf5Util.h:195
This class gets used by SparseFieldIO and SparseFileManager to read the block data....
This Field subclass stores voxel data in block-allocated arrays.
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
Definition SparseFile.h:527
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full....
Definition SparseFile.h:538
int getNextId(const std::string filename, const std::string layerPath)
Returns the id of the next cache item. This is stored in the SparseField in order to reference its fi...
int64_t m_memUse
Current amount of memory in use in bytes.
Definition SparseFile.h:523
void activateBlock(int fileId, int blockIdx)
Called by SparseField when it's about to read from a block. This should not be called by the user,...
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
Definition SparseFile.h:542
void removeFieldFromCache(int refIdx)
void decBlockRef(int fileId, int blockIdx)
Decrements the usage reference count on the specified block, after its value is no longer being used ...
std::list< SparseFile::CacheBlock > CacheList
Definition SparseFile.h:407
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
Definition SparseFile.h:546
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
Definition SparseFile.h:531
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
static boost::scoped_ptr< SparseFileManager > ms_singleton
Pointer to singleton.
Definition SparseFile.h:498
float m_maxMemUse
Max amount om memory to use in megabytes.
Definition SparseFile.h:517
void deallocateBlocks(int64_t bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks.
void incBlockRef(int fileId, int blockIdx)
Increments the usage reference count on the specified block, to prevent it from getting unloaded whil...
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
Definition SparseFile.h:520
CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn)
Definition SparseFile.h:327
DataTypeEnum blockType
Definition SparseFile.h:324
Reference< Data_T > * ref(size_t idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed.
long long int memSize() const
Returns the memory use for the refs.
std::deque< Reference< double >::Ptr > m_dRefs
Definition SparseFile.h:309
std::deque< Reference< float >::Ptr > m_fRefs
Definition SparseFile.h:307
std::deque< Reference< V3d >::Ptr > m_vdRefs
Definition SparseFile.h:310
size_t numRefs() const
Returns the number of file references of the corresponding collection.
std::deque< Reference< V3f >::Ptr > m_vfRefs
Definition SparseFile.h:308
size_t append(typename Reference< Data_T >::Ptr ref)
Appends a reference to the collection. This is specialized so that the correct data member is accesse...
std::deque< Reference< half >::Ptr > m_hRefs
Definition SparseFile.h:305
std::deque< Reference< V3h >::Ptr > m_vhRefs
Definition SparseFile.h:306
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
std::string layerPath
Definition SparseFile.h:128
std::vector< bool > blockUsed
Flags of whether the blocks have been accessed since they were last considered for deallocation by th...
Definition SparseFile.h:145
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please.
Definition SparseFile.h:141
std::vector< int > fileBlockIndices
Index in file for each block.
Definition SparseFile.h:135
static Ptr create(const std::string a_filename, const std::string a_layerPath)
Returns a shared pointer to a reference. Preferred way of creating References.
Definition SparseFile.h:660
void incBlockRef(int blockIdx)
Increment reference count on a block, indicates the block is currently in use, so prevents it from be...
Definition SparseFile.h:745
OgIGroupPtr m_ogRoot
Ogawa archive root.
Definition SparseFile.h:249
Sparse::SparseBlock< Data_T > * BlockPtrs
Definition SparseFile.h:120
long long int memSize() const
Memory use for the Reference.
Definition SparseFile.h:863
Reference(const std::string filename, const std::string layerPath)
Constructor. Requires the filename and layer path of the field to be known.
Definition SparseFile.h:559
Mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
Definition SparseFile.h:254
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference's existence.
Definition SparseFile.h:235
IArchivePtr m_ogArchive
Ogawa archive.
Definition SparseFile.h:247
void closeFile()
Closes the file. Will be re-opened as needed.
Definition SparseFile.h:710
OgIGroupPtr m_ogLayerGroup
Ogawa layer group.
Definition SparseFile.h:251
int blockMutexSize
Size of the mutex array. Used as modulus base.
Definition SparseFile.h:159
std::vector< int > blockLoaded
Whether each block is loaded. We don't use bools since vector<bool> is weird.
Definition SparseFile.h:138
int totalLoadedBlocks() const
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks),...
Definition SparseFile.h:808
float averageLoads() const
Returns the average number of loads per accessed block in this file, for cache statistics.
Definition SparseFile.h:833
~Reference()
Destructor.
Definition SparseFile.h:572
OgSparseDataReader< Data_T > * m_ogReader
Pointer to the ogawa reader. NULL at construction time. Created in openFile().
Definition SparseFile.h:245
boost::shared_ptr< OgSparseDataReader< Data_T > > m_ogReaderPtr
Shared pointer to the ogawa reader.
Definition SparseFile.h:242
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
Definition SparseFile.h:239
int blockSize(int blockIdx) const
Returns the number of bytes used by the data in the block.
Definition SparseFile.h:771
std::vector< int > refCounts
Per-block counts of the number of current references to the blocks. If a block's ref count is non-zer...
Definition SparseFile.h:152
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
Definition SparseFile.h:231
int totalLoads() const
Returns the total number of loads of the blocks of this file, for cache statistics.
Definition SparseFile.h:779
boost::mutex * blockMutex
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-s...
Definition SparseFile.h:156
void decBlockRef(int blockIdx)
Decrement reference count on a block.
Definition SparseFile.h:758
size_t m_numActiveBlocks
Number of currently active blocks.
Definition SparseFile.h:257
int numLoadedBlocks() const
Returns the total number of blocks that are currently loaded, for statistics.
Definition SparseFile.h:793
Reference & operator=(const Reference &o)
Assignment operator. Clears ref counts and rebuilds mutex array.
Definition SparseFile.h:600
void unloadBlock(int blockIdx)
Unloads the block with the given index from memory.
Definition SparseFile.h:722
void setNumBlocks(int numBlocks)
Sets the number of blocks used by the SparseField we're supporting.
Definition SparseFile.h:677
std::vector< int > loadCounts
Per-block counts of the number of times each block has been loaded, for cache statistics.
Definition SparseFile.h:148
void resetCacheStatistics()
Resets counts of total block loads.
Definition SparseFile.h:851
bool fileIsOpen()
Checks if the file used by this reference is open already.
Definition SparseFile.h:669
@ SevWarning
Definition Log.h:68
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity.
Definition Log.cpp:70
Namespace for sparse field specifics.
#define FIELD3D_API
Definition ns.h:77
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition ns.h:58
static DataTypeEnum typeEnum()
Storage for one individual block of a SparseField.