4#ifndef OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
5#define OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
79template <
typename ValueT,
typename MaskT>
83 static inline bool eq(
const ValueT& a,
const ValueT& b) {
84 return math::isExactlyEqual(a, b);
88 const MaskT& valueMask,
const MaskT& childMask,
89 const ValueT* srcBuf,
const ValueT& background)
92 inactiveVal[0] = inactiveVal[1] = background;
93 int numUniqueInactiveVals = 0;
94 for (
typename MaskT::OffIterator it = valueMask.beginOff();
95 numUniqueInactiveVals < 3 && it; ++it)
100 if (childMask.isOn(idx))
continue;
102 const ValueT& val = srcBuf[idx];
103 const bool unique = !(
104 (numUniqueInactiveVals > 0 && MaskCompress::eq(val, inactiveVal[0])) ||
105 (numUniqueInactiveVals > 1 && MaskCompress::eq(val, inactiveVal[1]))
108 if (numUniqueInactiveVals < 2) inactiveVal[numUniqueInactiveVals] = val;
109 ++numUniqueInactiveVals;
115 if (numUniqueInactiveVals == 1) {
116 if (!MaskCompress::eq(inactiveVal[0], background)) {
117 if (MaskCompress::eq(inactiveVal[0], math::negative(background))) {
123 }
else if (numUniqueInactiveVals == 2) {
125 if (!MaskCompress::eq(inactiveVal[0], background) && !MaskCompress::eq(inactiveVal[1], background)) {
130 }
else if (MaskCompress::eq(inactiveVal[1], background)) {
131 if (MaskCompress::eq(inactiveVal[0], math::negative(background))) {
142 }
else if (MaskCompress::eq(inactiveVal[0], background)) {
143 if (MaskCompress::eq(inactiveVal[1], math::negative(background))) {
148 std::swap(inactiveVal[0], inactiveVal[1]);
153 std::swap(inactiveVal[0], inactiveVal[1]);
157 }
else if (numUniqueInactiveVals > 2) {
163 ValueT inactiveVal[2];
174 enum { isReal =
false };
179 enum { isReal =
true };
184 enum { isReal =
true };
190 enum { isReal =
true };
195 enum { isReal =
true };
201 enum { isReal =
true };
206 enum { isReal =
true };
250 const bool seek = data ==
nullptr;
256 if (metadata && seek && hasCompression) {
257 size_t compressedSize = metadata->getCompressedSize(metadataOffset);
258 is.seekg(compressedSize, std::ios_base::cur);
264 is.seekg(
sizeof(T) * count, std::ios_base::cur);
266 is.read(
reinterpret_cast<char*
>(data),
sizeof(T) * count);
273readData<std::string>(std::istream& is, std::string* data,
Index count, uint32_t ,
276 for (
Index i = 0; i < count; ++i) {
282 std::string buffer(len+1,
' ');
283 is.read(&buffer[0], len+1);
284 if (data !=
nullptr) data[i].assign(buffer, 0, len);
296 static inline void read(std::istream& is, T* data,
Index count, uint32_t compression,
298 readData(is, data, count, compression, metadata, metadataOffset);
305 static inline void read(std::istream& is, T* data,
Index count, uint32_t compression,
307 if (count < 1)
return;
308 if (data ==
nullptr) {
310 readData<HalfT>(is,
nullptr, count, compression, metadata, metadataOffset);
312 std::vector<HalfT> halfData(count);
313 readData<HalfT>(is,
reinterpret_cast<HalfT*
>(&halfData[0]), count, compression,
314 metadata, metadataOffset);
316 std::copy(halfData.begin(), halfData.end(), data);
329 return zipToStreamSize(
reinterpret_cast<const char*
>(data),
sizeof(T) * count);
331 return sizeof(T) * count;
339writeDataSize<std::string>(
const std::string* data,
Index count,
343 for (
Index i = 0; i < count; ++i) {
344 const size_t len = data[i].size();
345 size +=
sizeof(size_t) + (len+1);
366 bloscToStream(os,
reinterpret_cast<const char*
>(data),
sizeof(T), count);
368 zipToStream(os,
reinterpret_cast<const char*
>(data),
sizeof(T) * count);
370 os.write(
reinterpret_cast<const char*
>(data),
sizeof(T) * count);
377writeData<std::string>(std::ostream& os,
const std::string* data,
Index count,
380 for (
Index i = 0; i < count; ++i) {
381 const size_t len = data[i].size();
383 os.write(data[i].c_str(), len+1);
396 static inline size_t writeSize(
const T* data,
Index count, uint32_t compression) {
399 static inline void write(std::ostream& os,
const T* data,
Index count, uint32_t compression) {
407 static inline size_t writeSize(
const T* data,
Index count, uint32_t compression) {
408 if (count < 1)
return size_t(0);
410 std::vector<HalfT> halfData(count);
411 for (
Index i = 0; i < count; ++i) halfData[i] = RealToHalf<T>::convert(data[i]);
412 return writeDataSize<HalfT>(
reinterpret_cast<const HalfT*
>(&halfData[0]), count, compression);
414 static inline void write(std::ostream& os,
const T* data,
Index count, uint32_t compression) {
415 if (count < 1)
return;
417 std::vector<HalfT> halfData(count);
418 for (
Index i = 0; i < count; ++i) halfData[i] = RealToHalf<T>::convert(data[i]);
419 writeData<HalfT>(os,
reinterpret_cast<const HalfT*
>(&halfData[0]), count, compression);
425struct HalfWriter<true, double> {
426 using HalfT = RealToHalf<double>::HalfT;
427 static inline size_t writeSize(
const double* data,
Index count, uint32_t compression)
429 if (count < 1)
return size_t(0);
431 std::vector<HalfT> halfData(count);
432 for (
Index i = 0; i < count; ++i) halfData[i] = RealToHalf<double>::convert(data[i]);
433 return writeDataSize<HalfT>(
reinterpret_cast<const HalfT*
>(&halfData[0]), count, compression);
435 static inline void write(std::ostream& os,
const double* data,
Index count,
436 uint32_t compression)
438 if (count < 1)
return;
440 std::vector<HalfT> halfData(count);
441 for (
Index i = 0; i < count; ++i) halfData[i] = RealToHalf<double>::convert(data[i]);
442 writeData<HalfT>(os,
reinterpret_cast<const HalfT*
>(&halfData[0]), count, compression);
463template<
typename ValueT,
typename MaskT>
466 const MaskT& valueMask,
bool fromHalf)
473 const bool seek = (destBuf ==
nullptr);
474 assert(!seek || (!meta || meta->seekable()));
479 uint64_t leafIndex(0);
480 if (seek && meta && meta->delayedLoadMeta()) {
483 leafIndex = meta->leaf();
490 if (seek && !maskCompressed) {
491 is.seekg(1, std::ios_base::cur);
492 }
else if (seek && delayLoadMeta) {
493 metadata = delayLoadMeta->getMask(leafIndex);
494 is.seekg(1, std::ios_base::cur);
496 is.read(
reinterpret_cast<char*
>(&metadata), 1);
500 ValueT background = zeroVal<ValueT>();
502 background = *
static_cast<const ValueT*
>(bgPtr);
504 ValueT inactiveVal1 = background;
505 ValueT inactiveVal0 =
514 is.seekg(
sizeof(ValueT), std::ios_base::cur);
516 is.read(
reinterpret_cast<char*
>(&inactiveVal0),
sizeof(ValueT));
521 is.seekg(
sizeof(ValueT), std::ios_base::cur);
523 is.read(
reinterpret_cast<char*
>(&inactiveVal1),
sizeof(ValueT));
536 is.seekg(selectionMask.memUsage(), std::ios_base::cur);
538 selectionMask.load(is);
542 ValueT* tempBuf = destBuf;
543 std::unique_ptr<ValueT[]> scopedTempBuf;
545 Index tempCount = destCount;
550 tempCount = valueMask.countOn();
551 if (!seek && tempCount != destCount) {
554 scopedTempBuf.reset(
new ValueT[tempCount]);
555 tempBuf = scopedTempBuf.get();
562 is, (seek ?
nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
565 is, (seek ?
nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
571 if (!seek && maskCompressed && tempCount != destCount) {
575 for (
Index destIdx = 0, tempIdx = 0; destIdx < MaskT::SIZE; ++destIdx) {
576 if (valueMask.isOn(destIdx)) {
578 destBuf[destIdx] = tempBuf[tempIdx];
582 destBuf[destIdx] = (selectionMask.isOn(destIdx) ? inactiveVal1 : inactiveVal0);
589template<
typename ValueT,
typename MaskT>
592 const MaskT& valueMask, uint8_t maskMetadata,
bool toHalf, uint32_t compress)
594 using NonConstValueT =
typename std::remove_const<ValueT>::type;
598 Index tempCount = srcCount;
599 ValueT* tempBuf = srcBuf;
600 std::unique_ptr<NonConstValueT[]> scopedTempBuf;
606 Index64 onVoxels = valueMask.countOn();
609 scopedTempBuf.reset(
new NonConstValueT[onVoxels]);
610 NonConstValueT* localTempBuf = scopedTempBuf.get();
613 for (
typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
614 localTempBuf[tempCount] = srcBuf[it.pos()];
617 tempBuf = scopedTempBuf.get();
624 tempBuf, tempCount, compress);
626 return writeDataSize<NonConstValueT>(tempBuf, tempCount, compress);
643template<
typename ValueT,
typename MaskT>
646 const MaskT& valueMask,
const MaskT& childMask,
bool toHalf)
652 Index tempCount = srcCount;
653 ValueT* tempBuf = srcBuf;
654 std::unique_ptr<ValueT[]> scopedTempBuf;
659 os.write(
reinterpret_cast<const char*
>(&metadata), 1);
667 const ValueT zero = zeroVal<ValueT>();
668 ValueT background = zero;
670 background = *
static_cast<const ValueT*
>(bgPtr);
674 metadata = maskCompressData.
metadata;
676 os.write(
reinterpret_cast<const char*
>(&metadata), 1);
684 os.write(
reinterpret_cast<const char*
>(&maskCompressData.
inactiveVal[0]),
sizeof(ValueT));
687 os.write(
reinterpret_cast<const char*
>(&maskCompressData.
inactiveVal[1]),
sizeof(ValueT));
692 os.write(
reinterpret_cast<const char*
>(&truncatedVal),
sizeof(ValueT));
696 os.write(
reinterpret_cast<const char*
>(&truncatedVal),
sizeof(ValueT));
708 scopedTempBuf.reset(
new ValueT[srcCount]);
709 tempBuf = scopedTempBuf.get();
717 for (
typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
718 tempBuf[tempCount] = srcBuf[it.pos()];
725 for (
Index srcIdx = 0; srcIdx < srcCount; ++srcIdx) {
726 if (valueMask.isOn(srcIdx)) {
727 tempBuf[tempCount] = srcBuf[srcIdx];
731 selectionMask.setOn(srcIdx);
735 assert(tempCount == valueMask.countOn());
738 selectionMask.save(os);
747 writeData(os, tempBuf, tempCount, compress);
General-purpose arithmetic and comparison routines, most of which accept arbitrary value types (or at...
OPENVDB_API uint32_t getDataCompression(std::ios_base &)
Return a bitwise OR of compression option flags (COMPRESS_ZIP, COMPRESS_ACTIVE_MASK,...
OPENVDB_API size_t zipToStreamSize(const char *data, size_t numBytes)
void writeCompressedValues(std::ostream &os, ValueT *srcBuf, Index srcCount, const MaskT &valueMask, const MaskT &childMask, bool toHalf)
Definition: Compression.h:645
OPENVDB_API uint32_t getFormatVersion(std::ios_base &)
Return the file format version number associated with the given input stream.
OPENVDB_API void zipToStream(std::ostream &, const char *data, size_t numBytes)
T truncateRealToHalf(const T &val)
Return the given value truncated to 16-bit float precision.
Definition: Compression.h:216
@ MASK_AND_NO_INACTIVE_VALS
Definition: Compression.h:72
@ MASK_AND_TWO_INACTIVE_VALS
Definition: Compression.h:74
@ NO_MASK_AND_MINUS_BG
Definition: Compression.h:70
@ NO_MASK_AND_ONE_INACTIVE_VAL
Definition: Compression.h:71
@ MASK_AND_ONE_INACTIVE_VAL
Definition: Compression.h:73
@ NO_MASK_AND_ALL_VALS
Definition: Compression.h:75
@ NO_MASK_OR_INACTIVE_VALS
Definition: Compression.h:69
OPENVDB_API void unzipFromStream(std::istream &, char *data, size_t numBytes)
@ COMPRESS_BLOSC
Definition: Compression.h:56
@ COMPRESS_NONE
Definition: Compression.h:53
@ COMPRESS_ACTIVE_MASK
Definition: Compression.h:55
@ COMPRESS_ZIP
Definition: Compression.h:54
OPENVDB_API size_t bloscToStreamSize(const char *data, size_t valSize, size_t numVals)
OPENVDB_API void bloscToStream(std::ostream &, const char *data, size_t valSize, size_t numVals)
void readCompressedValues(std::istream &is, ValueT *destBuf, Index destCount, const MaskT &valueMask, bool fromHalf)
Definition: Compression.h:465
void writeData(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition: Compression.h:363
OPENVDB_API const void * getGridBackgroundValuePtr(std::ios_base &)
Return a pointer to the background value of the grid currently being read from or written to the give...
OPENVDB_API std::string compressionToString(uint32_t flags)
Return a string describing the given compression flags.
OPENVDB_API SharedPtr< StreamMetadata > getStreamMetadataPtr(std::ios_base &)
Return a shared pointer to an object that stores metadata (file format, compression scheme,...
OPENVDB_API void bloscFromStream(std::istream &, char *data, size_t numBytes)
void readData(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Read data from a stream.
Definition: Compression.h:247
size_t writeCompressedValuesSize(ValueT *srcBuf, Index srcCount, const MaskT &valueMask, uint8_t maskMetadata, bool toHalf, uint32_t compress)
Definition: Compression.h:591
size_t writeDataSize(const T *data, Index count, uint32_t compression)
Definition: Compression.h:324
Index32 Index
Definition: Types.h:54
uint32_t Index32
Definition: Types.h:52
@ OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION
Definition: version.h.in:256
uint64_t Index64
Definition: Types.h:53
Definition: Exceptions.h:13
static void read(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Definition: Compression.h:296
typename RealToHalf< T >::HalfT HalfT
Definition: Compression.h:304
static void read(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Definition: Compression.h:305
Definition: Compression.h:292
static size_t writeSize(const T *data, Index count, uint32_t compression)
Definition: Compression.h:396
static void write(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition: Compression.h:399
static size_t writeSize(const T *data, Index count, uint32_t compression)
Definition: Compression.h:407
typename RealToHalf< T >::HalfT HalfT
Definition: Compression.h:406
static void write(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition: Compression.h:414
Definition: Compression.h:392
Definition: Compression.h:81
int8_t metadata
Definition: Compression.h:162
MaskCompress(const MaskT &valueMask, const MaskT &childMask, const ValueT *srcBuf, const ValueT &background)
Definition: Compression.h:87
static bool eq(const ValueT &a, const ValueT &b)
Definition: Compression.h:83
ValueT inactiveVal[2]
Definition: Compression.h:163
static HalfT convert(const Vec2d &val)
Definition: Compression.h:198
static HalfT convert(const Vec2s &val)
Definition: Compression.h:192
static HalfT convert(const Vec3d &val)
Definition: Compression.h:209
static HalfT convert(const Vec3s &val)
Definition: Compression.h:203
static HalfT convert(double val)
Definition: Compression.h:187
static HalfT convert(float val)
Definition: Compression.h:181
RealToHalf and its specializations define a mapping from floating-point data types to analogous half ...
Definition: Compression.h:173
T HalfT
Definition: Compression.h:175
static HalfT convert(const T &val)
Definition: Compression.h:176
#define OPENVDB_VERSION_NAME
The version namespace name for this library version.
Definition: version.h.in:121
#define OPENVDB_USE_VERSION_NAMESPACE
Definition: version.h.in:212