OpenVDB  2.3.0
Compression.h
Go to the documentation of this file.
1 //
3 // Copyright (c) 2012-2013 DreamWorks Animation LLC
4 //
5 // All rights reserved. This software is distributed under the
6 // Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )
7 //
8 // Redistributions of source code must retain the above copyright
9 // and license notice and the following restrictions and disclaimer.
10 //
11 // * Neither the name of DreamWorks Animation nor the names of
12 // its contributors may be used to endorse or promote products derived
13 // from this software without specific prior written permission.
14 //
15 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY INDIRECT, INCIDENTAL,
20 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 // IN NO EVENT SHALL THE COPYRIGHT HOLDERS' AND CONTRIBUTORS' AGGREGATE
27 // LIABILITY FOR ALL CLAIMS REGARDLESS OF THEIR BASIS EXCEED US$250.00.
28 //
30 
31 #ifndef OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
32 #define OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
33 
34 #include <openvdb/Types.h>
35 #include <openvdb/math/Math.h> // for negative()
36 #include <boost/scoped_array.hpp>
37 #include <algorithm>
38 #include <iostream>
39 #include <string>
40 #include <vector>
41 
42 
43 namespace openvdb {
45 namespace OPENVDB_VERSION_NAME {
46 namespace io {
47 
68 enum {
70  COMPRESS_ZIP = 0x1,
72 };
73 
75 OPENVDB_API std::string compressionToString(uint32_t flags);
76 
77 
79 
80 
83 enum {
84  /*0*/ NO_MASK_OR_INACTIVE_VALS, // no inactive vals, or all inactive vals are +background
85  /*1*/ NO_MASK_AND_MINUS_BG, // all inactive vals are -background
86  /*2*/ NO_MASK_AND_ONE_INACTIVE_VAL, // all inactive vals have the same non-background val
87  /*3*/ MASK_AND_NO_INACTIVE_VALS, // mask selects between -background and +background
88  /*4*/ MASK_AND_ONE_INACTIVE_VAL, // mask selects between backgd and one other inactive val
89  /*5*/ MASK_AND_TWO_INACTIVE_VALS, // mask selects between two non-background inactive vals
90  /*6*/ NO_MASK_AND_ALL_VALS // > 2 inactive vals, so no mask compression at all
91 };
92 
93 
95 
96 
99 template<typename T>
100 struct RealToHalf {
101  enum { isReal = false }; // unless otherwise specified, type T is not a floating-point type
102  typedef T HalfT; // type T's half float analogue is T itself
103 };
104 template<> struct RealToHalf<float> { enum { isReal = true }; typedef half HalfT; };
105 template<> struct RealToHalf<double> { enum { isReal = true }; typedef half HalfT; };
106 template<> struct RealToHalf<Vec2s> { enum { isReal = true }; typedef Vec2H HalfT; };
107 template<> struct RealToHalf<Vec2d> { enum { isReal = true }; typedef Vec2H HalfT; };
108 template<> struct RealToHalf<Vec3s> { enum { isReal = true }; typedef Vec3H HalfT; };
109 template<> struct RealToHalf<Vec3d> { enum { isReal = true }; typedef Vec3H HalfT; };
110 
111 
113 template<typename T>
114 inline T
115 truncateRealToHalf(const T& val)
116 {
117  return T(typename RealToHalf<T>::HalfT(val));
118 }
119 
120 
122 
123 
124 OPENVDB_API void zipToStream(std::ostream&, const char* data, size_t numBytes);
125 OPENVDB_API void unzipFromStream(std::istream&, char* data, size_t numBytes);
126 
134 template<typename T>
135 inline void
136 readData(std::istream& is, T* data, Index count, bool compressed)
137 {
138  if (compressed) {
139  unzipFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
140  } else {
141  is.read(reinterpret_cast<char*>(data), sizeof(T) * count);
142  }
143 }
144 
146 template<>
147 inline void
148 readData<std::string>(std::istream& is, std::string* data, Index count, bool /*compressed*/)
149 {
150  for (Index i = 0; i < count; ++i) {
151  size_t len = 0;
152  is >> len;
153  //data[i].resize(len);
154  //is.read(&(data[i][0]), len);
155 
156  std::string buffer(len+1, ' ');
157  is.read(&buffer[0], len+1 );
158  data[i].assign(buffer, 0, len);
159  }
160 }
161 
166 template<bool IsReal, typename T> struct HalfReader;
168 template<typename T>
169 struct HalfReader</*IsReal=*/false, T> {
170  static inline void read(std::istream& is, T* data, Index count, bool compressed) {
171  readData(is, data, count, compressed);
172  }
173 };
175 template<typename T>
176 struct HalfReader</*IsReal=*/true, T> {
177  typedef typename RealToHalf<T>::HalfT HalfT;
178  static inline void read(std::istream& is, T* data, Index count, bool compressed) {
179  if (count < 1) return;
180  std::vector<HalfT> halfData(count); // temp buffer into which to read half float values
181  readData<HalfT>(is, reinterpret_cast<HalfT*>(&halfData[0]), count, compressed);
182  // Copy half float values from the temporary buffer to the full float output array.
183  std::copy(halfData.begin(), halfData.end(), data);
184  }
185 };
186 
187 
195 template<typename T>
196 inline void
197 writeData(std::ostream &os, const T *data, Index count, bool compress)
198 {
199  if (compress) {
200  zipToStream(os, reinterpret_cast<const char*>(data), sizeof(T) * count);
201  } else {
202  os.write(reinterpret_cast<const char*>(data), sizeof(T) * count);
203  }
204 }
205 
208 template<>
209 inline void
210 writeData<std::string>(std::ostream& os, const std::string* data, Index count, bool /*compress*/)
211 {
212  for (Index i = 0; i < count; ++i) {
213  const size_t len = data[i].size();
214  os << len;
215  os.write(data[i].c_str(), len+1);
216  //os.write(&(data[i][0]), len );
217  }
218 }
219 
224 template<bool IsReal, typename T> struct HalfWriter;
226 template<typename T>
227 struct HalfWriter</*IsReal=*/false, T> {
228  static inline void write(std::ostream& os, const T* data, Index count, bool compress) {
229  writeData(os, data, count, compress);
230  }
231 };
233 template<typename T>
234 struct HalfWriter</*IsReal=*/true, T> {
235  typedef typename RealToHalf<T>::HalfT HalfT;
236  static inline void write(std::ostream& os, const T* data, Index count, bool compress) {
237  if (count < 1) return;
238  // Convert full float values to half float, then output the half float array.
239  std::vector<HalfT> halfData(count);
240  std::copy(data, data + count, halfData.begin());
241  writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compress);
242  }
243 };
244 #ifdef _MSC_VER
245 template<>
247 struct HalfWriter</*IsReal=*/true, double> {
248  typedef RealToHalf<double>::HalfT HalfT;
249  static inline void write(std::ostream& os, const double* data, Index count, bool compress) {
250  if (count < 1) return;
251  // Convert full float values to half float, then output the half float array.
252  std::vector<HalfT> halfData(count);
253  for (Index i = 0; i < count; ++i) halfData[i] = float(data[i]);
254  writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compress);
255  }
256 };
257 #endif // _MSC_VER
258 
259 
261 
262 
275 template<typename ValueT, typename MaskT>
276 inline void
277 readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
278  const MaskT& valueMask, bool fromHalf)
279 {
280  // Get the stream's compression settings.
281  const uint32_t compression = getDataCompression(is);
282  const bool
283  zipped = compression & COMPRESS_ZIP,
284  maskCompressed = compression & COMPRESS_ACTIVE_MASK;
285 
286  int8_t metadata = NO_MASK_AND_ALL_VALS;
288  // Read the flag that specifies what, if any, additional metadata
289  // (selection mask and/or inactive value(s)) is saved.
290  is.read(reinterpret_cast<char*>(&metadata), /*bytes=*/1);
291  }
292 
293  ValueT background = zeroVal<ValueT>();
294  if (const void* bgPtr = getGridBackgroundValuePtr(is)) {
295  background = *static_cast<const ValueT*>(bgPtr);
296  }
297  ValueT inactiveVal1 = background;
298  ValueT inactiveVal0 =
299  ((metadata == NO_MASK_OR_INACTIVE_VALS) ? background : math::negative(background));
300 
301  if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL ||
302  metadata == MASK_AND_ONE_INACTIVE_VAL ||
303  metadata == MASK_AND_TWO_INACTIVE_VALS)
304  {
305  // Read one of at most two distinct inactive values.
306  is.read(reinterpret_cast<char*>(&inactiveVal0), sizeof(ValueT));
307  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
308  // Read the second of two distinct inactive values.
309  is.read(reinterpret_cast<char*>(&inactiveVal1), sizeof(ValueT));
310  }
311  }
312 
313  MaskT selectionMask;
314  if (metadata == MASK_AND_NO_INACTIVE_VALS ||
315  metadata == MASK_AND_ONE_INACTIVE_VAL ||
316  metadata == MASK_AND_TWO_INACTIVE_VALS)
317  {
318  // For use in mask compression (only), read the bitmask that selects
319  // between two distinct inactive values.
320  selectionMask.load(is);
321  }
322 
323  ValueT* tempBuf = destBuf;
324  boost::scoped_array<ValueT> scopedTempBuf;
325 
326  Index tempCount = destCount;
327  if (maskCompressed && metadata != NO_MASK_AND_ALL_VALS
329  {
330  tempCount = valueMask.countOn();
331  if (tempCount != destCount) {
332  // If this node has inactive voxels, allocate a temporary buffer
333  // into which to read just the active values.
334  scopedTempBuf.reset(new ValueT[tempCount]);
335  tempBuf = scopedTempBuf.get();
336  }
337  }
338 
339  // Read in the buffer.
340  if (fromHalf) {
341  HalfReader<RealToHalf<ValueT>::isReal, ValueT>::read(is, tempBuf, tempCount, zipped);
342  } else {
343  readData<ValueT>(is, tempBuf, tempCount, zipped);
344  }
345 
346  // If mask compression is enabled and the number of active values read into
347  // the temp buffer is smaller than the size of the destination buffer,
348  // then there are missing (inactive) values.
349  if (maskCompressed && tempCount != destCount) {
350  // Restore inactive values, using the background value and, if available,
351  // the inside/outside mask. (For fog volumes, the destination buffer is assumed
352  // to be initialized to background value zero, so inactive values can be ignored.)
353  for (Index destIdx = 0, tempIdx = 0; destIdx < MaskT::SIZE; ++destIdx) {
354  if (valueMask.isOn(destIdx)) {
355  // Copy a saved active value into this node's buffer.
356  destBuf[destIdx] = tempBuf[tempIdx];
357  ++tempIdx;
358  } else {
359  // Reconstruct an unsaved inactive value and copy it into this node's buffer.
360  destBuf[destIdx] = (selectionMask.isOn(destIdx) ? inactiveVal1 : inactiveVal0);
361  }
362  }
363  }
364 }
365 
366 
379 template<typename ValueT, typename MaskT>
380 inline void
381 writeCompressedValues(std::ostream& os, ValueT* srcBuf, Index srcCount,
382  const MaskT& valueMask, const MaskT& childMask, bool toHalf)
383 {
384  struct Local {
385  // Comparison function for values
386  static inline bool eq(const ValueT& a, const ValueT& b) {
387  return math::isExactlyEqual(a, b);
388  }
389  };
390 
391  // Get the stream's compression settings.
392  const uint32_t compress = getDataCompression(os);
393  const bool
394  zip = compress & COMPRESS_ZIP,
395  maskCompress = compress & COMPRESS_ACTIVE_MASK;
396 
397  Index tempCount = srcCount;
398  ValueT* tempBuf = srcBuf;
399  boost::scoped_array<ValueT> scopedTempBuf;
400 
401  int8_t metadata = NO_MASK_AND_ALL_VALS;
402 
403  if (!maskCompress) {
404  os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
405  } else {
406  // A valid level set's inactive values are either +background (outside)
407  // or -background (inside), and a fog volume's inactive values are all zero.
408  // Rather than write out all of these values, we can store just the active values
409  // (given that the value mask specifies their positions) and, if necessary,
410  // an inside/outside bitmask.
411 
412  const ValueT zero = zeroVal<ValueT>();
413  ValueT background = zero;
414  if (const void* bgPtr = getGridBackgroundValuePtr(os)) {
415  background = *static_cast<const ValueT*>(bgPtr);
416  }
417 
419  ValueT inactiveVal[2] = { background, background };
420  int numUniqueInactiveVals = 0;
421  for (typename MaskT::OffIterator it = valueMask.beginOff();
422  numUniqueInactiveVals < 3 && it; ++it)
423  {
424  const Index32 idx = it.pos();
425 
426  // Skip inactive values that are actually child node pointers.
427  if (childMask.isOn(idx)) continue;
428 
429  const ValueT& val = srcBuf[idx];
430  const bool unique = !(
431  (numUniqueInactiveVals > 0 && Local::eq(val, inactiveVal[0])) ||
432  (numUniqueInactiveVals > 1 && Local::eq(val, inactiveVal[1]))
433  );
434  if (unique) {
435  if (numUniqueInactiveVals < 2) inactiveVal[numUniqueInactiveVals] = val;
436  ++numUniqueInactiveVals;
437  }
438  }
439 
440  metadata = NO_MASK_OR_INACTIVE_VALS;
441 
442  if (numUniqueInactiveVals == 1) {
443  if (!Local::eq(inactiveVal[0], background)) {
444  if (Local::eq(inactiveVal[0], math::negative(background))) {
445  metadata = NO_MASK_AND_MINUS_BG;
446  } else {
447  metadata = NO_MASK_AND_ONE_INACTIVE_VAL;
448  }
449  }
450  } else if (numUniqueInactiveVals == 2) {
451  metadata = NO_MASK_OR_INACTIVE_VALS;
452  if (!Local::eq(inactiveVal[0], background) && !Local::eq(inactiveVal[1], background)) {
453  // If neither inactive value is equal to the background, both values
454  // need to be saved, along with a mask that selects between them.
455  metadata = MASK_AND_TWO_INACTIVE_VALS;
456 
457  } else if (Local::eq(inactiveVal[1], background)) {
458  if (Local::eq(inactiveVal[0], math::negative(background))) {
459  // If the second inactive value is equal to the background and
460  // the first is equal to -background, neither value needs to be saved,
461  // but save a mask that selects between -background and +background.
462  metadata = MASK_AND_NO_INACTIVE_VALS;
463  } else {
464  // If the second inactive value is equal to the background, only
465  // the first value needs to be saved, along with a mask that selects
466  // between it and the background.
467  metadata = MASK_AND_ONE_INACTIVE_VAL;
468  }
469  } else if (Local::eq(inactiveVal[0], background)) {
470  if (Local::eq(inactiveVal[1], math::negative(background))) {
471  // If the first inactive value is equal to the background and
472  // the second is equal to -background, neither value needs to be saved,
473  // but save a mask that selects between -background and +background.
474  metadata = MASK_AND_NO_INACTIVE_VALS;
475  std::swap(inactiveVal[0], inactiveVal[1]);
476  } else {
477  // If the first inactive value is equal to the background, swap it
478  // with the second value and save only that value, along with a mask
479  // that selects between it and the background.
480  std::swap(inactiveVal[0], inactiveVal[1]);
481  metadata = MASK_AND_ONE_INACTIVE_VAL;
482  }
483  }
484  } else if (numUniqueInactiveVals > 2) {
485  metadata = NO_MASK_AND_ALL_VALS;
486  }
487 
488  os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
489 
490  if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL ||
491  metadata == MASK_AND_ONE_INACTIVE_VAL ||
492  metadata == MASK_AND_TWO_INACTIVE_VALS)
493  {
494  if (!toHalf) {
495  // Write one of at most two distinct inactive values.
496  os.write(reinterpret_cast<const char*>(&inactiveVal[0]), sizeof(ValueT));
497  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
498  // Write the second of two distinct inactive values.
499  os.write(reinterpret_cast<const char*>(&inactiveVal[1]), sizeof(ValueT));
500  }
501  } else {
502  // Write one of at most two distinct inactive values.
503  ValueT truncatedVal = truncateRealToHalf(inactiveVal[0]);
504  os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
505  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
506  // Write the second of two distinct inactive values.
507  truncatedVal = truncateRealToHalf(inactiveVal[1]);
508  os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
509  }
510  }
511  }
512 
513  if (metadata == NO_MASK_AND_ALL_VALS) {
514  // If there are more than two unique inactive values, the entire input buffer
515  // needs to be saved (both active and inactive values).
518  } else {
519  // Create a new array to hold just the active values.
520  scopedTempBuf.reset(new ValueT[srcCount]);
521  tempBuf = scopedTempBuf.get();
522 
523  if (metadata == NO_MASK_OR_INACTIVE_VALS ||
524  metadata == NO_MASK_AND_MINUS_BG ||
525  metadata == NO_MASK_AND_ONE_INACTIVE_VAL)
526  {
527  // Copy active values to the contiguous array.
528  tempCount = 0;
529  for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
530  tempBuf[tempCount] = srcBuf[it.pos()];
531  }
532  } else {
533  // Copy active values to a new, contiguous array and populate a bitmask
534  // that selects between two distinct inactive values.
535  MaskT selectionMask;
536  tempCount = 0;
537  for (Index srcIdx = 0; srcIdx < srcCount; ++srcIdx) {
538  if (valueMask.isOn(srcIdx)) { // active value
539  tempBuf[tempCount] = srcBuf[srcIdx];
540  ++tempCount;
541  } else { // inactive value
542  if (Local::eq(srcBuf[srcIdx], inactiveVal[1])) {
543  selectionMask.setOn(srcIdx); // inactive value 1
544  } // else inactive value 0
545  }
546  }
547  assert(tempCount == valueMask.countOn());
548 
549  // Write out the mask that selects between two inactive values.
550  selectionMask.save(os);
551  }
552  }
553  }
554 
555  // Write out the buffer.
556  if (toHalf) {
557  HalfWriter<RealToHalf<ValueT>::isReal, ValueT>::write(os, tempBuf, tempCount, zip);
558  } else {
559  writeData(os, tempBuf, tempCount, zip);
560  }
561 }
562 
563 } // namespace io
564 } // namespace OPENVDB_VERSION_NAME
565 } // namespace openvdb
566 
567 #endif // OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
568 
569 // Copyright (c) 2012-2013 DreamWorks Animation LLC
570 // All rights reserved. This software is distributed under the
571 // Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )
Definition: Compression.h:90
#define OPENVDB_API
Helper macros for defining library symbol visibility.
Definition: Platform.h:187
Vec2< double > Vec2d
Definition: Vec2.h:521
OPENVDB_API void unzipFromStream(std::istream &, char *data, size_t numBytes)
half HalfT
Definition: Compression.h:105
void writeCompressedValues(std::ostream &os, ValueT *srcBuf, Index srcCount, const MaskT &valueMask, const MaskT &childMask, bool toHalf)
Definition: Compression.h:381
void readData(std::istream &is, T *data, Index count, bool compressed)
Read data from a stream.
Definition: Compression.h:136
static void read(std::istream &is, T *data, Index count, bool compressed)
Definition: Compression.h:178
T HalfT
Definition: Compression.h:102
Index32 Index
Definition: Types.h:57
RealToHalf< T >::HalfT HalfT
Definition: Compression.h:235
Definition: Mat.h:146
OPENVDB_API std::string compressionToString(uint32_t flags)
Return a string describing the given compression flags.
void writeData(std::ostream &os, const T *data, Index count, bool compress)
Definition: Compression.h:197
OPENVDB_API void zipToStream(std::ostream &, const char *data, size_t numBytes)
half HalfT
Definition: Compression.h:104
RealToHalf and its specializations define a mapping from floating-point data types to analogous half ...
Definition: Compression.h:100
static void write(std::ostream &os, const T *data, Index count, bool compress)
Definition: Compression.h:236
#define OPENVDB_VERSION_NAME
Definition: version.h:45
Definition: Compression.h:85
bool isExactlyEqual(const T0 &a, const T1 &b)
Return true if a is exactly equal to b.
Definition: Math.h:351
Definition: Compression.h:166
static void write(std::ostream &os, const T *data, Index count, bool compress)
Definition: Compression.h:228
Definition: Compression.h:71
Definition: Compression.h:70
uint32_t Index32
Definition: Types.h:55
OPENVDB_IMPORT uint32_t getFormatVersion(std::istream &)
Return the file format version number associated with the given input stream.
Definition: Compression.h:69
Vec2< float > Vec2s
Definition: Vec2.h:520
Vec3< double > Vec3d
Definition: Vec3.h:625
Definition: Vec2.h:48
Vec3< float > Vec3s
Definition: Vec3.h:624
OPENVDB_IMPORT const void * getGridBackgroundValuePtr(std::ios_base &)
Return a pointer to the background value of the grid currently being read from or written to the give...
RealToHalf< T >::HalfT HalfT
Definition: Compression.h:177
static void read(std::istream &is, T *data, Index count, bool compressed)
Definition: Compression.h:170
T negative(const T &val)
Return the unary negation of the given value.
Definition: Math.h:107
#define OPENVDB_USE_VERSION_NAMESPACE
Definition: version.h:67
Definition: Compression.h:224
T truncateRealToHalf(const T &val)
Return the given value truncated to 16-bit float precision.
Definition: Compression.h:115
OPENVDB_IMPORT uint32_t getDataCompression(std::ios_base &)
Return a bitwise OR of compression option flags (COMPRESS_ZIP, COMPRESS_ACTIVE_MASK, etc.) specifying whether and how input data is compressed or output data should be compressed.
void readCompressedValues(std::istream &is, ValueT *destBuf, Index destCount, const MaskT &valueMask, bool fromHalf)
Definition: Compression.h:277