Bitcoin Core  0.19.99
P2P Digital Currency
serialize.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2019 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #ifndef BITCOIN_SERIALIZE_H
7 #define BITCOIN_SERIALIZE_H
8 
9 #include <compat/endian.h>
10 
11 #include <algorithm>
12 #include <cstring>
13 #include <ios>
14 #include <limits>
15 #include <map>
16 #include <memory>
17 #include <set>
18 #include <stdint.h>
19 #include <string>
20 #include <string.h>
21 #include <utility>
22 #include <vector>
23 
24 #include <prevector.h>
25 #include <span.h>
26 
27 static const unsigned int MAX_SIZE = 0x02000000;
28 
30 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
31 
43 struct deserialize_type {};
45 
50 template<typename T>
51 inline T& REF(const T& val)
52 {
53  return const_cast<T&>(val);
54 }
55 
60 template<typename T>
61 inline T* NCONST_PTR(const T* val)
62 {
63  return const_cast<T*>(val);
64 }
65 
67 inline char* CharCast(char* c) { return c; }
68 inline char* CharCast(unsigned char* c) { return (char*)c; }
69 inline const char* CharCast(const char* c) { return c; }
70 inline const char* CharCast(const unsigned char* c) { return (const char*)c; }
71 
72 /*
73  * Lowest-level serialization and conversion.
74  * @note Sizes of these types are verified in the tests
75  */
76 template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
77 {
78  s.write((char*)&obj, 1);
79 }
80 template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
81 {
82  obj = htole16(obj);
83  s.write((char*)&obj, 2);
84 }
85 template<typename Stream> inline void ser_writedata16be(Stream &s, uint16_t obj)
86 {
87  obj = htobe16(obj);
88  s.write((char*)&obj, 2);
89 }
90 template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
91 {
92  obj = htole32(obj);
93  s.write((char*)&obj, 4);
94 }
95 template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj)
96 {
97  obj = htobe32(obj);
98  s.write((char*)&obj, 4);
99 }
100 template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
101 {
102  obj = htole64(obj);
103  s.write((char*)&obj, 8);
104 }
105 template<typename Stream> inline uint8_t ser_readdata8(Stream &s)
106 {
107  uint8_t obj;
108  s.read((char*)&obj, 1);
109  return obj;
110 }
111 template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
112 {
113  uint16_t obj;
114  s.read((char*)&obj, 2);
115  return le16toh(obj);
116 }
117 template<typename Stream> inline uint16_t ser_readdata16be(Stream &s)
118 {
119  uint16_t obj;
120  s.read((char*)&obj, 2);
121  return be16toh(obj);
122 }
123 template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
124 {
125  uint32_t obj;
126  s.read((char*)&obj, 4);
127  return le32toh(obj);
128 }
129 template<typename Stream> inline uint32_t ser_readdata32be(Stream &s)
130 {
131  uint32_t obj;
132  s.read((char*)&obj, 4);
133  return be32toh(obj);
134 }
135 template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
136 {
137  uint64_t obj;
138  s.read((char*)&obj, 8);
139  return le64toh(obj);
140 }
141 inline uint64_t ser_double_to_uint64(double x)
142 {
143  uint64_t tmp;
144  std::memcpy(&tmp, &x, sizeof(x));
145  static_assert(sizeof(tmp) == sizeof(x), "double and uint64_t assumed to have the same size");
146  return tmp;
147 }
148 inline uint32_t ser_float_to_uint32(float x)
149 {
150  uint32_t tmp;
151  std::memcpy(&tmp, &x, sizeof(x));
152  static_assert(sizeof(tmp) == sizeof(x), "float and uint32_t assumed to have the same size");
153  return tmp;
154 }
155 inline double ser_uint64_to_double(uint64_t y)
156 {
157  double tmp;
158  std::memcpy(&tmp, &y, sizeof(y));
159  static_assert(sizeof(tmp) == sizeof(y), "double and uint64_t assumed to have the same size");
160  return tmp;
161 }
162 inline float ser_uint32_to_float(uint32_t y)
163 {
164  float tmp;
165  std::memcpy(&tmp, &y, sizeof(y));
166  static_assert(sizeof(tmp) == sizeof(y), "float and uint32_t assumed to have the same size");
167  return tmp;
168 }
169 
170 
172 //
173 // Templates for serializing to anything that looks like a stream,
174 // i.e. anything that supports .read(char*, size_t) and .write(char*, size_t)
175 //
176 
177 class CSizeComputer;
178 
179 enum
180 {
181  // primary actions
182  SER_NETWORK = (1 << 0),
183  SER_DISK = (1 << 1),
184  SER_GETHASH = (1 << 2),
185 };
186 
188 template<typename X> X& ReadWriteAsHelper(X& x) { return x; }
189 template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; }
190 
191 #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__))
192 #define READWRITEAS(type, obj) (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper<type>(obj)))
193 
200 #define ADD_SERIALIZE_METHODS \
201  template<typename Stream> \
202  void Serialize(Stream& s) const { \
203  NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); \
204  } \
205  template<typename Stream> \
206  void Unserialize(Stream& s) { \
207  SerializationOp(s, CSerActionUnserialize()); \
208  }
209 
226 #define FORMATTER_METHODS(cls, obj) \
227  template<typename Stream> \
228  static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, CSerActionSerialize()); } \
229  template<typename Stream> \
230  static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, CSerActionUnserialize()); } \
231  template<typename Stream, typename Type, typename Operation> \
232  static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \
233 
234 
241 #define SERIALIZE_METHODS(cls, obj) \
242  template<typename Stream> \
243  void Serialize(Stream& s) const \
244  { \
245  static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \
246  Ser(s, *this); \
247  } \
248  template<typename Stream> \
249  void Unserialize(Stream& s) \
250  { \
251  static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \
252  Unser(s, *this); \
253  } \
254  FORMATTER_METHODS(cls, obj)
255 
256 #ifndef CHAR_EQUALS_INT8
257 template<typename Stream> inline void Serialize(Stream& s, char a ) { ser_writedata8(s, a); } // TODO Get rid of bare char
258 #endif
259 template<typename Stream> inline void Serialize(Stream& s, int8_t a ) { ser_writedata8(s, a); }
260 template<typename Stream> inline void Serialize(Stream& s, uint8_t a ) { ser_writedata8(s, a); }
261 template<typename Stream> inline void Serialize(Stream& s, int16_t a ) { ser_writedata16(s, a); }
262 template<typename Stream> inline void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); }
263 template<typename Stream> inline void Serialize(Stream& s, int32_t a ) { ser_writedata32(s, a); }
264 template<typename Stream> inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); }
265 template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); }
266 template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); }
267 template<typename Stream> inline void Serialize(Stream& s, float a ) { ser_writedata32(s, ser_float_to_uint32(a)); }
268 template<typename Stream> inline void Serialize(Stream& s, double a ) { ser_writedata64(s, ser_double_to_uint64(a)); }
269 template<typename Stream, int N> inline void Serialize(Stream& s, const char (&a)[N]) { s.write(a, N); }
270 template<typename Stream, int N> inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(CharCast(a), N); }
271 template<typename Stream> inline void Serialize(Stream& s, const Span<const unsigned char>& span) { s.write(CharCast(span.data()), span.size()); }
272 template<typename Stream> inline void Serialize(Stream& s, const Span<unsigned char>& span) { s.write(CharCast(span.data()), span.size()); }
273 
274 #ifndef CHAR_EQUALS_INT8
275 template<typename Stream> inline void Unserialize(Stream& s, char& a ) { a = ser_readdata8(s); } // TODO Get rid of bare char
276 #endif
277 template<typename Stream> inline void Unserialize(Stream& s, int8_t& a ) { a = ser_readdata8(s); }
278 template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a ) { a = ser_readdata8(s); }
279 template<typename Stream> inline void Unserialize(Stream& s, int16_t& a ) { a = ser_readdata16(s); }
280 template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); }
281 template<typename Stream> inline void Unserialize(Stream& s, int32_t& a ) { a = ser_readdata32(s); }
282 template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); }
283 template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); }
284 template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); }
285 template<typename Stream> inline void Unserialize(Stream& s, float& a ) { a = ser_uint32_to_float(ser_readdata32(s)); }
286 template<typename Stream> inline void Unserialize(Stream& s, double& a ) { a = ser_uint64_to_double(ser_readdata64(s)); }
287 template<typename Stream, int N> inline void Unserialize(Stream& s, char (&a)[N]) { s.read(a, N); }
288 template<typename Stream, int N> inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(CharCast(a), N); }
289 template<typename Stream> inline void Unserialize(Stream& s, Span<unsigned char>& span) { s.read(CharCast(span.data()), span.size()); }
290 
291 template<typename Stream> inline void Serialize(Stream& s, bool a) { char f=a; ser_writedata8(s, f); }
292 template<typename Stream> inline void Unserialize(Stream& s, bool& a) { char f=ser_readdata8(s); a=f; }
293 
294 
295 
296 
297 
298 
306 inline unsigned int GetSizeOfCompactSize(uint64_t nSize)
307 {
308  if (nSize < 253) return sizeof(unsigned char);
309  else if (nSize <= std::numeric_limits<unsigned short>::max()) return sizeof(unsigned char) + sizeof(unsigned short);
310  else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int);
311  else return sizeof(unsigned char) + sizeof(uint64_t);
312 }
313 
314 inline void WriteCompactSize(CSizeComputer& os, uint64_t nSize);
315 
316 template<typename Stream>
317 void WriteCompactSize(Stream& os, uint64_t nSize)
318 {
319  if (nSize < 253)
320  {
321  ser_writedata8(os, nSize);
322  }
323  else if (nSize <= std::numeric_limits<unsigned short>::max())
324  {
325  ser_writedata8(os, 253);
326  ser_writedata16(os, nSize);
327  }
328  else if (nSize <= std::numeric_limits<unsigned int>::max())
329  {
330  ser_writedata8(os, 254);
331  ser_writedata32(os, nSize);
332  }
333  else
334  {
335  ser_writedata8(os, 255);
336  ser_writedata64(os, nSize);
337  }
338  return;
339 }
340 
341 template<typename Stream>
342 uint64_t ReadCompactSize(Stream& is)
343 {
344  uint8_t chSize = ser_readdata8(is);
345  uint64_t nSizeRet = 0;
346  if (chSize < 253)
347  {
348  nSizeRet = chSize;
349  }
350  else if (chSize == 253)
351  {
352  nSizeRet = ser_readdata16(is);
353  if (nSizeRet < 253)
354  throw std::ios_base::failure("non-canonical ReadCompactSize()");
355  }
356  else if (chSize == 254)
357  {
358  nSizeRet = ser_readdata32(is);
359  if (nSizeRet < 0x10000u)
360  throw std::ios_base::failure("non-canonical ReadCompactSize()");
361  }
362  else
363  {
364  nSizeRet = ser_readdata64(is);
365  if (nSizeRet < 0x100000000ULL)
366  throw std::ios_base::failure("non-canonical ReadCompactSize()");
367  }
368  if (nSizeRet > (uint64_t)MAX_SIZE)
369  throw std::ios_base::failure("ReadCompactSize(): size too large");
370  return nSizeRet;
371 }
372 
408 
409 template <VarIntMode Mode, typename I>
411  constexpr CheckVarIntMode()
412  {
413  static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned<I>::value, "Unsigned type required with mode DEFAULT.");
414  static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed<I>::value, "Signed type required with mode NONNEGATIVE_SIGNED.");
415  }
416 };
417 
418 template<VarIntMode Mode, typename I>
419 inline unsigned int GetSizeOfVarInt(I n)
420 {
422  int nRet = 0;
423  while(true) {
424  nRet++;
425  if (n <= 0x7F)
426  break;
427  n = (n >> 7) - 1;
428  }
429  return nRet;
430 }
431 
432 template<typename I>
433 inline void WriteVarInt(CSizeComputer& os, I n);
434 
435 template<typename Stream, VarIntMode Mode, typename I>
436 void WriteVarInt(Stream& os, I n)
437 {
439  unsigned char tmp[(sizeof(n)*8+6)/7];
440  int len=0;
441  while(true) {
442  tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
443  if (n <= 0x7F)
444  break;
445  n = (n >> 7) - 1;
446  len++;
447  }
448  do {
449  ser_writedata8(os, tmp[len]);
450  } while(len--);
451 }
452 
453 template<typename Stream, VarIntMode Mode, typename I>
454 I ReadVarInt(Stream& is)
455 {
457  I n = 0;
458  while(true) {
459  unsigned char chData = ser_readdata8(is);
460  if (n > (std::numeric_limits<I>::max() >> 7)) {
461  throw std::ios_base::failure("ReadVarInt(): size too large");
462  }
463  n = (n << 7) | (chData & 0x7F);
464  if (chData & 0x80) {
465  if (n == std::numeric_limits<I>::max()) {
466  throw std::ios_base::failure("ReadVarInt(): size too large");
467  }
468  n++;
469  } else {
470  return n;
471  }
472  }
473 }
474 
476 template<typename Formatter, typename T>
477 class Wrapper
478 {
479  static_assert(std::is_lvalue_reference<T>::value, "Wrapper needs an lvalue reference type T");
480 protected:
481  T m_object;
482 public:
483  explicit Wrapper(T obj) : m_object(obj) {}
484  template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); }
485  template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); }
486 };
487 
498 template<typename Formatter, typename T>
499 static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); }
500 
501 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
502 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
503 #define COMPACTSIZE(obj) CCompactSize(REF(obj))
504 #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj))
505 
507 template<VarIntMode Mode>
509 {
510  template<typename Stream, typename I> void Ser(Stream &s, I v)
511  {
512  WriteVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s, v);
513  }
514 
515  template<typename Stream, typename I> void Unser(Stream& s, I& v)
516  {
517  v = ReadVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s);
518  }
519 };
520 
530 template<typename I>
532 {
533 protected:
534  I& m_val;
535 public:
536  explicit BigEndian(I& val) : m_val(val)
537  {
538  static_assert(std::is_unsigned<I>::value, "BigEndian type must be unsigned integer");
539  static_assert(sizeof(I) == 2 && std::numeric_limits<I>::min() == 0 && std::numeric_limits<I>::max() == std::numeric_limits<uint16_t>::max(), "Unsupported BigEndian size");
540  }
541 
542  template<typename Stream>
543  void Serialize(Stream& s) const
544  {
545  ser_writedata16be(s, m_val);
546  }
547 
548  template<typename Stream>
549  void Unserialize(Stream& s)
550  {
551  m_val = ser_readdata16be(s);
552  }
553 };
554 
556 {
557 protected:
558  uint64_t &n;
559 public:
560  explicit CCompactSize(uint64_t& nIn) : n(nIn) { }
561 
562  template<typename Stream>
563  void Serialize(Stream &s) const {
564  WriteCompactSize<Stream>(s, n);
565  }
566 
567  template<typename Stream>
568  void Unserialize(Stream& s) {
569  n = ReadCompactSize<Stream>(s);
570  }
571 };
572 
573 template<size_t Limit>
575 {
576 protected:
577  std::string& string;
578 public:
579  explicit LimitedString(std::string& _string) : string(_string) {}
580 
581  template<typename Stream>
582  void Unserialize(Stream& s)
583  {
584  size_t size = ReadCompactSize(s);
585  if (size > Limit) {
586  throw std::ios_base::failure("String length limit exceeded");
587  }
588  string.resize(size);
589  if (size != 0)
590  s.read((char*)string.data(), size);
591  }
592 
593  template<typename Stream>
594  void Serialize(Stream& s) const
595  {
596  WriteCompactSize(s, string.size());
597  if (!string.empty())
598  s.write((char*)string.data(), string.size());
599  }
600 };
601 
602 template<typename I>
604 
618 template<class Formatter>
620 {
621  template<typename Stream, typename V>
622  void Ser(Stream& s, const V& v)
623  {
624  WriteCompactSize(s, v.size());
625  for (const typename V::value_type& elem : v) {
626  s << Using<Formatter>(elem);
627  }
628  }
629 
630  template<typename Stream, typename V>
631  void Unser(Stream& s, V& v)
632  {
633  v.clear();
634  size_t size = ReadCompactSize(s);
635  size_t allocated = 0;
636  while (allocated < size) {
637  // For DoS prevention, do not blindly allocate as much as the stream claims to contain.
638  // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
639  // X MiB of data to make us allocate X+5 Mib.
640  static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
641  allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
642  v.reserve(allocated);
643  while (v.size() < allocated) {
644  typename V::value_type val;
645  s >> Using<Formatter>(val);
646  v.push_back(std::move(val));
647  }
648  }
649  };
650 };
651 
659 template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str);
660 template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str);
661 
666 template<typename Stream, unsigned int N, typename T> void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&);
667 template<typename Stream, unsigned int N, typename T, typename V> void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&);
668 template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v);
669 template<typename Stream, unsigned int N, typename T> void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&);
670 template<typename Stream, unsigned int N, typename T, typename V> void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&);
671 template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v);
672 
677 template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&);
678 template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&);
679 template<typename Stream, typename T, typename A, typename V> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&);
680 template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v);
681 template<typename Stream, typename T, typename A> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&);
682 template<typename Stream, typename T, typename A, typename V> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&);
683 template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v);
684 
688 template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item);
689 template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item);
690 
694 template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m);
695 template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m);
696 
700 template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m);
701 template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m);
702 
706 template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p);
707 template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p);
708 
712 template<typename Stream, typename T> void Serialize(Stream& os, const std::unique_ptr<const T>& p);
713 template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p);
714 
715 
716 
720 template<typename Stream, typename T>
721 inline void Serialize(Stream& os, const T& a)
722 {
723  a.Serialize(os);
724 }
725 
726 template<typename Stream, typename T>
727 inline void Unserialize(Stream& is, T&& a)
728 {
729  a.Unserialize(is);
730 }
731 
738 {
739  template<typename Stream, typename T>
740  static void Ser(Stream& s, const T& t) { Serialize(s, t); }
741 
742  template<typename Stream, typename T>
743  static void Unser(Stream& s, T& t) { Unserialize(s, t); }
744 };
745 
746 
747 
748 
749 
753 template<typename Stream, typename C>
754 void Serialize(Stream& os, const std::basic_string<C>& str)
755 {
756  WriteCompactSize(os, str.size());
757  if (!str.empty())
758  os.write((char*)str.data(), str.size() * sizeof(C));
759 }
760 
761 template<typename Stream, typename C>
762 void Unserialize(Stream& is, std::basic_string<C>& str)
763 {
764  unsigned int nSize = ReadCompactSize(is);
765  str.resize(nSize);
766  if (nSize != 0)
767  is.read((char*)str.data(), nSize * sizeof(C));
768 }
769 
770 
771 
775 template<typename Stream, unsigned int N, typename T>
776 void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&)
777 {
778  WriteCompactSize(os, v.size());
779  if (!v.empty())
780  os.write((char*)v.data(), v.size() * sizeof(T));
781 }
782 
783 template<typename Stream, unsigned int N, typename T, typename V>
784 void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&)
785 {
787 }
788 
789 template<typename Stream, unsigned int N, typename T>
790 inline void Serialize(Stream& os, const prevector<N, T>& v)
791 {
792  Serialize_impl(os, v, T());
793 }
794 
795 
796 template<typename Stream, unsigned int N, typename T>
797 void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&)
798 {
799  // Limit size per read so bogus size value won't cause out of memory
800  v.clear();
801  unsigned int nSize = ReadCompactSize(is);
802  unsigned int i = 0;
803  while (i < nSize)
804  {
805  unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
806  v.resize_uninitialized(i + blk);
807  is.read((char*)&v[i], blk * sizeof(T));
808  i += blk;
809  }
810 }
811 
812 template<typename Stream, unsigned int N, typename T, typename V>
813 void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&)
814 {
816 }
817 
818 template<typename Stream, unsigned int N, typename T>
819 inline void Unserialize(Stream& is, prevector<N, T>& v)
820 {
821  Unserialize_impl(is, v, T());
822 }
823 
824 
825 
829 template<typename Stream, typename T, typename A>
830 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&)
831 {
832  WriteCompactSize(os, v.size());
833  if (!v.empty())
834  os.write((char*)v.data(), v.size() * sizeof(T));
835 }
836 
837 template<typename Stream, typename T, typename A>
838 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&)
839 {
840  // A special case for std::vector<bool>, as dereferencing
841  // std::vector<bool>::const_iterator does not result in a const bool&
842  // due to std::vector's special casing for bool arguments.
843  WriteCompactSize(os, v.size());
844  for (bool elem : v) {
845  ::Serialize(os, elem);
846  }
847 }
848 
849 template<typename Stream, typename T, typename A, typename V>
850 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&)
851 {
853 }
854 
855 template<typename Stream, typename T, typename A>
856 inline void Serialize(Stream& os, const std::vector<T, A>& v)
857 {
858  Serialize_impl(os, v, T());
859 }
860 
861 
862 template<typename Stream, typename T, typename A>
863 void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&)
864 {
865  // Limit size per read so bogus size value won't cause out of memory
866  v.clear();
867  unsigned int nSize = ReadCompactSize(is);
868  unsigned int i = 0;
869  while (i < nSize)
870  {
871  unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
872  v.resize(i + blk);
873  is.read((char*)&v[i], blk * sizeof(T));
874  i += blk;
875  }
876 }
877 
878 template<typename Stream, typename T, typename A, typename V>
879 void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&)
880 {
882 }
883 
884 template<typename Stream, typename T, typename A>
885 inline void Unserialize(Stream& is, std::vector<T, A>& v)
886 {
887  Unserialize_impl(is, v, T());
888 }
889 
890 
891 
895 template<typename Stream, typename K, typename T>
896 void Serialize(Stream& os, const std::pair<K, T>& item)
897 {
898  Serialize(os, item.first);
899  Serialize(os, item.second);
900 }
901 
902 template<typename Stream, typename K, typename T>
903 void Unserialize(Stream& is, std::pair<K, T>& item)
904 {
905  Unserialize(is, item.first);
906  Unserialize(is, item.second);
907 }
908 
909 
910 
914 template<typename Stream, typename K, typename T, typename Pred, typename A>
915 void Serialize(Stream& os, const std::map<K, T, Pred, A>& m)
916 {
917  WriteCompactSize(os, m.size());
918  for (const auto& entry : m)
919  Serialize(os, entry);
920 }
921 
922 template<typename Stream, typename K, typename T, typename Pred, typename A>
923 void Unserialize(Stream& is, std::map<K, T, Pred, A>& m)
924 {
925  m.clear();
926  unsigned int nSize = ReadCompactSize(is);
927  typename std::map<K, T, Pred, A>::iterator mi = m.begin();
928  for (unsigned int i = 0; i < nSize; i++)
929  {
930  std::pair<K, T> item;
931  Unserialize(is, item);
932  mi = m.insert(mi, item);
933  }
934 }
935 
936 
937 
941 template<typename Stream, typename K, typename Pred, typename A>
942 void Serialize(Stream& os, const std::set<K, Pred, A>& m)
943 {
944  WriteCompactSize(os, m.size());
945  for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it)
946  Serialize(os, (*it));
947 }
948 
949 template<typename Stream, typename K, typename Pred, typename A>
950 void Unserialize(Stream& is, std::set<K, Pred, A>& m)
951 {
952  m.clear();
953  unsigned int nSize = ReadCompactSize(is);
954  typename std::set<K, Pred, A>::iterator it = m.begin();
955  for (unsigned int i = 0; i < nSize; i++)
956  {
957  K key;
958  Unserialize(is, key);
959  it = m.insert(it, key);
960  }
961 }
962 
963 
964 
968 template<typename Stream, typename T> void
969 Serialize(Stream& os, const std::unique_ptr<const T>& p)
970 {
971  Serialize(os, *p);
972 }
973 
974 template<typename Stream, typename T>
975 void Unserialize(Stream& is, std::unique_ptr<const T>& p)
976 {
977  p.reset(new T(deserialize, is));
978 }
979 
980 
981 
985 template<typename Stream, typename T> void
986 Serialize(Stream& os, const std::shared_ptr<const T>& p)
987 {
988  Serialize(os, *p);
989 }
990 
991 template<typename Stream, typename T>
992 void Unserialize(Stream& is, std::shared_ptr<const T>& p)
993 {
994  p = std::make_shared<const T>(deserialize, is);
995 }
996 
997 
998 
1003 {
1004  constexpr bool ForRead() const { return false; }
1005 };
1007 {
1008  constexpr bool ForRead() const { return true; }
1009 };
1010 
1011 
1012 
1013 
1014 
1015 
1016 
1017 
1018 /* ::GetSerializeSize implementations
1019  *
1020  * Computing the serialized size of objects is done through a special stream
1021  * object of type CSizeComputer, which only records the number of bytes written
1022  * to it.
1023  *
1024  * If your Serialize or SerializationOp method has non-trivial overhead for
1025  * serialization, it may be worthwhile to implement a specialized version for
1026  * CSizeComputer, which uses the s.seek() method to record bytes that would
1027  * be written instead.
1028  */
1030 {
1031 protected:
1032  size_t nSize;
1033 
1034  const int nVersion;
1035 public:
1036  explicit CSizeComputer(int nVersionIn) : nSize(0), nVersion(nVersionIn) {}
1037 
1038  void write(const char *psz, size_t _nSize)
1039  {
1040  this->nSize += _nSize;
1041  }
1042 
1044  void seek(size_t _nSize)
1045  {
1046  this->nSize += _nSize;
1047  }
1048 
1049  template<typename T>
1050  CSizeComputer& operator<<(const T& obj)
1051  {
1052  ::Serialize(*this, obj);
1053  return (*this);
1054  }
1055 
1056  size_t size() const {
1057  return nSize;
1058  }
1059 
1060  int GetVersion() const { return nVersion; }
1061 };
1062 
1063 template<typename Stream>
1064 void SerializeMany(Stream& s)
1065 {
1066 }
1067 
1068 template<typename Stream, typename Arg, typename... Args>
1069 void SerializeMany(Stream& s, const Arg& arg, const Args&... args)
1070 {
1071  ::Serialize(s, arg);
1072  ::SerializeMany(s, args...);
1073 }
1074 
1075 template<typename Stream>
1076 inline void UnserializeMany(Stream& s)
1077 {
1078 }
1079 
1080 template<typename Stream, typename Arg, typename... Args>
1081 inline void UnserializeMany(Stream& s, Arg&& arg, Args&&... args)
1082 {
1083  ::Unserialize(s, arg);
1084  ::UnserializeMany(s, args...);
1085 }
1086 
1087 template<typename Stream, typename... Args>
1088 inline void SerReadWriteMany(Stream& s, CSerActionSerialize ser_action, const Args&... args)
1089 {
1090  ::SerializeMany(s, args...);
1091 }
1092 
1093 template<typename Stream, typename... Args>
1094 inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&&... args)
1095 {
1096  ::UnserializeMany(s, args...);
1097 }
1098 
1099 template<typename I>
1100 inline void WriteVarInt(CSizeComputer &s, I n)
1101 {
1102  s.seek(GetSizeOfVarInt<I>(n));
1103 }
1104 
1105 inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize)
1106 {
1107  s.seek(GetSizeOfCompactSize(nSize));
1108 }
1109 
1110 template <typename T>
1111 size_t GetSerializeSize(const T& t, int nVersion = 0)
1112 {
1113  return (CSizeComputer(nVersion) << t).size();
1114 }
1115 
1116 template <typename... T>
1117 size_t GetSerializeSizeMany(int nVersion, const T&... t)
1118 {
1119  CSizeComputer sc(nVersion);
1120  SerializeMany(sc, t...);
1121  return sc.size();
1122 }
1123 
1124 #endif // BITCOIN_SERIALIZE_H
uint64_t ser_double_to_uint64(double x)
Definition: serialize.h:141
CSizeComputer(int nVersionIn)
Definition: serialize.h:1036
std::string & string
Definition: serialize.h:577
X & ReadWriteAsHelper(X &x)
Convert the reference base type to X, without changing constness or reference type.
Definition: serialize.h:188
constexpr std::ptrdiff_t size() const noexcept
Definition: span.h:30
uint32_t ser_float_to_uint32(float x)
Definition: serialize.h:148
uint8_t ser_readdata8(Stream &s)
Definition: serialize.h:105
void clear()
Definition: prevector.h:343
void ser_writedata64(Stream &s, uint64_t obj)
Definition: serialize.h:100
uint64_t & n
Definition: serialize.h:558
uint64_t ReadCompactSize(Stream &is)
Definition: serialize.h:342
void WriteVarInt(CSizeComputer &os, I n)
Definition: serialize.h:1100
void WriteCompactSize(CSizeComputer &os, uint64_t nSize)
Definition: serialize.h:1105
static const unsigned int MAX_VECTOR_ALLOCATE
Maximum amount of memory (in bytes) to allocate at once when deserializing vectors.
Definition: serialize.h:30
void Unser(Stream &s, I &v)
Definition: serialize.h:515
size_t size() const
Definition: serialize.h:1056
void ser_writedata32(Stream &s, uint32_t obj)
Definition: serialize.h:90
constexpr deserialize_type deserialize
Definition: serialize.h:44
unsigned int GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 ...
Definition: serialize.h:306
CCompactSize(uint64_t &nIn)
Definition: serialize.h:560
void Unserialize(Stream &s)
Definition: serialize.h:568
uint32_t be32toh(uint32_t big_endian_32bits)
Definition: endian.h:198
void resize_uninitialized(size_type new_size)
Definition: prevector.h:386
BigEndian(I &val)
Definition: serialize.h:536
const int nVersion
Definition: serialize.h:1034
void UnserializeMany(Stream &s)
Definition: serialize.h:1076
BigEndian< I > WrapBigEndian(I &n)
Definition: serialize.h:603
Simple wrapper class to serialize objects using a formatter; used by Using().
Definition: serialize.h:477
void Ser(Stream &s, const V &v)
Definition: serialize.h:622
uint32_t htole32(uint32_t host_32bits)
Definition: endian.h:191
Dummy data type to identify deserializing constructors.
Definition: serialize.h:43
static Wrapper< Formatter, T & > Using(T &&t)
Cause serialization/deserialization of an object to be done using a specified formatter class...
Definition: serialize.h:499
char * CharCast(char *c)
Safely convert odd char pointer types to standard ones.
Definition: serialize.h:67
value_type * data()
Definition: prevector.h:523
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1111
void Serialize(Stream &s, char a)
Definition: serialize.h:257
uint32_t htobe32(uint32_t host_32bits)
Definition: endian.h:184
void Serialize(Stream &s) const
Definition: serialize.h:484
void Unserialize(Stream &s)
Definition: serialize.h:582
CSizeComputer & operator<<(const T &obj)
Definition: serialize.h:1050
Serialization wrapper class for integers in VarInt format.
Definition: serialize.h:508
void Serialize(Stream &s) const
Definition: serialize.h:594
void SerReadWriteMany(Stream &s, CSerActionSerialize ser_action, const Args &... args)
Definition: serialize.h:1088
void Unser(Stream &s, V &v)
Definition: serialize.h:631
uint16_t ser_readdata16(Stream &s)
Definition: serialize.h:111
uint32_t ser_readdata32(Stream &s)
Definition: serialize.h:123
void Serialize(Stream &s) const
Definition: serialize.h:563
Serialization wrapper class for big-endian integers.
Definition: serialize.h:531
constexpr CheckVarIntMode()
Definition: serialize.h:411
void ser_writedata16(Stream &s, uint16_t obj)
Definition: serialize.h:80
uint16_t htobe16(uint16_t host_16bits)
Definition: endian.h:156
constexpr bool ForRead() const
Definition: serialize.h:1008
VarIntMode
Variable-length integers: bytes are a MSB base-128 encoding of the number.
Definition: serialize.h:407
I ReadVarInt(Stream &is)
Definition: serialize.h:454
uint16_t le16toh(uint16_t little_endian_16bits)
Definition: endian.h:177
void ser_writedata32be(Stream &s, uint32_t obj)
Definition: serialize.h:95
double ser_uint64_to_double(uint64_t y)
Definition: serialize.h:155
Support for ADD_SERIALIZE_METHODS and READWRITE macro.
Definition: serialize.h:1002
constexpr bool ForRead() const
Definition: serialize.h:1004
void SerializeMany(Stream &s)
Definition: serialize.h:1064
Implements a drop-in replacement for std::vector<T> which stores up to N elements directly (without h...
Definition: prevector.h:37
uint64_t ser_readdata64(Stream &s)
Definition: serialize.h:135
Default formatter.
Definition: serialize.h:737
#define X(name)
Definition: net.cpp:500
void seek(size_t _nSize)
Pretend _nSize bytes are written, without specifying them.
Definition: serialize.h:1044
uint16_t htole16(uint16_t host_16bits)
Definition: endian.h:163
uint16_t ser_readdata16be(Stream &s)
Definition: serialize.h:117
static const unsigned int MAX_SIZE
Definition: serialize.h:27
uint64_t le64toh(uint64_t little_endian_64bits)
Definition: endian.h:233
uint16_t be16toh(uint16_t big_endian_16bits)
Definition: endian.h:170
constexpr C * data() const noexcept
Definition: span.h:27
int GetVersion() const
Definition: serialize.h:1060
void * memcpy(void *a, const void *b, size_t c)
void Unserialize(Stream &s)
Definition: serialize.h:549
bool empty() const
Definition: prevector.h:286
uint64_t htole64(uint64_t host_64bits)
Definition: endian.h:219
void write(const char *psz, size_t _nSize)
Definition: serialize.h:1038
uint32_t ser_readdata32be(Stream &s)
Definition: serialize.h:129
unsigned int GetSizeOfVarInt(I n)
Definition: serialize.h:419
size_t GetSerializeSizeMany(int nVersion, const T &... t)
Definition: serialize.h:1117
I & m_val
Definition: serialize.h:534
void Unserialize(Stream &s, char &a)
Definition: serialize.h:275
void ser_writedata16be(Stream &s, uint16_t obj)
Definition: serialize.h:85
size_type size() const
Definition: prevector.h:282
void Unserialize(Stream &s)
Definition: serialize.h:485
void Serialize(Stream &s) const
Definition: serialize.h:543
Formatter to serialize/deserialize vector elements using another formatter.
Definition: serialize.h:619
A Span is an object that can refer to a contiguous sequence of objects.
Definition: span.h:17
void Serialize_impl(Stream &os, const prevector< N, T > &v, const unsigned char &)
prevector prevectors of unsigned char are a special case and are intended to be serialized as a singl...
Definition: serialize.h:776
T * NCONST_PTR(const T *val)
Used to acquire a non-const pointer "this" to generate bodies of const serialization operations from ...
Definition: serialize.h:61
void Unserialize_impl(Stream &is, prevector< N, T > &v, const unsigned char &)
Definition: serialize.h:797
T m_object
Definition: serialize.h:479
auto it
Definition: validation.cpp:362
uint32_t le32toh(uint32_t little_endian_32bits)
Definition: endian.h:205
T & REF(const T &val)
Used to bypass the rule against non-const reference to temporary where it makes sense with wrappers...
Definition: serialize.h:51
static void Unser(Stream &s, T &t)
Definition: serialize.h:743
size_t nSize
Definition: serialize.h:1032
LimitedString(std::string &_string)
Definition: serialize.h:579
static void Ser(Stream &s, const T &t)
Definition: serialize.h:740
void Ser(Stream &s, I v)
Definition: serialize.h:510
void ser_writedata8(Stream &s, uint8_t obj)
Definition: serialize.h:76
float ser_uint32_to_float(uint32_t y)
Definition: serialize.h:162
Wrapper(T obj)
Definition: serialize.h:483