Bitcoin Core  0.20.99
P2P Digital Currency
serialize.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2020 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #ifndef BITCOIN_SERIALIZE_H
7 #define BITCOIN_SERIALIZE_H
8 
9 #include <compat/endian.h>
10 
11 #include <algorithm>
12 #include <cstdint>
13 #include <cstring>
14 #include <ios>
15 #include <limits>
16 #include <map>
17 #include <memory>
18 #include <set>
19 #include <string>
20 #include <string.h>
21 #include <utility>
22 #include <vector>
23 
24 #include <prevector.h>
25 #include <span.h>
26 
27 static const unsigned int MAX_SIZE = 0x02000000;
28 
30 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
31 
43 struct deserialize_type {};
45 
47 inline char* CharCast(char* c) { return c; }
48 inline char* CharCast(unsigned char* c) { return (char*)c; }
49 inline const char* CharCast(const char* c) { return c; }
50 inline const char* CharCast(const unsigned char* c) { return (const char*)c; }
51 
52 /*
53  * Lowest-level serialization and conversion.
54  * @note Sizes of these types are verified in the tests
55  */
56 template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
57 {
58  s.write((char*)&obj, 1);
59 }
60 template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
61 {
62  obj = htole16(obj);
63  s.write((char*)&obj, 2);
64 }
65 template<typename Stream> inline void ser_writedata16be(Stream &s, uint16_t obj)
66 {
67  obj = htobe16(obj);
68  s.write((char*)&obj, 2);
69 }
70 template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
71 {
72  obj = htole32(obj);
73  s.write((char*)&obj, 4);
74 }
75 template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj)
76 {
77  obj = htobe32(obj);
78  s.write((char*)&obj, 4);
79 }
80 template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
81 {
82  obj = htole64(obj);
83  s.write((char*)&obj, 8);
84 }
85 template<typename Stream> inline uint8_t ser_readdata8(Stream &s)
86 {
87  uint8_t obj;
88  s.read((char*)&obj, 1);
89  return obj;
90 }
91 template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
92 {
93  uint16_t obj;
94  s.read((char*)&obj, 2);
95  return le16toh(obj);
96 }
97 template<typename Stream> inline uint16_t ser_readdata16be(Stream &s)
98 {
99  uint16_t obj;
100  s.read((char*)&obj, 2);
101  return be16toh(obj);
102 }
103 template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
104 {
105  uint32_t obj;
106  s.read((char*)&obj, 4);
107  return le32toh(obj);
108 }
109 template<typename Stream> inline uint32_t ser_readdata32be(Stream &s)
110 {
111  uint32_t obj;
112  s.read((char*)&obj, 4);
113  return be32toh(obj);
114 }
115 template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
116 {
117  uint64_t obj;
118  s.read((char*)&obj, 8);
119  return le64toh(obj);
120 }
121 inline uint64_t ser_double_to_uint64(double x)
122 {
123  uint64_t tmp;
124  std::memcpy(&tmp, &x, sizeof(x));
125  static_assert(sizeof(tmp) == sizeof(x), "double and uint64_t assumed to have the same size");
126  return tmp;
127 }
128 inline uint32_t ser_float_to_uint32(float x)
129 {
130  uint32_t tmp;
131  std::memcpy(&tmp, &x, sizeof(x));
132  static_assert(sizeof(tmp) == sizeof(x), "float and uint32_t assumed to have the same size");
133  return tmp;
134 }
135 inline double ser_uint64_to_double(uint64_t y)
136 {
137  double tmp;
138  std::memcpy(&tmp, &y, sizeof(y));
139  static_assert(sizeof(tmp) == sizeof(y), "double and uint64_t assumed to have the same size");
140  return tmp;
141 }
142 inline float ser_uint32_to_float(uint32_t y)
143 {
144  float tmp;
145  std::memcpy(&tmp, &y, sizeof(y));
146  static_assert(sizeof(tmp) == sizeof(y), "float and uint32_t assumed to have the same size");
147  return tmp;
148 }
149 
150 
152 //
153 // Templates for serializing to anything that looks like a stream,
154 // i.e. anything that supports .read(char*, size_t) and .write(char*, size_t)
155 //
156 
157 class CSizeComputer;
158 
159 enum
160 {
161  // primary actions
162  SER_NETWORK = (1 << 0),
163  SER_DISK = (1 << 1),
164  SER_GETHASH = (1 << 2),
165 };
166 
168 template<typename X> X& ReadWriteAsHelper(X& x) { return x; }
169 template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; }
170 
171 #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__))
172 #define READWRITEAS(type, obj) (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper<type>(obj)))
173 #define SER_READ(obj, code) ::SerRead(s, ser_action, obj, [&](Stream& s, typename std::remove_const<Type>::type& obj) { code; })
174 #define SER_WRITE(obj, code) ::SerWrite(s, ser_action, obj, [&](Stream& s, const Type& obj) { code; })
175 
192 #define FORMATTER_METHODS(cls, obj) \
193  template<typename Stream> \
194  static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, CSerActionSerialize()); } \
195  template<typename Stream> \
196  static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, CSerActionUnserialize()); } \
197  template<typename Stream, typename Type, typename Operation> \
198  static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \
199 
200 
207 #define SERIALIZE_METHODS(cls, obj) \
208  template<typename Stream> \
209  void Serialize(Stream& s) const \
210  { \
211  static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \
212  Ser(s, *this); \
213  } \
214  template<typename Stream> \
215  void Unserialize(Stream& s) \
216  { \
217  static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \
218  Unser(s, *this); \
219  } \
220  FORMATTER_METHODS(cls, obj)
221 
222 #ifndef CHAR_EQUALS_INT8
223 template<typename Stream> inline void Serialize(Stream& s, char a ) { ser_writedata8(s, a); } // TODO Get rid of bare char
224 #endif
225 template<typename Stream> inline void Serialize(Stream& s, int8_t a ) { ser_writedata8(s, a); }
226 template<typename Stream> inline void Serialize(Stream& s, uint8_t a ) { ser_writedata8(s, a); }
227 template<typename Stream> inline void Serialize(Stream& s, int16_t a ) { ser_writedata16(s, a); }
228 template<typename Stream> inline void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); }
229 template<typename Stream> inline void Serialize(Stream& s, int32_t a ) { ser_writedata32(s, a); }
230 template<typename Stream> inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); }
231 template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); }
232 template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); }
233 template<typename Stream> inline void Serialize(Stream& s, float a ) { ser_writedata32(s, ser_float_to_uint32(a)); }
234 template<typename Stream> inline void Serialize(Stream& s, double a ) { ser_writedata64(s, ser_double_to_uint64(a)); }
235 template<typename Stream, int N> inline void Serialize(Stream& s, const char (&a)[N]) { s.write(a, N); }
236 template<typename Stream, int N> inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(CharCast(a), N); }
237 template<typename Stream> inline void Serialize(Stream& s, const Span<const unsigned char>& span) { s.write(CharCast(span.data()), span.size()); }
238 template<typename Stream> inline void Serialize(Stream& s, const Span<unsigned char>& span) { s.write(CharCast(span.data()), span.size()); }
239 
240 #ifndef CHAR_EQUALS_INT8
241 template<typename Stream> inline void Unserialize(Stream& s, char& a ) { a = ser_readdata8(s); } // TODO Get rid of bare char
242 #endif
243 template<typename Stream> inline void Unserialize(Stream& s, int8_t& a ) { a = ser_readdata8(s); }
244 template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a ) { a = ser_readdata8(s); }
245 template<typename Stream> inline void Unserialize(Stream& s, int16_t& a ) { a = ser_readdata16(s); }
246 template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); }
247 template<typename Stream> inline void Unserialize(Stream& s, int32_t& a ) { a = ser_readdata32(s); }
248 template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); }
249 template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); }
250 template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); }
251 template<typename Stream> inline void Unserialize(Stream& s, float& a ) { a = ser_uint32_to_float(ser_readdata32(s)); }
252 template<typename Stream> inline void Unserialize(Stream& s, double& a ) { a = ser_uint64_to_double(ser_readdata64(s)); }
253 template<typename Stream, int N> inline void Unserialize(Stream& s, char (&a)[N]) { s.read(a, N); }
254 template<typename Stream, int N> inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(CharCast(a), N); }
255 template<typename Stream> inline void Unserialize(Stream& s, Span<unsigned char>& span) { s.read(CharCast(span.data()), span.size()); }
256 
257 template<typename Stream> inline void Serialize(Stream& s, bool a) { char f=a; ser_writedata8(s, f); }
258 template<typename Stream> inline void Unserialize(Stream& s, bool& a) { char f=ser_readdata8(s); a=f; }
259 
260 
261 
262 
263 
264 
272 inline unsigned int GetSizeOfCompactSize(uint64_t nSize)
273 {
274  if (nSize < 253) return sizeof(unsigned char);
275  else if (nSize <= std::numeric_limits<uint16_t>::max()) return sizeof(unsigned char) + sizeof(uint16_t);
276  else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int);
277  else return sizeof(unsigned char) + sizeof(uint64_t);
278 }
279 
280 inline void WriteCompactSize(CSizeComputer& os, uint64_t nSize);
281 
282 template<typename Stream>
283 void WriteCompactSize(Stream& os, uint64_t nSize)
284 {
285  if (nSize < 253)
286  {
287  ser_writedata8(os, nSize);
288  }
289  else if (nSize <= std::numeric_limits<uint16_t>::max())
290  {
291  ser_writedata8(os, 253);
292  ser_writedata16(os, nSize);
293  }
294  else if (nSize <= std::numeric_limits<unsigned int>::max())
295  {
296  ser_writedata8(os, 254);
297  ser_writedata32(os, nSize);
298  }
299  else
300  {
301  ser_writedata8(os, 255);
302  ser_writedata64(os, nSize);
303  }
304  return;
305 }
306 
307 template<typename Stream>
308 uint64_t ReadCompactSize(Stream& is)
309 {
310  uint8_t chSize = ser_readdata8(is);
311  uint64_t nSizeRet = 0;
312  if (chSize < 253)
313  {
314  nSizeRet = chSize;
315  }
316  else if (chSize == 253)
317  {
318  nSizeRet = ser_readdata16(is);
319  if (nSizeRet < 253)
320  throw std::ios_base::failure("non-canonical ReadCompactSize()");
321  }
322  else if (chSize == 254)
323  {
324  nSizeRet = ser_readdata32(is);
325  if (nSizeRet < 0x10000u)
326  throw std::ios_base::failure("non-canonical ReadCompactSize()");
327  }
328  else
329  {
330  nSizeRet = ser_readdata64(is);
331  if (nSizeRet < 0x100000000ULL)
332  throw std::ios_base::failure("non-canonical ReadCompactSize()");
333  }
334  if (nSizeRet > (uint64_t)MAX_SIZE)
335  throw std::ios_base::failure("ReadCompactSize(): size too large");
336  return nSizeRet;
337 }
338 
374 
375 template <VarIntMode Mode, typename I>
377  constexpr CheckVarIntMode()
378  {
379  static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned<I>::value, "Unsigned type required with mode DEFAULT.");
380  static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed<I>::value, "Signed type required with mode NONNEGATIVE_SIGNED.");
381  }
382 };
383 
384 template<VarIntMode Mode, typename I>
385 inline unsigned int GetSizeOfVarInt(I n)
386 {
388  int nRet = 0;
389  while(true) {
390  nRet++;
391  if (n <= 0x7F)
392  break;
393  n = (n >> 7) - 1;
394  }
395  return nRet;
396 }
397 
398 template<typename I>
399 inline void WriteVarInt(CSizeComputer& os, I n);
400 
401 template<typename Stream, VarIntMode Mode, typename I>
402 void WriteVarInt(Stream& os, I n)
403 {
405  unsigned char tmp[(sizeof(n)*8+6)/7];
406  int len=0;
407  while(true) {
408  tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
409  if (n <= 0x7F)
410  break;
411  n = (n >> 7) - 1;
412  len++;
413  }
414  do {
415  ser_writedata8(os, tmp[len]);
416  } while(len--);
417 }
418 
419 template<typename Stream, VarIntMode Mode, typename I>
420 I ReadVarInt(Stream& is)
421 {
423  I n = 0;
424  while(true) {
425  unsigned char chData = ser_readdata8(is);
426  if (n > (std::numeric_limits<I>::max() >> 7)) {
427  throw std::ios_base::failure("ReadVarInt(): size too large");
428  }
429  n = (n << 7) | (chData & 0x7F);
430  if (chData & 0x80) {
431  if (n == std::numeric_limits<I>::max()) {
432  throw std::ios_base::failure("ReadVarInt(): size too large");
433  }
434  n++;
435  } else {
436  return n;
437  }
438  }
439 }
440 
442 template<typename Formatter, typename T>
443 class Wrapper
444 {
445  static_assert(std::is_lvalue_reference<T>::value, "Wrapper needs an lvalue reference type T");
446 protected:
447  T m_object;
448 public:
449  explicit Wrapper(T obj) : m_object(obj) {}
450  template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); }
451  template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); }
452 };
453 
464 template<typename Formatter, typename T>
465 static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); }
466 
467 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
468 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
469 #define COMPACTSIZE(obj) Using<CompactSizeFormatter>(obj)
470 #define LIMITED_STRING(obj,n) Using<LimitedStringFormatter<n>>(obj)
471 
473 template<VarIntMode Mode>
475 {
476  template<typename Stream, typename I> void Ser(Stream &s, I v)
477  {
478  WriteVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s, v);
479  }
480 
481  template<typename Stream, typename I> void Unser(Stream& s, I& v)
482  {
483  v = ReadVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s);
484  }
485 };
486 
496 template<int Bytes, bool BigEndian = false>
498 {
499  static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
500  static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
501 
502  template <typename Stream, typename I> void Ser(Stream& s, I v)
503  {
504  if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
505  if (BigEndian) {
506  uint64_t raw = htobe64(v);
507  s.write(((const char*)&raw) + 8 - Bytes, Bytes);
508  } else {
509  uint64_t raw = htole64(v);
510  s.write((const char*)&raw, Bytes);
511  }
512  }
513 
514  template <typename Stream, typename I> void Unser(Stream& s, I& v)
515  {
516  using U = typename std::conditional<std::is_enum<I>::value, std::underlying_type<I>, std::common_type<I>>::type::type;
517  static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small");
518  uint64_t raw = 0;
519  if (BigEndian) {
520  s.read(((char*)&raw) + 8 - Bytes, Bytes);
521  v = static_cast<I>(be64toh(raw));
522  } else {
523  s.read((char*)&raw, Bytes);
524  v = static_cast<I>(le64toh(raw));
525  }
526  }
527 };
528 
530 
533 {
534  template<typename Stream, typename I>
535  void Unser(Stream& s, I& v)
536  {
537  uint64_t n = ReadCompactSize<Stream>(s);
538  if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
539  throw std::ios_base::failure("CompactSize exceeds limit of type");
540  }
541  v = n;
542  }
543 
544  template<typename Stream, typename I>
545  void Ser(Stream& s, I v)
546  {
547  static_assert(std::is_unsigned<I>::value, "CompactSize only supported for unsigned integers");
548  static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
549 
550  WriteCompactSize<Stream>(s, v);
551  }
552 };
553 
554 template<size_t Limit>
556 {
557  template<typename Stream>
558  void Unser(Stream& s, std::string& v)
559  {
560  size_t size = ReadCompactSize(s);
561  if (size > Limit) {
562  throw std::ios_base::failure("String length limit exceeded");
563  }
564  v.resize(size);
565  if (size != 0) s.read((char*)v.data(), size);
566  }
567 
568  template<typename Stream>
569  void Ser(Stream& s, const std::string& v)
570  {
571  s << v;
572  }
573 };
574 
588 template<class Formatter>
590 {
591  template<typename Stream, typename V>
592  void Ser(Stream& s, const V& v)
593  {
594  Formatter formatter;
595  WriteCompactSize(s, v.size());
596  for (const typename V::value_type& elem : v) {
597  formatter.Ser(s, elem);
598  }
599  }
600 
601  template<typename Stream, typename V>
602  void Unser(Stream& s, V& v)
603  {
604  Formatter formatter;
605  v.clear();
606  size_t size = ReadCompactSize(s);
607  size_t allocated = 0;
608  while (allocated < size) {
609  // For DoS prevention, do not blindly allocate as much as the stream claims to contain.
610  // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
611  // X MiB of data to make us allocate X+5 Mib.
612  static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
613  allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
614  v.reserve(allocated);
615  while (v.size() < allocated) {
616  v.emplace_back();
617  formatter.Unser(s, v.back());
618  }
619  }
620  };
621 };
622 
630 template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str);
631 template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str);
632 
637 template<typename Stream, unsigned int N, typename T> void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&);
638 template<typename Stream, unsigned int N, typename T, typename V> void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&);
639 template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v);
640 template<typename Stream, unsigned int N, typename T> void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&);
641 template<typename Stream, unsigned int N, typename T, typename V> void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&);
642 template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v);
643 
648 template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&);
649 template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&);
650 template<typename Stream, typename T, typename A, typename V> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&);
651 template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v);
652 template<typename Stream, typename T, typename A> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&);
653 template<typename Stream, typename T, typename A, typename V> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&);
654 template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v);
655 
659 template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item);
660 template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item);
661 
665 template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m);
666 template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m);
667 
671 template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m);
672 template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m);
673 
677 template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p);
678 template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p);
679 
683 template<typename Stream, typename T> void Serialize(Stream& os, const std::unique_ptr<const T>& p);
684 template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p);
685 
686 
687 
691 template<typename Stream, typename T>
692 inline void Serialize(Stream& os, const T& a)
693 {
694  a.Serialize(os);
695 }
696 
697 template<typename Stream, typename T>
698 inline void Unserialize(Stream& is, T&& a)
699 {
700  a.Unserialize(is);
701 }
702 
709 {
710  template<typename Stream, typename T>
711  static void Ser(Stream& s, const T& t) { Serialize(s, t); }
712 
713  template<typename Stream, typename T>
714  static void Unser(Stream& s, T& t) { Unserialize(s, t); }
715 };
716 
717 
718 
719 
720 
724 template<typename Stream, typename C>
725 void Serialize(Stream& os, const std::basic_string<C>& str)
726 {
727  WriteCompactSize(os, str.size());
728  if (!str.empty())
729  os.write((char*)str.data(), str.size() * sizeof(C));
730 }
731 
732 template<typename Stream, typename C>
733 void Unserialize(Stream& is, std::basic_string<C>& str)
734 {
735  unsigned int nSize = ReadCompactSize(is);
736  str.resize(nSize);
737  if (nSize != 0)
738  is.read((char*)str.data(), nSize * sizeof(C));
739 }
740 
741 
742 
746 template<typename Stream, unsigned int N, typename T>
747 void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&)
748 {
749  WriteCompactSize(os, v.size());
750  if (!v.empty())
751  os.write((char*)v.data(), v.size() * sizeof(T));
752 }
753 
754 template<typename Stream, unsigned int N, typename T, typename V>
755 void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&)
756 {
758 }
759 
760 template<typename Stream, unsigned int N, typename T>
761 inline void Serialize(Stream& os, const prevector<N, T>& v)
762 {
763  Serialize_impl(os, v, T());
764 }
765 
766 
767 template<typename Stream, unsigned int N, typename T>
768 void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&)
769 {
770  // Limit size per read so bogus size value won't cause out of memory
771  v.clear();
772  unsigned int nSize = ReadCompactSize(is);
773  unsigned int i = 0;
774  while (i < nSize)
775  {
776  unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
777  v.resize_uninitialized(i + blk);
778  is.read((char*)&v[i], blk * sizeof(T));
779  i += blk;
780  }
781 }
782 
783 template<typename Stream, unsigned int N, typename T, typename V>
784 void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&)
785 {
787 }
788 
789 template<typename Stream, unsigned int N, typename T>
790 inline void Unserialize(Stream& is, prevector<N, T>& v)
791 {
792  Unserialize_impl(is, v, T());
793 }
794 
795 
796 
800 template<typename Stream, typename T, typename A>
801 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&)
802 {
803  WriteCompactSize(os, v.size());
804  if (!v.empty())
805  os.write((char*)v.data(), v.size() * sizeof(T));
806 }
807 
808 template<typename Stream, typename T, typename A>
809 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&)
810 {
811  // A special case for std::vector<bool>, as dereferencing
812  // std::vector<bool>::const_iterator does not result in a const bool&
813  // due to std::vector's special casing for bool arguments.
814  WriteCompactSize(os, v.size());
815  for (bool elem : v) {
816  ::Serialize(os, elem);
817  }
818 }
819 
820 template<typename Stream, typename T, typename A, typename V>
821 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&)
822 {
824 }
825 
826 template<typename Stream, typename T, typename A>
827 inline void Serialize(Stream& os, const std::vector<T, A>& v)
828 {
829  Serialize_impl(os, v, T());
830 }
831 
832 
833 template<typename Stream, typename T, typename A>
834 void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&)
835 {
836  // Limit size per read so bogus size value won't cause out of memory
837  v.clear();
838  unsigned int nSize = ReadCompactSize(is);
839  unsigned int i = 0;
840  while (i < nSize)
841  {
842  unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
843  v.resize(i + blk);
844  is.read((char*)&v[i], blk * sizeof(T));
845  i += blk;
846  }
847 }
848 
849 template<typename Stream, typename T, typename A, typename V>
850 void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&)
851 {
853 }
854 
855 template<typename Stream, typename T, typename A>
856 inline void Unserialize(Stream& is, std::vector<T, A>& v)
857 {
858  Unserialize_impl(is, v, T());
859 }
860 
861 
862 
866 template<typename Stream, typename K, typename T>
867 void Serialize(Stream& os, const std::pair<K, T>& item)
868 {
869  Serialize(os, item.first);
870  Serialize(os, item.second);
871 }
872 
873 template<typename Stream, typename K, typename T>
874 void Unserialize(Stream& is, std::pair<K, T>& item)
875 {
876  Unserialize(is, item.first);
877  Unserialize(is, item.second);
878 }
879 
880 
881 
885 template<typename Stream, typename K, typename T, typename Pred, typename A>
886 void Serialize(Stream& os, const std::map<K, T, Pred, A>& m)
887 {
888  WriteCompactSize(os, m.size());
889  for (const auto& entry : m)
890  Serialize(os, entry);
891 }
892 
893 template<typename Stream, typename K, typename T, typename Pred, typename A>
894 void Unserialize(Stream& is, std::map<K, T, Pred, A>& m)
895 {
896  m.clear();
897  unsigned int nSize = ReadCompactSize(is);
898  typename std::map<K, T, Pred, A>::iterator mi = m.begin();
899  for (unsigned int i = 0; i < nSize; i++)
900  {
901  std::pair<K, T> item;
902  Unserialize(is, item);
903  mi = m.insert(mi, item);
904  }
905 }
906 
907 
908 
912 template<typename Stream, typename K, typename Pred, typename A>
913 void Serialize(Stream& os, const std::set<K, Pred, A>& m)
914 {
915  WriteCompactSize(os, m.size());
916  for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it)
917  Serialize(os, (*it));
918 }
919 
920 template<typename Stream, typename K, typename Pred, typename A>
921 void Unserialize(Stream& is, std::set<K, Pred, A>& m)
922 {
923  m.clear();
924  unsigned int nSize = ReadCompactSize(is);
925  typename std::set<K, Pred, A>::iterator it = m.begin();
926  for (unsigned int i = 0; i < nSize; i++)
927  {
928  K key;
929  Unserialize(is, key);
930  it = m.insert(it, key);
931  }
932 }
933 
934 
935 
939 template<typename Stream, typename T> void
940 Serialize(Stream& os, const std::unique_ptr<const T>& p)
941 {
942  Serialize(os, *p);
943 }
944 
945 template<typename Stream, typename T>
946 void Unserialize(Stream& is, std::unique_ptr<const T>& p)
947 {
948  p.reset(new T(deserialize, is));
949 }
950 
951 
952 
956 template<typename Stream, typename T> void
957 Serialize(Stream& os, const std::shared_ptr<const T>& p)
958 {
959  Serialize(os, *p);
960 }
961 
962 template<typename Stream, typename T>
963 void Unserialize(Stream& is, std::shared_ptr<const T>& p)
964 {
965  p = std::make_shared<const T>(deserialize, is);
966 }
967 
968 
969 
974 {
975  constexpr bool ForRead() const { return false; }
976 };
978 {
979  constexpr bool ForRead() const { return true; }
980 };
981 
982 
983 
984 
985 
986 
987 
988 
989 /* ::GetSerializeSize implementations
990  *
991  * Computing the serialized size of objects is done through a special stream
992  * object of type CSizeComputer, which only records the number of bytes written
993  * to it.
994  *
995  * If your Serialize or SerializationOp method has non-trivial overhead for
996  * serialization, it may be worthwhile to implement a specialized version for
997  * CSizeComputer, which uses the s.seek() method to record bytes that would
998  * be written instead.
999  */
1001 {
1002 protected:
1003  size_t nSize;
1004 
1005  const int nVersion;
1006 public:
1007  explicit CSizeComputer(int nVersionIn) : nSize(0), nVersion(nVersionIn) {}
1008 
1009  void write(const char *psz, size_t _nSize)
1010  {
1011  this->nSize += _nSize;
1012  }
1013 
1015  void seek(size_t _nSize)
1016  {
1017  this->nSize += _nSize;
1018  }
1019 
1020  template<typename T>
1021  CSizeComputer& operator<<(const T& obj)
1022  {
1023  ::Serialize(*this, obj);
1024  return (*this);
1025  }
1026 
1027  size_t size() const {
1028  return nSize;
1029  }
1030 
1031  int GetVersion() const { return nVersion; }
1032 };
1033 
1034 template<typename Stream>
1035 void SerializeMany(Stream& s)
1036 {
1037 }
1038 
1039 template<typename Stream, typename Arg, typename... Args>
1040 void SerializeMany(Stream& s, const Arg& arg, const Args&... args)
1041 {
1042  ::Serialize(s, arg);
1043  ::SerializeMany(s, args...);
1044 }
1045 
1046 template<typename Stream>
1047 inline void UnserializeMany(Stream& s)
1048 {
1049 }
1050 
1051 template<typename Stream, typename Arg, typename... Args>
1052 inline void UnserializeMany(Stream& s, Arg&& arg, Args&&... args)
1053 {
1054  ::Unserialize(s, arg);
1055  ::UnserializeMany(s, args...);
1056 }
1057 
1058 template<typename Stream, typename... Args>
1059 inline void SerReadWriteMany(Stream& s, CSerActionSerialize ser_action, const Args&... args)
1060 {
1061  ::SerializeMany(s, args...);
1062 }
1063 
1064 template<typename Stream, typename... Args>
1065 inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&&... args)
1066 {
1067  ::UnserializeMany(s, args...);
1068 }
1069 
1070 template<typename Stream, typename Type, typename Fn>
1071 inline void SerRead(Stream& s, CSerActionSerialize ser_action, Type&&, Fn&&)
1072 {
1073 }
1074 
1075 template<typename Stream, typename Type, typename Fn>
1076 inline void SerRead(Stream& s, CSerActionUnserialize ser_action, Type&& obj, Fn&& fn)
1077 {
1078  fn(s, std::forward<Type>(obj));
1079 }
1080 
1081 template<typename Stream, typename Type, typename Fn>
1082 inline void SerWrite(Stream& s, CSerActionSerialize ser_action, Type&& obj, Fn&& fn)
1083 {
1084  fn(s, std::forward<Type>(obj));
1085 }
1086 
1087 template<typename Stream, typename Type, typename Fn>
1088 inline void SerWrite(Stream& s, CSerActionUnserialize ser_action, Type&&, Fn&&)
1089 {
1090 }
1091 
1092 template<typename I>
1093 inline void WriteVarInt(CSizeComputer &s, I n)
1094 {
1095  s.seek(GetSizeOfVarInt<I>(n));
1096 }
1097 
1098 inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize)
1099 {
1100  s.seek(GetSizeOfCompactSize(nSize));
1101 }
1102 
1103 template <typename T>
1104 size_t GetSerializeSize(const T& t, int nVersion = 0)
1105 {
1106  return (CSizeComputer(nVersion) << t).size();
1107 }
1108 
1109 template <typename... T>
1110 size_t GetSerializeSizeMany(int nVersion, const T&... t)
1111 {
1112  CSizeComputer sc(nVersion);
1113  SerializeMany(sc, t...);
1114  return sc.size();
1115 }
1116 
1117 #endif // BITCOIN_SERIALIZE_H
uint64_t ser_double_to_uint64(double x)
Definition: serialize.h:121
CSizeComputer(int nVersionIn)
Definition: serialize.h:1007
X & ReadWriteAsHelper(X &x)
Convert the reference base type to X, without changing constness or reference type.
Definition: serialize.h:168
uint32_t ser_float_to_uint32(float x)
Definition: serialize.h:128
uint8_t ser_readdata8(Stream &s)
Definition: serialize.h:85
void clear()
Definition: prevector.h:343
void ser_writedata64(Stream &s, uint64_t obj)
Definition: serialize.h:80
uint64_t ReadCompactSize(Stream &is)
Definition: serialize.h:308
void WriteVarInt(CSizeComputer &os, I n)
Definition: serialize.h:1093
void WriteCompactSize(CSizeComputer &os, uint64_t nSize)
Definition: serialize.h:1098
void SerRead(Stream &s, CSerActionSerialize ser_action, Type &&, Fn &&)
Definition: serialize.h:1071
uint64_t htobe64(uint64_t host_64bits)
Definition: endian.h:212
static const unsigned int MAX_VECTOR_ALLOCATE
Maximum amount of memory (in bytes) to allocate at once when deserializing vectors.
Definition: serialize.h:30
void Unser(Stream &s, I &v)
Definition: serialize.h:481
size_t size() const
Definition: serialize.h:1027
void ser_writedata32(Stream &s, uint32_t obj)
Definition: serialize.h:70
constexpr deserialize_type deserialize
Definition: serialize.h:44
constexpr std::size_t size() const noexcept
Definition: span.h:153
unsigned int GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 ...
Definition: serialize.h:272
Formatter for integers in CompactSize format.
Definition: serialize.h:532
uint32_t be32toh(uint32_t big_endian_32bits)
Definition: endian.h:198
void resize_uninitialized(size_type new_size)
Definition: prevector.h:386
const int nVersion
Definition: serialize.h:1005
void UnserializeMany(Stream &s)
Definition: serialize.h:1047
Simple wrapper class to serialize objects using a formatter; used by Using().
Definition: serialize.h:443
void Ser(Stream &s, const V &v)
Definition: serialize.h:592
uint32_t htole32(uint32_t host_32bits)
Definition: endian.h:191
Dummy data type to identify deserializing constructors.
Definition: serialize.h:43
static Wrapper< Formatter, T & > Using(T &&t)
Cause serialization/deserialization of an object to be done using a specified formatter class...
Definition: serialize.h:465
char * CharCast(char *c)
Safely convert odd char pointer types to standard ones.
Definition: serialize.h:47
value_type * data()
Definition: prevector.h:528
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1104
void Serialize(Stream &s, char a)
Definition: serialize.h:223
uint32_t htobe32(uint32_t host_32bits)
Definition: endian.h:184
void Serialize(Stream &s) const
Definition: serialize.h:450
void Unser(Stream &s, I &v)
Definition: serialize.h:514
CSizeComputer & operator<<(const T &obj)
Definition: serialize.h:1021
uint64_t be64toh(uint64_t big_endian_64bits)
Definition: endian.h:226
Serialization wrapper class for integers in VarInt format.
Definition: serialize.h:474
void SerReadWriteMany(Stream &s, CSerActionSerialize ser_action, const Args &... args)
Definition: serialize.h:1059
void Unser(Stream &s, V &v)
Definition: serialize.h:602
uint16_t ser_readdata16(Stream &s)
Definition: serialize.h:91
uint32_t ser_readdata32(Stream &s)
Definition: serialize.h:103
constexpr CheckVarIntMode()
Definition: serialize.h:377
void ser_writedata16(Stream &s, uint16_t obj)
Definition: serialize.h:60
void Unser(Stream &s, I &v)
Definition: serialize.h:535
uint16_t htobe16(uint16_t host_16bits)
Definition: endian.h:156
constexpr bool ForRead() const
Definition: serialize.h:979
VarIntMode
Variable-length integers: bytes are a MSB base-128 encoding of the number.
Definition: serialize.h:373
I ReadVarInt(Stream &is)
Definition: serialize.h:420
uint16_t le16toh(uint16_t little_endian_16bits)
Definition: endian.h:177
void ser_writedata32be(Stream &s, uint32_t obj)
Definition: serialize.h:75
double ser_uint64_to_double(uint64_t y)
Definition: serialize.h:135
Support for SERIALIZE_METHODS and READWRITE macro.
Definition: serialize.h:973
constexpr bool ForRead() const
Definition: serialize.h:975
void SerializeMany(Stream &s)
Definition: serialize.h:1035
Implements a drop-in replacement for std::vector<T> which stores up to N elements directly (without h...
Definition: prevector.h:37
uint64_t ser_readdata64(Stream &s)
Definition: serialize.h:115
void Ser(Stream &s, I v)
Definition: serialize.h:502
Default formatter.
Definition: serialize.h:708
#define X(name)
Definition: net.cpp:515
void seek(size_t _nSize)
Pretend _nSize bytes are written, without specifying them.
Definition: serialize.h:1015
uint16_t htole16(uint16_t host_16bits)
Definition: endian.h:163
void Ser(Stream &s, const std::string &v)
Definition: serialize.h:569
uint16_t ser_readdata16be(Stream &s)
Definition: serialize.h:97
static const unsigned int MAX_SIZE
Definition: serialize.h:27
uint64_t le64toh(uint64_t little_endian_64bits)
Definition: endian.h:233
uint16_t be16toh(uint16_t big_endian_16bits)
Definition: endian.h:170
constexpr C * data() const noexcept
Definition: span.h:140
int GetVersion() const
Definition: serialize.h:1031
void * memcpy(void *a, const void *b, size_t c)
bool empty() const
Definition: prevector.h:286
Serialization wrapper class for custom integers and enums.
Definition: serialize.h:497
uint64_t htole64(uint64_t host_64bits)
Definition: endian.h:219
void write(const char *psz, size_t _nSize)
Definition: serialize.h:1009
uint32_t ser_readdata32be(Stream &s)
Definition: serialize.h:109
void SerWrite(Stream &s, CSerActionSerialize ser_action, Type &&obj, Fn &&fn)
Definition: serialize.h:1082
unsigned int GetSizeOfVarInt(I n)
Definition: serialize.h:385
size_t GetSerializeSizeMany(int nVersion, const T &... t)
Definition: serialize.h:1110
void Unserialize(Stream &s, char &a)
Definition: serialize.h:241
void ser_writedata16be(Stream &s, uint16_t obj)
Definition: serialize.h:65
size_type size() const
Definition: prevector.h:282
void Unserialize(Stream &s)
Definition: serialize.h:451
void Unser(Stream &s, std::string &v)
Definition: serialize.h:558
Formatter to serialize/deserialize vector elements using another formatter.
Definition: serialize.h:589
A Span is an object that can refer to a contiguous sequence of objects.
Definition: span.h:82
void Serialize_impl(Stream &os, const prevector< N, T > &v, const unsigned char &)
prevector prevectors of unsigned char are a special case and are intended to be serialized as a singl...
Definition: serialize.h:747
void Ser(Stream &s, I v)
Definition: serialize.h:545
void Unserialize_impl(Stream &is, prevector< N, T > &v, const unsigned char &)
Definition: serialize.h:768
T m_object
Definition: serialize.h:445
auto it
Definition: validation.cpp:383
uint32_t le32toh(uint32_t little_endian_32bits)
Definition: endian.h:205
static void Unser(Stream &s, T &t)
Definition: serialize.h:714
size_t nSize
Definition: serialize.h:1003
static void Ser(Stream &s, const T &t)
Definition: serialize.h:711
void Ser(Stream &s, I v)
Definition: serialize.h:476
void ser_writedata8(Stream &s, uint8_t obj)
Definition: serialize.h:56
float ser_uint32_to_float(uint32_t y)
Definition: serialize.h:142
Wrapper(T obj)
Definition: serialize.h:449