Bitcoin Core  22.99.0
P2P Digital Currency
serialize.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2021 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #ifndef BITCOIN_SERIALIZE_H
7 #define BITCOIN_SERIALIZE_H
8 
9 #include <compat/endian.h>
10 
11 #include <algorithm>
12 #include <cstdint>
13 #include <cstring>
14 #include <ios>
15 #include <limits>
16 #include <map>
17 #include <memory>
18 #include <set>
19 #include <string>
20 #include <string.h>
21 #include <utility>
22 #include <vector>
23 
24 #include <prevector.h>
25 #include <span.h>
26 
31 static constexpr uint64_t MAX_SIZE = 0x02000000;
32 
34 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
35 
47 struct deserialize_type {};
49 
50 /*
51  * Lowest-level serialization and conversion.
52  */
53 template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
54 {
55  s.write(AsBytes(Span{&obj, 1}));
56 }
57 template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
58 {
59  obj = htole16(obj);
60  s.write(AsBytes(Span{&obj, 1}));
61 }
62 template<typename Stream> inline void ser_writedata16be(Stream &s, uint16_t obj)
63 {
64  obj = htobe16(obj);
65  s.write(AsBytes(Span{&obj, 1}));
66 }
67 template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
68 {
69  obj = htole32(obj);
70  s.write(AsBytes(Span{&obj, 1}));
71 }
72 template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj)
73 {
74  obj = htobe32(obj);
75  s.write(AsBytes(Span{&obj, 1}));
76 }
77 template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
78 {
79  obj = htole64(obj);
80  s.write(AsBytes(Span{&obj, 1}));
81 }
82 template<typename Stream> inline uint8_t ser_readdata8(Stream &s)
83 {
84  uint8_t obj;
85  s.read(AsWritableBytes(Span{&obj, 1}));
86  return obj;
87 }
88 template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
89 {
90  uint16_t obj;
91  s.read(AsWritableBytes(Span{&obj, 1}));
92  return le16toh(obj);
93 }
94 template<typename Stream> inline uint16_t ser_readdata16be(Stream &s)
95 {
96  uint16_t obj;
97  s.read(AsWritableBytes(Span{&obj, 1}));
98  return be16toh(obj);
99 }
100 template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
101 {
102  uint32_t obj;
103  s.read(AsWritableBytes(Span{&obj, 1}));
104  return le32toh(obj);
105 }
106 template<typename Stream> inline uint32_t ser_readdata32be(Stream &s)
107 {
108  uint32_t obj;
109  s.read(AsWritableBytes(Span{&obj, 1}));
110  return be32toh(obj);
111 }
112 template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
113 {
114  uint64_t obj;
115  s.read(AsWritableBytes(Span{&obj, 1}));
116  return le64toh(obj);
117 }
118 
119 
121 //
122 // Templates for serializing to anything that looks like a stream,
123 // i.e. anything that supports .read(Span<std::byte>) and .write(Span<const std::byte>)
124 //
125 
126 class CSizeComputer;
127 
128 enum
129 {
130  // primary actions
131  SER_NETWORK = (1 << 0),
132  SER_DISK = (1 << 1),
133  SER_GETHASH = (1 << 2),
134 };
135 
137 template<typename X> X& ReadWriteAsHelper(X& x) { return x; }
138 template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; }
139 
140 #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__))
141 #define READWRITEAS(type, obj) (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper<type>(obj)))
142 #define SER_READ(obj, code) ::SerRead(s, ser_action, obj, [&](Stream& s, typename std::remove_const<Type>::type& obj) { code; })
143 #define SER_WRITE(obj, code) ::SerWrite(s, ser_action, obj, [&](Stream& s, const Type& obj) { code; })
144 
161 #define FORMATTER_METHODS(cls, obj) \
162  template<typename Stream> \
163  static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, CSerActionSerialize()); } \
164  template<typename Stream> \
165  static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, CSerActionUnserialize()); } \
166  template<typename Stream, typename Type, typename Operation> \
167  static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \
168 
169 
176 #define SERIALIZE_METHODS(cls, obj) \
177  template<typename Stream> \
178  void Serialize(Stream& s) const \
179  { \
180  static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \
181  Ser(s, *this); \
182  } \
183  template<typename Stream> \
184  void Unserialize(Stream& s) \
185  { \
186  static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \
187  Unser(s, *this); \
188  } \
189  FORMATTER_METHODS(cls, obj)
190 
191 #ifndef CHAR_EQUALS_INT8
192 template <typename Stream> void Serialize(Stream&, char) = delete; // char serialization forbidden. Use uint8_t or int8_t
193 #endif
194 template<typename Stream> inline void Serialize(Stream& s, int8_t a ) { ser_writedata8(s, a); }
195 template<typename Stream> inline void Serialize(Stream& s, uint8_t a ) { ser_writedata8(s, a); }
196 template<typename Stream> inline void Serialize(Stream& s, int16_t a ) { ser_writedata16(s, a); }
197 template<typename Stream> inline void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); }
198 template<typename Stream> inline void Serialize(Stream& s, int32_t a ) { ser_writedata32(s, a); }
199 template<typename Stream> inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); }
200 template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); }
201 template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); }
202 template<typename Stream, int N> inline void Serialize(Stream& s, const char (&a)[N]) { s.write(MakeByteSpan(a)); }
203 template<typename Stream, int N> inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(MakeByteSpan(a)); }
204 template<typename Stream> inline void Serialize(Stream& s, const Span<const unsigned char>& span) { s.write(AsBytes(span)); }
205 template<typename Stream> inline void Serialize(Stream& s, const Span<unsigned char>& span) { s.write(AsBytes(span)); }
206 
207 #ifndef CHAR_EQUALS_INT8
208 template <typename Stream> void Unserialize(Stream&, char) = delete; // char serialization forbidden. Use uint8_t or int8_t
209 #endif
210 template<typename Stream> inline void Unserialize(Stream& s, int8_t& a ) { a = ser_readdata8(s); }
211 template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a ) { a = ser_readdata8(s); }
212 template<typename Stream> inline void Unserialize(Stream& s, int16_t& a ) { a = ser_readdata16(s); }
213 template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); }
214 template<typename Stream> inline void Unserialize(Stream& s, int32_t& a ) { a = ser_readdata32(s); }
215 template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); }
216 template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); }
217 template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); }
218 template<typename Stream, int N> inline void Unserialize(Stream& s, char (&a)[N]) { s.read(MakeWritableByteSpan(a)); }
219 template<typename Stream, int N> inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(MakeWritableByteSpan(a)); }
220 template<typename Stream> inline void Unserialize(Stream& s, Span<unsigned char>& span) { s.read(AsWritableBytes(span)); }
221 
222 template <typename Stream> inline void Serialize(Stream& s, bool a) { uint8_t f = a; ser_writedata8(s, f); }
223 template <typename Stream> inline void Unserialize(Stream& s, bool& a) { uint8_t f = ser_readdata8(s); a = f; }
224 
225 
233 inline unsigned int GetSizeOfCompactSize(uint64_t nSize)
234 {
235  if (nSize < 253) return sizeof(unsigned char);
236  else if (nSize <= std::numeric_limits<uint16_t>::max()) return sizeof(unsigned char) + sizeof(uint16_t);
237  else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int);
238  else return sizeof(unsigned char) + sizeof(uint64_t);
239 }
240 
241 inline void WriteCompactSize(CSizeComputer& os, uint64_t nSize);
242 
243 template<typename Stream>
244 void WriteCompactSize(Stream& os, uint64_t nSize)
245 {
246  if (nSize < 253)
247  {
248  ser_writedata8(os, nSize);
249  }
250  else if (nSize <= std::numeric_limits<uint16_t>::max())
251  {
252  ser_writedata8(os, 253);
253  ser_writedata16(os, nSize);
254  }
255  else if (nSize <= std::numeric_limits<unsigned int>::max())
256  {
257  ser_writedata8(os, 254);
258  ser_writedata32(os, nSize);
259  }
260  else
261  {
262  ser_writedata8(os, 255);
263  ser_writedata64(os, nSize);
264  }
265  return;
266 }
267 
274 template<typename Stream>
275 uint64_t ReadCompactSize(Stream& is, bool range_check = true)
276 {
277  uint8_t chSize = ser_readdata8(is);
278  uint64_t nSizeRet = 0;
279  if (chSize < 253)
280  {
281  nSizeRet = chSize;
282  }
283  else if (chSize == 253)
284  {
285  nSizeRet = ser_readdata16(is);
286  if (nSizeRet < 253)
287  throw std::ios_base::failure("non-canonical ReadCompactSize()");
288  }
289  else if (chSize == 254)
290  {
291  nSizeRet = ser_readdata32(is);
292  if (nSizeRet < 0x10000u)
293  throw std::ios_base::failure("non-canonical ReadCompactSize()");
294  }
295  else
296  {
297  nSizeRet = ser_readdata64(is);
298  if (nSizeRet < 0x100000000ULL)
299  throw std::ios_base::failure("non-canonical ReadCompactSize()");
300  }
301  if (range_check && nSizeRet > MAX_SIZE) {
302  throw std::ios_base::failure("ReadCompactSize(): size too large");
303  }
304  return nSizeRet;
305 }
306 
342 
343 template <VarIntMode Mode, typename I>
345  constexpr CheckVarIntMode()
346  {
347  static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned<I>::value, "Unsigned type required with mode DEFAULT.");
348  static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed<I>::value, "Signed type required with mode NONNEGATIVE_SIGNED.");
349  }
350 };
351 
352 template<VarIntMode Mode, typename I>
353 inline unsigned int GetSizeOfVarInt(I n)
354 {
356  int nRet = 0;
357  while(true) {
358  nRet++;
359  if (n <= 0x7F)
360  break;
361  n = (n >> 7) - 1;
362  }
363  return nRet;
364 }
365 
366 template<typename I>
367 inline void WriteVarInt(CSizeComputer& os, I n);
368 
369 template<typename Stream, VarIntMode Mode, typename I>
370 void WriteVarInt(Stream& os, I n)
371 {
373  unsigned char tmp[(sizeof(n)*8+6)/7];
374  int len=0;
375  while(true) {
376  tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
377  if (n <= 0x7F)
378  break;
379  n = (n >> 7) - 1;
380  len++;
381  }
382  do {
383  ser_writedata8(os, tmp[len]);
384  } while(len--);
385 }
386 
387 template<typename Stream, VarIntMode Mode, typename I>
388 I ReadVarInt(Stream& is)
389 {
391  I n = 0;
392  while(true) {
393  unsigned char chData = ser_readdata8(is);
394  if (n > (std::numeric_limits<I>::max() >> 7)) {
395  throw std::ios_base::failure("ReadVarInt(): size too large");
396  }
397  n = (n << 7) | (chData & 0x7F);
398  if (chData & 0x80) {
399  if (n == std::numeric_limits<I>::max()) {
400  throw std::ios_base::failure("ReadVarInt(): size too large");
401  }
402  n++;
403  } else {
404  return n;
405  }
406  }
407 }
408 
410 template<typename Formatter, typename T>
411 class Wrapper
412 {
413  static_assert(std::is_lvalue_reference<T>::value, "Wrapper needs an lvalue reference type T");
414 protected:
415  T m_object;
416 public:
417  explicit Wrapper(T obj) : m_object(obj) {}
418  template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); }
419  template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); }
420 };
421 
432 template<typename Formatter, typename T>
433 static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); }
434 
435 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
436 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
437 #define COMPACTSIZE(obj) Using<CompactSizeFormatter<true>>(obj)
438 #define LIMITED_STRING(obj,n) Using<LimitedStringFormatter<n>>(obj)
439 
441 template<VarIntMode Mode>
443 {
444  template<typename Stream, typename I> void Ser(Stream &s, I v)
445  {
446  WriteVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s, v);
447  }
448 
449  template<typename Stream, typename I> void Unser(Stream& s, I& v)
450  {
451  v = ReadVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s);
452  }
453 };
454 
464 template<int Bytes, bool BigEndian = false>
466 {
467  static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
468  static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
469 
470  template <typename Stream, typename I> void Ser(Stream& s, I v)
471  {
472  if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
473  if (BigEndian) {
474  uint64_t raw = htobe64(v);
475  s.write({BytePtr(&raw) + 8 - Bytes, Bytes});
476  } else {
477  uint64_t raw = htole64(v);
478  s.write({BytePtr(&raw), Bytes});
479  }
480  }
481 
482  template <typename Stream, typename I> void Unser(Stream& s, I& v)
483  {
484  using U = typename std::conditional<std::is_enum<I>::value, std::underlying_type<I>, std::common_type<I>>::type::type;
485  static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small");
486  uint64_t raw = 0;
487  if (BigEndian) {
488  s.read({BytePtr(&raw) + 8 - Bytes, Bytes});
489  v = static_cast<I>(be64toh(raw));
490  } else {
491  s.read({BytePtr(&raw), Bytes});
492  v = static_cast<I>(le64toh(raw));
493  }
494  }
495 };
496 
498 
500 template<bool RangeCheck>
502 {
503  template<typename Stream, typename I>
504  void Unser(Stream& s, I& v)
505  {
506  uint64_t n = ReadCompactSize<Stream>(s, RangeCheck);
507  if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
508  throw std::ios_base::failure("CompactSize exceeds limit of type");
509  }
510  v = n;
511  }
512 
513  template<typename Stream, typename I>
514  void Ser(Stream& s, I v)
515  {
516  static_assert(std::is_unsigned<I>::value, "CompactSize only supported for unsigned integers");
517  static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
518 
519  WriteCompactSize<Stream>(s, v);
520  }
521 };
522 
524 {
525 protected:
526  uint64_t n;
527 public:
528  explicit CompactSizeWriter(uint64_t n_in) : n(n_in) { }
529 
530  template<typename Stream>
531  void Serialize(Stream &s) const {
532  WriteCompactSize<Stream>(s, n);
533  }
534 };
535 
536 template<size_t Limit>
538 {
539  template<typename Stream>
540  void Unser(Stream& s, std::string& v)
541  {
542  size_t size = ReadCompactSize(s);
543  if (size > Limit) {
544  throw std::ios_base::failure("String length limit exceeded");
545  }
546  v.resize(size);
547  if (size != 0) s.read(MakeWritableByteSpan(v));
548  }
549 
550  template<typename Stream>
551  void Ser(Stream& s, const std::string& v)
552  {
553  s << v;
554  }
555 };
556 
570 template<class Formatter>
572 {
573  template<typename Stream, typename V>
574  void Ser(Stream& s, const V& v)
575  {
576  Formatter formatter;
577  WriteCompactSize(s, v.size());
578  for (const typename V::value_type& elem : v) {
579  formatter.Ser(s, elem);
580  }
581  }
582 
583  template<typename Stream, typename V>
584  void Unser(Stream& s, V& v)
585  {
586  Formatter formatter;
587  v.clear();
588  size_t size = ReadCompactSize(s);
589  size_t allocated = 0;
590  while (allocated < size) {
591  // For DoS prevention, do not blindly allocate as much as the stream claims to contain.
592  // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
593  // X MiB of data to make us allocate X+5 Mib.
594  static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
595  allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
596  v.reserve(allocated);
597  while (v.size() < allocated) {
598  v.emplace_back();
599  formatter.Unser(s, v.back());
600  }
601  }
602  };
603 };
604 
612 template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str);
613 template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str);
614 
619 template<typename Stream, unsigned int N, typename T> void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&);
620 template<typename Stream, unsigned int N, typename T, typename V> void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&);
621 template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v);
622 template<typename Stream, unsigned int N, typename T> void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&);
623 template<typename Stream, unsigned int N, typename T, typename V> void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&);
624 template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v);
625 
630 template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&);
631 template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&);
632 template<typename Stream, typename T, typename A, typename V> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&);
633 template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v);
634 template<typename Stream, typename T, typename A> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&);
635 template<typename Stream, typename T, typename A, typename V> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&);
636 template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v);
637 
641 template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item);
642 template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item);
643 
647 template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m);
648 template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m);
649 
653 template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m);
654 template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m);
655 
659 template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p);
660 template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p);
661 
665 template<typename Stream, typename T> void Serialize(Stream& os, const std::unique_ptr<const T>& p);
666 template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p);
667 
668 
669 
673 template<typename Stream, typename T>
674 inline void Serialize(Stream& os, const T& a)
675 {
676  a.Serialize(os);
677 }
678 
679 template<typename Stream, typename T>
680 inline void Unserialize(Stream& is, T&& a)
681 {
682  a.Unserialize(is);
683 }
684 
691 {
692  template<typename Stream, typename T>
693  static void Ser(Stream& s, const T& t) { Serialize(s, t); }
694 
695  template<typename Stream, typename T>
696  static void Unser(Stream& s, T& t) { Unserialize(s, t); }
697 };
698 
699 
700 
701 
702 
706 template<typename Stream, typename C>
707 void Serialize(Stream& os, const std::basic_string<C>& str)
708 {
709  WriteCompactSize(os, str.size());
710  if (!str.empty())
711  os.write(MakeByteSpan(str));
712 }
713 
714 template<typename Stream, typename C>
715 void Unserialize(Stream& is, std::basic_string<C>& str)
716 {
717  unsigned int nSize = ReadCompactSize(is);
718  str.resize(nSize);
719  if (nSize != 0)
720  is.read(MakeWritableByteSpan(str));
721 }
722 
723 
724 
728 template<typename Stream, unsigned int N, typename T>
729 void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&)
730 {
731  WriteCompactSize(os, v.size());
732  if (!v.empty())
733  os.write(MakeByteSpan(v));
734 }
735 
736 template<typename Stream, unsigned int N, typename T, typename V>
737 void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&)
738 {
740 }
741 
742 template<typename Stream, unsigned int N, typename T>
743 inline void Serialize(Stream& os, const prevector<N, T>& v)
744 {
745  Serialize_impl(os, v, T());
746 }
747 
748 
749 template<typename Stream, unsigned int N, typename T>
750 void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&)
751 {
752  // Limit size per read so bogus size value won't cause out of memory
753  v.clear();
754  unsigned int nSize = ReadCompactSize(is);
755  unsigned int i = 0;
756  while (i < nSize)
757  {
758  unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
759  v.resize_uninitialized(i + blk);
760  is.read(AsWritableBytes(Span{&v[i], blk}));
761  i += blk;
762  }
763 }
764 
765 template<typename Stream, unsigned int N, typename T, typename V>
766 void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&)
767 {
769 }
770 
771 template<typename Stream, unsigned int N, typename T>
772 inline void Unserialize(Stream& is, prevector<N, T>& v)
773 {
774  Unserialize_impl(is, v, T());
775 }
776 
777 
778 
782 template<typename Stream, typename T, typename A>
783 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&)
784 {
785  WriteCompactSize(os, v.size());
786  if (!v.empty())
787  os.write(MakeByteSpan(v));
788 }
789 
790 template<typename Stream, typename T, typename A>
791 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&)
792 {
793  // A special case for std::vector<bool>, as dereferencing
794  // std::vector<bool>::const_iterator does not result in a const bool&
795  // due to std::vector's special casing for bool arguments.
796  WriteCompactSize(os, v.size());
797  for (bool elem : v) {
798  ::Serialize(os, elem);
799  }
800 }
801 
802 template<typename Stream, typename T, typename A, typename V>
803 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&)
804 {
806 }
807 
808 template<typename Stream, typename T, typename A>
809 inline void Serialize(Stream& os, const std::vector<T, A>& v)
810 {
811  Serialize_impl(os, v, T());
812 }
813 
814 
815 template<typename Stream, typename T, typename A>
816 void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&)
817 {
818  // Limit size per read so bogus size value won't cause out of memory
819  v.clear();
820  unsigned int nSize = ReadCompactSize(is);
821  unsigned int i = 0;
822  while (i < nSize)
823  {
824  unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
825  v.resize(i + blk);
826  is.read(AsWritableBytes(Span{&v[i], blk}));
827  i += blk;
828  }
829 }
830 
831 template<typename Stream, typename T, typename A, typename V>
832 void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&)
833 {
835 }
836 
837 template<typename Stream, typename T, typename A>
838 inline void Unserialize(Stream& is, std::vector<T, A>& v)
839 {
840  Unserialize_impl(is, v, T());
841 }
842 
843 
844 
848 template<typename Stream, typename K, typename T>
849 void Serialize(Stream& os, const std::pair<K, T>& item)
850 {
851  Serialize(os, item.first);
852  Serialize(os, item.second);
853 }
854 
855 template<typename Stream, typename K, typename T>
856 void Unserialize(Stream& is, std::pair<K, T>& item)
857 {
858  Unserialize(is, item.first);
859  Unserialize(is, item.second);
860 }
861 
862 
863 
867 template<typename Stream, typename K, typename T, typename Pred, typename A>
868 void Serialize(Stream& os, const std::map<K, T, Pred, A>& m)
869 {
870  WriteCompactSize(os, m.size());
871  for (const auto& entry : m)
872  Serialize(os, entry);
873 }
874 
875 template<typename Stream, typename K, typename T, typename Pred, typename A>
876 void Unserialize(Stream& is, std::map<K, T, Pred, A>& m)
877 {
878  m.clear();
879  unsigned int nSize = ReadCompactSize(is);
880  typename std::map<K, T, Pred, A>::iterator mi = m.begin();
881  for (unsigned int i = 0; i < nSize; i++)
882  {
883  std::pair<K, T> item;
884  Unserialize(is, item);
885  mi = m.insert(mi, item);
886  }
887 }
888 
889 
890 
894 template<typename Stream, typename K, typename Pred, typename A>
895 void Serialize(Stream& os, const std::set<K, Pred, A>& m)
896 {
897  WriteCompactSize(os, m.size());
898  for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it)
899  Serialize(os, (*it));
900 }
901 
902 template<typename Stream, typename K, typename Pred, typename A>
903 void Unserialize(Stream& is, std::set<K, Pred, A>& m)
904 {
905  m.clear();
906  unsigned int nSize = ReadCompactSize(is);
907  typename std::set<K, Pred, A>::iterator it = m.begin();
908  for (unsigned int i = 0; i < nSize; i++)
909  {
910  K key;
911  Unserialize(is, key);
912  it = m.insert(it, key);
913  }
914 }
915 
916 
917 
921 template<typename Stream, typename T> void
922 Serialize(Stream& os, const std::unique_ptr<const T>& p)
923 {
924  Serialize(os, *p);
925 }
926 
927 template<typename Stream, typename T>
928 void Unserialize(Stream& is, std::unique_ptr<const T>& p)
929 {
930  p.reset(new T(deserialize, is));
931 }
932 
933 
934 
938 template<typename Stream, typename T> void
939 Serialize(Stream& os, const std::shared_ptr<const T>& p)
940 {
941  Serialize(os, *p);
942 }
943 
944 template<typename Stream, typename T>
945 void Unserialize(Stream& is, std::shared_ptr<const T>& p)
946 {
947  p = std::make_shared<const T>(deserialize, is);
948 }
949 
950 
951 
956 {
957  constexpr bool ForRead() const { return false; }
958 };
960 {
961  constexpr bool ForRead() const { return true; }
962 };
963 
964 
965 
966 
967 
968 
969 
970 
971 /* ::GetSerializeSize implementations
972  *
973  * Computing the serialized size of objects is done through a special stream
974  * object of type CSizeComputer, which only records the number of bytes written
975  * to it.
976  *
977  * If your Serialize or SerializationOp method has non-trivial overhead for
978  * serialization, it may be worthwhile to implement a specialized version for
979  * CSizeComputer, which uses the s.seek() method to record bytes that would
980  * be written instead.
981  */
983 {
984 protected:
985  size_t nSize;
986 
987  const int nVersion;
988 public:
989  explicit CSizeComputer(int nVersionIn) : nSize(0), nVersion(nVersionIn) {}
990 
992  {
993  this->nSize += src.size();
994  }
995 
997  void seek(size_t _nSize)
998  {
999  this->nSize += _nSize;
1000  }
1001 
1002  template<typename T>
1004  {
1005  ::Serialize(*this, obj);
1006  return (*this);
1007  }
1008 
1009  size_t size() const {
1010  return nSize;
1011  }
1012 
1013  int GetVersion() const { return nVersion; }
1014 };
1015 
1016 template<typename Stream>
1017 void SerializeMany(Stream& s)
1018 {
1019 }
1020 
1021 template<typename Stream, typename Arg, typename... Args>
1022 void SerializeMany(Stream& s, const Arg& arg, const Args&... args)
1023 {
1024  ::Serialize(s, arg);
1025  ::SerializeMany(s, args...);
1026 }
1027 
1028 template<typename Stream>
1029 inline void UnserializeMany(Stream& s)
1030 {
1031 }
1032 
1033 template<typename Stream, typename Arg, typename... Args>
1034 inline void UnserializeMany(Stream& s, Arg&& arg, Args&&... args)
1035 {
1036  ::Unserialize(s, arg);
1037  ::UnserializeMany(s, args...);
1038 }
1039 
1040 template<typename Stream, typename... Args>
1041 inline void SerReadWriteMany(Stream& s, CSerActionSerialize ser_action, const Args&... args)
1042 {
1043  ::SerializeMany(s, args...);
1044 }
1045 
1046 template<typename Stream, typename... Args>
1047 inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&&... args)
1048 {
1049  ::UnserializeMany(s, args...);
1050 }
1051 
1052 template<typename Stream, typename Type, typename Fn>
1053 inline void SerRead(Stream& s, CSerActionSerialize ser_action, Type&&, Fn&&)
1054 {
1055 }
1056 
1057 template<typename Stream, typename Type, typename Fn>
1058 inline void SerRead(Stream& s, CSerActionUnserialize ser_action, Type&& obj, Fn&& fn)
1059 {
1060  fn(s, std::forward<Type>(obj));
1061 }
1062 
1063 template<typename Stream, typename Type, typename Fn>
1064 inline void SerWrite(Stream& s, CSerActionSerialize ser_action, Type&& obj, Fn&& fn)
1065 {
1066  fn(s, std::forward<Type>(obj));
1067 }
1068 
1069 template<typename Stream, typename Type, typename Fn>
1070 inline void SerWrite(Stream& s, CSerActionUnserialize ser_action, Type&&, Fn&&)
1071 {
1072 }
1073 
1074 template<typename I>
1075 inline void WriteVarInt(CSizeComputer &s, I n)
1076 {
1077  s.seek(GetSizeOfVarInt<I>(n));
1078 }
1079 
1080 inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize)
1081 {
1082  s.seek(GetSizeOfCompactSize(nSize));
1083 }
1084 
1085 template <typename T>
1086 size_t GetSerializeSize(const T& t, int nVersion = 0)
1087 {
1088  return (CSizeComputer(nVersion) << t).size();
1089 }
1090 
1091 template <typename... T>
1092 size_t GetSerializeSizeMany(int nVersion, const T&... t)
1093 {
1094  CSizeComputer sc(nVersion);
1095  SerializeMany(sc, t...);
1096  return sc.size();
1097 }
1098 
1099 #endif // BITCOIN_SERIALIZE_H
DefaultFormatter
Default formatter.
Definition: serialize.h:690
GetSerializeSize
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1086
htole16
uint16_t htole16(uint16_t host_16bits)
Definition: endian.h:163
MakeByteSpan
Span< const std::byte > MakeByteSpan(V &&v) noexcept
Definition: span.h:264
deserialize_type
Dummy data type to identify deserializing constructors.
Definition: serialize.h:47
GetSizeOfVarInt
unsigned int GetSizeOfVarInt(I n)
Definition: serialize.h:353
ser_readdata32
uint32_t ser_readdata32(Stream &s)
Definition: serialize.h:100
ser_writedata16be
void ser_writedata16be(Stream &s, uint16_t obj)
Definition: serialize.h:62
SER_DISK
@ SER_DISK
Definition: serialize.h:132
CSizeComputer::GetVersion
int GetVersion() const
Definition: serialize.h:1013
BytePtr
const std::byte * BytePtr(const void *data)
Convert a data pointer to a std::byte data pointer.
Definition: span.h:248
DefaultFormatter::Unser
static void Unser(Stream &s, T &t)
Definition: serialize.h:696
VarIntFormatter
Serialization wrapper class for integers in VarInt format.
Definition: serialize.h:442
htole64
uint64_t htole64(uint64_t host_64bits)
Definition: endian.h:219
DefaultFormatter::Ser
static void Ser(Stream &s, const T &t)
Definition: serialize.h:693
ser_writedata64
void ser_writedata64(Stream &s, uint64_t obj)
Definition: serialize.h:77
string.h
prevector::clear
void clear()
Definition: prevector.h:343
ReadVarInt
I ReadVarInt(Stream &is)
Definition: serialize.h:388
VectorFormatter
Formatter to serialize/deserialize vector elements using another formatter.
Definition: serialize.h:571
le32toh
uint32_t le32toh(uint32_t little_endian_32bits)
Definition: endian.h:205
GetSerializeSizeMany
size_t GetSerializeSizeMany(int nVersion, const T &... t)
Definition: serialize.h:1092
SerWrite
void SerWrite(Stream &s, CSerActionSerialize ser_action, Type &&obj, Fn &&fn)
Definition: serialize.h:1064
VarIntMode::NONNEGATIVE_SIGNED
@ NONNEGATIVE_SIGNED
ReadWriteAsHelper
X & ReadWriteAsHelper(X &x)
Convert the reference base type to X, without changing constness or reference type.
Definition: serialize.h:137
CSizeComputer::size
size_t size() const
Definition: serialize.h:1009
CSizeComputer::operator<<
CSizeComputer & operator<<(const T &obj)
Definition: serialize.h:1003
CompactSizeWriter::n
uint64_t n
Definition: serialize.h:526
ReadCompactSize
uint64_t ReadCompactSize(Stream &is, bool range_check=true)
Decode a CompactSize-encoded variable-length integer.
Definition: serialize.h:275
CustomUintFormatter
Serialization wrapper class for custom integers and enums.
Definition: serialize.h:465
AsWritableBytes
Span< std::byte > AsWritableBytes(Span< T > s) noexcept
Definition: span.h:258
Unserialize_impl
void Unserialize_impl(Stream &is, prevector< N, T > &v, const unsigned char &)
Definition: serialize.h:750
CheckVarIntMode
Definition: serialize.h:344
CSerActionSerialize
Support for SERIALIZE_METHODS and READWRITE macro.
Definition: serialize.h:955
be64toh
uint64_t be64toh(uint64_t big_endian_64bits)
Definition: endian.h:226
Unserialize
void Unserialize(Stream &, char)=delete
Serialize
void Serialize(Stream &, char)=delete
CSerActionUnserialize
Definition: serialize.h:959
CompactSizeWriter::Serialize
void Serialize(Stream &s) const
Definition: serialize.h:531
be32toh
uint32_t be32toh(uint32_t big_endian_32bits)
Definition: endian.h:198
SerRead
void SerRead(Stream &s, CSerActionSerialize ser_action, Type &&, Fn &&)
Definition: serialize.h:1053
CustomUintFormatter::Unser
void Unser(Stream &s, I &v)
Definition: serialize.h:482
CSizeComputer::nVersion
const int nVersion
Definition: serialize.h:987
Span::size
constexpr std::size_t size() const noexcept
Definition: span.h:186
Span
A Span is an object that can refer to a contiguous sequence of objects.
Definition: span.h:96
CSerActionUnserialize::ForRead
constexpr bool ForRead() const
Definition: serialize.h:961
ser_readdata32be
uint32_t ser_readdata32be(Stream &s)
Definition: serialize.h:106
MAX_SIZE
static constexpr uint64_t MAX_SIZE
The maximum size of a serialized object in bytes or number of elements (for eg vectors) when the size...
Definition: serialize.h:31
MAX_VECTOR_ALLOCATE
static const unsigned int MAX_VECTOR_ALLOCATE
Maximum amount of memory (in bytes) to allocate at once when deserializing vectors.
Definition: serialize.h:34
LimitedStringFormatter
Definition: serialize.h:537
CSizeComputer
Definition: serialize.h:982
CompactSizeWriter::CompactSizeWriter
CompactSizeWriter(uint64_t n_in)
Definition: serialize.h:528
LimitedStringFormatter::Ser
void Ser(Stream &s, const std::string &v)
Definition: serialize.h:551
ser_readdata16
uint16_t ser_readdata16(Stream &s)
Definition: serialize.h:88
WriteCompactSize
void WriteCompactSize(CSizeComputer &os, uint64_t nSize)
Definition: serialize.h:1080
span.h
le16toh
uint16_t le16toh(uint16_t little_endian_16bits)
Definition: endian.h:177
Serialize_impl
void Serialize_impl(Stream &os, const prevector< N, T > &v, const unsigned char &)
prevector prevectors of unsigned char are a special case and are intended to be serialized as a singl...
Definition: serialize.h:729
htobe32
uint32_t htobe32(uint32_t host_32bits)
Definition: endian.h:184
ser_readdata16be
uint16_t ser_readdata16be(Stream &s)
Definition: serialize.h:94
be16toh
uint16_t be16toh(uint16_t big_endian_16bits)
Definition: endian.h:170
CSerActionSerialize::ForRead
constexpr bool ForRead() const
Definition: serialize.h:957
ser_readdata8
uint8_t ser_readdata8(Stream &s)
Definition: serialize.h:82
CompactSizeFormatter
Formatter for integers in CompactSize format.
Definition: serialize.h:501
GetSizeOfCompactSize
unsigned int GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 byt...
Definition: serialize.h:233
htobe16
uint16_t htobe16(uint16_t host_16bits)
Definition: endian.h:156
VarIntMode
VarIntMode
Variable-length integers: bytes are a MSB base-128 encoding of the number.
Definition: serialize.h:341
CSizeComputer::CSizeComputer
CSizeComputer(int nVersionIn)
Definition: serialize.h:989
UnserializeMany
void UnserializeMany(Stream &s)
Definition: serialize.h:1029
SerializeMany
void SerializeMany(Stream &s)
Definition: serialize.h:1017
CheckVarIntMode::CheckVarIntMode
constexpr CheckVarIntMode()
Definition: serialize.h:345
SerReadWriteMany
void SerReadWriteMany(Stream &s, CSerActionSerialize ser_action, const Args &... args)
Definition: serialize.h:1041
ser_writedata8
void ser_writedata8(Stream &s, uint8_t obj)
Definition: serialize.h:53
CustomUintFormatter::Ser
void Ser(Stream &s, I v)
Definition: serialize.h:470
Wrapper::Unserialize
void Unserialize(Stream &s)
Definition: serialize.h:419
htole32
uint32_t htole32(uint32_t host_32bits)
Definition: endian.h:191
ser_writedata32
void ser_writedata32(Stream &s, uint32_t obj)
Definition: serialize.h:67
VarIntFormatter::Unser
void Unser(Stream &s, I &v)
Definition: serialize.h:449
CSizeComputer::write
void write(Span< const std::byte > src)
Definition: serialize.h:991
CompactSizeFormatter::Ser
void Ser(Stream &s, I v)
Definition: serialize.h:514
AsBytes
Span< const std::byte > AsBytes(Span< T > s) noexcept
Definition: span.h:253
LimitedStringFormatter::Unser
void Unser(Stream &s, std::string &v)
Definition: serialize.h:540
prevector< N, T >
le64toh
uint64_t le64toh(uint64_t little_endian_64bits)
Definition: endian.h:233
ByteUnit::K
@ K
Wrapper::Serialize
void Serialize(Stream &s) const
Definition: serialize.h:418
WriteVarInt
void WriteVarInt(CSizeComputer &os, I n)
Definition: serialize.h:1075
X
#define X(name)
Definition: net.cpp:586
ser_writedata32be
void ser_writedata32be(Stream &s, uint32_t obj)
Definition: serialize.h:72
ser_writedata16
void ser_writedata16(Stream &s, uint16_t obj)
Definition: serialize.h:57
ByteUnit::T
@ T
Wrapper::Wrapper
Wrapper(T obj)
Definition: serialize.h:417
prevector::size
size_type size() const
Definition: prevector.h:282
prevector::resize_uninitialized
void resize_uninitialized(size_type new_size)
Definition: prevector.h:386
VectorFormatter::Unser
void Unser(Stream &s, V &v)
Definition: serialize.h:584
prevector::empty
bool empty() const
Definition: prevector.h:286
SER_NETWORK
@ SER_NETWORK
Definition: serialize.h:131
prevector.h
Wrapper::m_object
T m_object
Definition: serialize.h:413
Wrapper
Simple wrapper class to serialize objects using a formatter; used by Using().
Definition: serialize.h:411
VarIntMode::DEFAULT
@ DEFAULT
htobe64
uint64_t htobe64(uint64_t host_64bits)
Definition: endian.h:212
T
#define T(expected, seed, data)
Using
static Wrapper< Formatter, T & > Using(T &&t)
Cause serialization/deserialization of an object to be done using a specified formatter class.
Definition: serialize.h:433
VectorFormatter::Ser
void Ser(Stream &s, const V &v)
Definition: serialize.h:574
ByteUnit::m
@ m
CSizeComputer::seek
void seek(size_t _nSize)
Pretend _nSize bytes are written, without specifying them.
Definition: serialize.h:997
MakeWritableByteSpan
Span< std::byte > MakeWritableByteSpan(V &&v) noexcept
Definition: span.h:269
VarIntFormatter::Ser
void Ser(Stream &s, I v)
Definition: serialize.h:444
SER_GETHASH
@ SER_GETHASH
Definition: serialize.h:133
CompactSizeFormatter::Unser
void Unser(Stream &s, I &v)
Definition: serialize.h:504
deserialize
constexpr deserialize_type deserialize
Definition: serialize.h:48
CompactSizeWriter
Definition: serialize.h:523
ser_readdata64
uint64_t ser_readdata64(Stream &s)
Definition: serialize.h:112
ByteUnit::t
@ t
args
ArgsManager args
Definition: notifications.cpp:36
CustomUintFormatter::MAX
static constexpr uint64_t MAX
Definition: serialize.h:468
CSizeComputer::nSize
size_t nSize
Definition: serialize.h:985
endian.h