Bitcoin Core  27.99.0
P2P Digital Currency
serialize.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2022 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #ifndef BITCOIN_SERIALIZE_H
7 #define BITCOIN_SERIALIZE_H
8 
9 #include <attributes.h>
10 #include <compat/assumptions.h> // IWYU pragma: keep
11 #include <compat/endian.h>
12 #include <prevector.h>
13 #include <span.h>
14 
15 #include <algorithm>
16 #include <concepts>
17 #include <cstdint>
18 #include <cstring>
19 #include <ios>
20 #include <limits>
21 #include <map>
22 #include <memory>
23 #include <set>
24 #include <string>
25 #include <utility>
26 #include <vector>
27 
32 static constexpr uint64_t MAX_SIZE = 0x02000000;
33 
35 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
36 
48 struct deserialize_type {};
50 
51 /*
52  * Lowest-level serialization and conversion.
53  */
54 template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
55 {
56  s.write(AsBytes(Span{&obj, 1}));
57 }
58 template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
59 {
60  obj = htole16_internal(obj);
61  s.write(AsBytes(Span{&obj, 1}));
62 }
63 template<typename Stream> inline void ser_writedata16be(Stream &s, uint16_t obj)
64 {
65  obj = htobe16_internal(obj);
66  s.write(AsBytes(Span{&obj, 1}));
67 }
68 template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
69 {
70  obj = htole32_internal(obj);
71  s.write(AsBytes(Span{&obj, 1}));
72 }
73 template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj)
74 {
75  obj = htobe32_internal(obj);
76  s.write(AsBytes(Span{&obj, 1}));
77 }
78 template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
79 {
80  obj = htole64_internal(obj);
81  s.write(AsBytes(Span{&obj, 1}));
82 }
83 template<typename Stream> inline uint8_t ser_readdata8(Stream &s)
84 {
85  uint8_t obj;
86  s.read(AsWritableBytes(Span{&obj, 1}));
87  return obj;
88 }
89 template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
90 {
91  uint16_t obj;
92  s.read(AsWritableBytes(Span{&obj, 1}));
93  return le16toh_internal(obj);
94 }
95 template<typename Stream> inline uint16_t ser_readdata16be(Stream &s)
96 {
97  uint16_t obj;
98  s.read(AsWritableBytes(Span{&obj, 1}));
99  return be16toh_internal(obj);
100 }
101 template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
102 {
103  uint32_t obj;
104  s.read(AsWritableBytes(Span{&obj, 1}));
105  return le32toh_internal(obj);
106 }
107 template<typename Stream> inline uint32_t ser_readdata32be(Stream &s)
108 {
109  uint32_t obj;
110  s.read(AsWritableBytes(Span{&obj, 1}));
111  return be32toh_internal(obj);
112 }
113 template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
114 {
115  uint64_t obj;
116  s.read(AsWritableBytes(Span{&obj, 1}));
117  return le64toh_internal(obj);
118 }
119 
120 
121 class SizeComputer;
122 
143 template <class Out, class In>
145 {
146  static_assert(std::is_base_of_v<Out, In>);
147  return x;
148 }
149 template <class Out, class In>
150 const Out& AsBase(const In& x)
151 {
152  static_assert(std::is_base_of_v<Out, In>);
153  return x;
154 }
155 
156 #define READWRITE(...) (ser_action.SerReadWriteMany(s, __VA_ARGS__))
157 #define SER_READ(obj, code) ser_action.SerRead(s, obj, [&](Stream& s, typename std::remove_const<Type>::type& obj) { code; })
158 #define SER_WRITE(obj, code) ser_action.SerWrite(s, obj, [&](Stream& s, const Type& obj) { code; })
159 
176 #define FORMATTER_METHODS(cls, obj) \
177  template<typename Stream> \
178  static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, ActionSerialize{}); } \
179  template<typename Stream> \
180  static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, ActionUnserialize{}); } \
181  template<typename Stream, typename Type, typename Operation> \
182  static void SerializationOps(Type& obj, Stream& s, Operation ser_action)
183 
217 #define SER_PARAMS(type) (s.template GetParams<type>())
218 
219 #define BASE_SERIALIZE_METHODS(cls) \
220  template <typename Stream> \
221  void Serialize(Stream& s) const \
222  { \
223  static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \
224  Ser(s, *this); \
225  } \
226  template <typename Stream> \
227  void Unserialize(Stream& s) \
228  { \
229  static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \
230  Unser(s, *this); \
231  }
232 
240 #define SERIALIZE_METHODS(cls, obj) \
241  BASE_SERIALIZE_METHODS(cls) \
242  FORMATTER_METHODS(cls, obj)
243 
244 // Templates for serializing to anything that looks like a stream,
245 // i.e. anything that supports .read(Span<std::byte>) and .write(Span<const std::byte>)
246 //
247 // clang-format off
248 
249 // Typically int8_t and char are distinct types, but some systems may define int8_t
250 // in terms of char. Forbid serialization of char in the typical case, but allow it if
251 // it's the only way to describe an int8_t.
252 template<class T>
253 concept CharNotInt8 = std::same_as<T, char> && !std::same_as<T, int8_t>;
254 
255 template <typename Stream, CharNotInt8 V> void Serialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t
256 template <typename Stream> void Serialize(Stream& s, std::byte a) { ser_writedata8(s, uint8_t(a)); }
257 template<typename Stream> inline void Serialize(Stream& s, int8_t a ) { ser_writedata8(s, a); }
258 template<typename Stream> inline void Serialize(Stream& s, uint8_t a ) { ser_writedata8(s, a); }
259 template<typename Stream> inline void Serialize(Stream& s, int16_t a ) { ser_writedata16(s, a); }
260 template<typename Stream> inline void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); }
261 template<typename Stream> inline void Serialize(Stream& s, int32_t a ) { ser_writedata32(s, a); }
262 template<typename Stream> inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); }
263 template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); }
264 template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); }
265 template <typename Stream, BasicByte B, int N> void Serialize(Stream& s, const B (&a)[N]) { s.write(MakeByteSpan(a)); }
266 template <typename Stream, BasicByte B, std::size_t N> void Serialize(Stream& s, const std::array<B, N>& a) { s.write(MakeByteSpan(a)); }
267 template <typename Stream, BasicByte B> void Serialize(Stream& s, Span<B> span) { s.write(AsBytes(span)); }
268 
269 template <typename Stream, CharNotInt8 V> void Unserialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t
270 template <typename Stream> void Unserialize(Stream& s, std::byte& a) { a = std::byte{ser_readdata8(s)}; }
271 template<typename Stream> inline void Unserialize(Stream& s, int8_t& a ) { a = ser_readdata8(s); }
272 template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a ) { a = ser_readdata8(s); }
273 template<typename Stream> inline void Unserialize(Stream& s, int16_t& a ) { a = ser_readdata16(s); }
274 template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); }
275 template<typename Stream> inline void Unserialize(Stream& s, int32_t& a ) { a = ser_readdata32(s); }
276 template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); }
277 template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); }
278 template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); }
279 template <typename Stream, BasicByte B, int N> void Unserialize(Stream& s, B (&a)[N]) { s.read(MakeWritableByteSpan(a)); }
280 template <typename Stream, BasicByte B, std::size_t N> void Unserialize(Stream& s, std::array<B, N>& a) { s.read(MakeWritableByteSpan(a)); }
281 template <typename Stream, BasicByte B> void Unserialize(Stream& s, Span<B> span) { s.read(AsWritableBytes(span)); }
282 
283 template <typename Stream> inline void Serialize(Stream& s, bool a) { uint8_t f = a; ser_writedata8(s, f); }
284 template <typename Stream> inline void Unserialize(Stream& s, bool& a) { uint8_t f = ser_readdata8(s); a = f; }
285 // clang-format on
286 
287 
295 constexpr inline unsigned int GetSizeOfCompactSize(uint64_t nSize)
296 {
297  if (nSize < 253) return sizeof(unsigned char);
298  else if (nSize <= std::numeric_limits<uint16_t>::max()) return sizeof(unsigned char) + sizeof(uint16_t);
299  else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int);
300  else return sizeof(unsigned char) + sizeof(uint64_t);
301 }
302 
303 inline void WriteCompactSize(SizeComputer& os, uint64_t nSize);
304 
305 template<typename Stream>
306 void WriteCompactSize(Stream& os, uint64_t nSize)
307 {
308  if (nSize < 253)
309  {
310  ser_writedata8(os, nSize);
311  }
312  else if (nSize <= std::numeric_limits<uint16_t>::max())
313  {
314  ser_writedata8(os, 253);
315  ser_writedata16(os, nSize);
316  }
317  else if (nSize <= std::numeric_limits<unsigned int>::max())
318  {
319  ser_writedata8(os, 254);
320  ser_writedata32(os, nSize);
321  }
322  else
323  {
324  ser_writedata8(os, 255);
325  ser_writedata64(os, nSize);
326  }
327  return;
328 }
329 
336 template<typename Stream>
337 uint64_t ReadCompactSize(Stream& is, bool range_check = true)
338 {
339  uint8_t chSize = ser_readdata8(is);
340  uint64_t nSizeRet = 0;
341  if (chSize < 253)
342  {
343  nSizeRet = chSize;
344  }
345  else if (chSize == 253)
346  {
347  nSizeRet = ser_readdata16(is);
348  if (nSizeRet < 253)
349  throw std::ios_base::failure("non-canonical ReadCompactSize()");
350  }
351  else if (chSize == 254)
352  {
353  nSizeRet = ser_readdata32(is);
354  if (nSizeRet < 0x10000u)
355  throw std::ios_base::failure("non-canonical ReadCompactSize()");
356  }
357  else
358  {
359  nSizeRet = ser_readdata64(is);
360  if (nSizeRet < 0x100000000ULL)
361  throw std::ios_base::failure("non-canonical ReadCompactSize()");
362  }
363  if (range_check && nSizeRet > MAX_SIZE) {
364  throw std::ios_base::failure("ReadCompactSize(): size too large");
365  }
366  return nSizeRet;
367 }
368 
404 
405 template <VarIntMode Mode, typename I>
407  constexpr CheckVarIntMode()
408  {
409  static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned<I>::value, "Unsigned type required with mode DEFAULT.");
410  static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed<I>::value, "Signed type required with mode NONNEGATIVE_SIGNED.");
411  }
412 };
413 
414 template<VarIntMode Mode, typename I>
415 inline unsigned int GetSizeOfVarInt(I n)
416 {
418  int nRet = 0;
419  while(true) {
420  nRet++;
421  if (n <= 0x7F)
422  break;
423  n = (n >> 7) - 1;
424  }
425  return nRet;
426 }
427 
428 template<typename I>
429 inline void WriteVarInt(SizeComputer& os, I n);
430 
431 template<typename Stream, VarIntMode Mode, typename I>
432 void WriteVarInt(Stream& os, I n)
433 {
435  unsigned char tmp[(sizeof(n)*8+6)/7];
436  int len=0;
437  while(true) {
438  tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
439  if (n <= 0x7F)
440  break;
441  n = (n >> 7) - 1;
442  len++;
443  }
444  do {
445  ser_writedata8(os, tmp[len]);
446  } while(len--);
447 }
448 
449 template<typename Stream, VarIntMode Mode, typename I>
450 I ReadVarInt(Stream& is)
451 {
453  I n = 0;
454  while(true) {
455  unsigned char chData = ser_readdata8(is);
456  if (n > (std::numeric_limits<I>::max() >> 7)) {
457  throw std::ios_base::failure("ReadVarInt(): size too large");
458  }
459  n = (n << 7) | (chData & 0x7F);
460  if (chData & 0x80) {
461  if (n == std::numeric_limits<I>::max()) {
462  throw std::ios_base::failure("ReadVarInt(): size too large");
463  }
464  n++;
465  } else {
466  return n;
467  }
468  }
469 }
470 
472 template<typename Formatter, typename T>
473 class Wrapper
474 {
475  static_assert(std::is_lvalue_reference<T>::value, "Wrapper needs an lvalue reference type T");
476 protected:
478 public:
479  explicit Wrapper(T obj) : m_object(obj) {}
480  template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); }
481  template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); }
482 };
483 
494 template<typename Formatter, typename T>
495 static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); }
496 
497 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
498 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
499 #define COMPACTSIZE(obj) Using<CompactSizeFormatter<true>>(obj)
500 #define LIMITED_STRING(obj,n) Using<LimitedStringFormatter<n>>(obj)
501 
503 template<VarIntMode Mode>
505 {
506  template<typename Stream, typename I> void Ser(Stream &s, I v)
507  {
508  WriteVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s, v);
509  }
510 
511  template<typename Stream, typename I> void Unser(Stream& s, I& v)
512  {
513  v = ReadVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s);
514  }
515 };
516 
526 template<int Bytes, bool BigEndian = false>
528 {
529  static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
530  static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
531 
532  template <typename Stream, typename I> void Ser(Stream& s, I v)
533  {
534  if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
535  if (BigEndian) {
536  uint64_t raw = htobe64_internal(v);
537  s.write(AsBytes(Span{&raw, 1}).last(Bytes));
538  } else {
539  uint64_t raw = htole64_internal(v);
540  s.write(AsBytes(Span{&raw, 1}).first(Bytes));
541  }
542  }
543 
544  template <typename Stream, typename I> void Unser(Stream& s, I& v)
545  {
546  using U = typename std::conditional<std::is_enum<I>::value, std::underlying_type<I>, std::common_type<I>>::type::type;
547  static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small");
548  uint64_t raw = 0;
549  if (BigEndian) {
550  s.read(AsWritableBytes(Span{&raw, 1}).last(Bytes));
551  v = static_cast<I>(be64toh_internal(raw));
552  } else {
553  s.read(AsWritableBytes(Span{&raw, 1}).first(Bytes));
554  v = static_cast<I>(le64toh_internal(raw));
555  }
556  }
557 };
558 
560 
562 template<bool RangeCheck>
564 {
565  template<typename Stream, typename I>
566  void Unser(Stream& s, I& v)
567  {
568  uint64_t n = ReadCompactSize<Stream>(s, RangeCheck);
569  if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
570  throw std::ios_base::failure("CompactSize exceeds limit of type");
571  }
572  v = n;
573  }
574 
575  template<typename Stream, typename I>
576  void Ser(Stream& s, I v)
577  {
578  static_assert(std::is_unsigned<I>::value, "CompactSize only supported for unsigned integers");
579  static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
580 
581  WriteCompactSize<Stream>(s, v);
582  }
583 };
584 
585 template <typename U, bool LOSSY = false>
587  template <typename Stream, typename Tp>
588  void Unser(Stream& s, Tp& tp)
589  {
590  U u;
591  s >> u;
592  // Lossy deserialization does not make sense, so force Wnarrowing
593  tp = Tp{typename Tp::duration{typename Tp::duration::rep{u}}};
594  }
595  template <typename Stream, typename Tp>
596  void Ser(Stream& s, Tp tp)
597  {
598  if constexpr (LOSSY) {
599  s << U(tp.time_since_epoch().count());
600  } else {
601  s << U{tp.time_since_epoch().count()};
602  }
603  }
604 };
605 template <typename U>
607 
609 {
610 protected:
611  uint64_t n;
612 public:
613  explicit CompactSizeWriter(uint64_t n_in) : n(n_in) { }
614 
615  template<typename Stream>
616  void Serialize(Stream &s) const {
617  WriteCompactSize<Stream>(s, n);
618  }
619 };
620 
621 template<size_t Limit>
623 {
624  template<typename Stream>
625  void Unser(Stream& s, std::string& v)
626  {
627  size_t size = ReadCompactSize(s);
628  if (size > Limit) {
629  throw std::ios_base::failure("String length limit exceeded");
630  }
631  v.resize(size);
632  if (size != 0) s.read(MakeWritableByteSpan(v));
633  }
634 
635  template<typename Stream>
636  void Ser(Stream& s, const std::string& v)
637  {
638  s << v;
639  }
640 };
641 
655 template<class Formatter>
657 {
658  template<typename Stream, typename V>
659  void Ser(Stream& s, const V& v)
660  {
661  Formatter formatter;
662  WriteCompactSize(s, v.size());
663  for (const typename V::value_type& elem : v) {
664  formatter.Ser(s, elem);
665  }
666  }
667 
668  template<typename Stream, typename V>
669  void Unser(Stream& s, V& v)
670  {
671  Formatter formatter;
672  v.clear();
673  size_t size = ReadCompactSize(s);
674  size_t allocated = 0;
675  while (allocated < size) {
676  // For DoS prevention, do not blindly allocate as much as the stream claims to contain.
677  // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
678  // X MiB of data to make us allocate X+5 Mib.
679  static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
680  allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
681  v.reserve(allocated);
682  while (v.size() < allocated) {
683  v.emplace_back();
684  formatter.Unser(s, v.back());
685  }
686  }
687  };
688 };
689 
697 template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str);
698 template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str);
699 
703 template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v);
704 template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v);
705 
709 template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v);
710 template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v);
711 
715 template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item);
716 template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item);
717 
721 template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m);
722 template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m);
723 
727 template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m);
728 template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m);
729 
733 template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p);
734 template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p);
735 
739 template<typename Stream, typename T> void Serialize(Stream& os, const std::unique_ptr<const T>& p);
740 template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p);
741 
742 
746 template <class T, class Stream>
747 concept Serializable = requires(T a, Stream s) { a.Serialize(s); };
748 template <typename Stream, typename T>
749  requires Serializable<T, Stream>
750 void Serialize(Stream& os, const T& a)
751 {
752  a.Serialize(os);
753 }
754 
755 template <class T, class Stream>
756 concept Unserializable = requires(T a, Stream s) { a.Unserialize(s); };
757 template <typename Stream, typename T>
758  requires Unserializable<T, Stream>
759 void Unserialize(Stream& is, T&& a)
760 {
761  a.Unserialize(is);
762 }
763 
770 {
771  template<typename Stream, typename T>
772  static void Ser(Stream& s, const T& t) { Serialize(s, t); }
773 
774  template<typename Stream, typename T>
775  static void Unser(Stream& s, T& t) { Unserialize(s, t); }
776 };
777 
778 
779 
780 
781 
785 template<typename Stream, typename C>
786 void Serialize(Stream& os, const std::basic_string<C>& str)
787 {
788  WriteCompactSize(os, str.size());
789  if (!str.empty())
790  os.write(MakeByteSpan(str));
791 }
792 
793 template<typename Stream, typename C>
794 void Unserialize(Stream& is, std::basic_string<C>& str)
795 {
796  unsigned int nSize = ReadCompactSize(is);
797  str.resize(nSize);
798  if (nSize != 0)
799  is.read(MakeWritableByteSpan(str));
800 }
801 
802 
803 
807 template <typename Stream, unsigned int N, typename T>
808 void Serialize(Stream& os, const prevector<N, T>& v)
809 {
810  if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
811  WriteCompactSize(os, v.size());
812  if (!v.empty()) os.write(MakeByteSpan(v));
813  } else {
815  }
816 }
817 
818 
819 template <typename Stream, unsigned int N, typename T>
820 void Unserialize(Stream& is, prevector<N, T>& v)
821 {
822  if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
823  // Limit size per read so bogus size value won't cause out of memory
824  v.clear();
825  unsigned int nSize = ReadCompactSize(is);
826  unsigned int i = 0;
827  while (i < nSize) {
828  unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
829  v.resize_uninitialized(i + blk);
830  is.read(AsWritableBytes(Span{&v[i], blk}));
831  i += blk;
832  }
833  } else {
835  }
836 }
837 
838 
842 template <typename Stream, typename T, typename A>
843 void Serialize(Stream& os, const std::vector<T, A>& v)
844 {
845  if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
846  WriteCompactSize(os, v.size());
847  if (!v.empty()) os.write(MakeByteSpan(v));
848  } else if constexpr (std::is_same_v<T, bool>) {
849  // A special case for std::vector<bool>, as dereferencing
850  // std::vector<bool>::const_iterator does not result in a const bool&
851  // due to std::vector's special casing for bool arguments.
852  WriteCompactSize(os, v.size());
853  for (bool elem : v) {
854  ::Serialize(os, elem);
855  }
856  } else {
858  }
859 }
860 
861 
862 template <typename Stream, typename T, typename A>
863 void Unserialize(Stream& is, std::vector<T, A>& v)
864 {
865  if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
866  // Limit size per read so bogus size value won't cause out of memory
867  v.clear();
868  unsigned int nSize = ReadCompactSize(is);
869  unsigned int i = 0;
870  while (i < nSize) {
871  unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
872  v.resize(i + blk);
873  is.read(AsWritableBytes(Span{&v[i], blk}));
874  i += blk;
875  }
876  } else {
878  }
879 }
880 
881 
885 template<typename Stream, typename K, typename T>
886 void Serialize(Stream& os, const std::pair<K, T>& item)
887 {
888  Serialize(os, item.first);
889  Serialize(os, item.second);
890 }
891 
892 template<typename Stream, typename K, typename T>
893 void Unserialize(Stream& is, std::pair<K, T>& item)
894 {
895  Unserialize(is, item.first);
896  Unserialize(is, item.second);
897 }
898 
899 
900 
904 template<typename Stream, typename K, typename T, typename Pred, typename A>
905 void Serialize(Stream& os, const std::map<K, T, Pred, A>& m)
906 {
907  WriteCompactSize(os, m.size());
908  for (const auto& entry : m)
909  Serialize(os, entry);
910 }
911 
912 template<typename Stream, typename K, typename T, typename Pred, typename A>
913 void Unserialize(Stream& is, std::map<K, T, Pred, A>& m)
914 {
915  m.clear();
916  unsigned int nSize = ReadCompactSize(is);
917  typename std::map<K, T, Pred, A>::iterator mi = m.begin();
918  for (unsigned int i = 0; i < nSize; i++)
919  {
920  std::pair<K, T> item;
921  Unserialize(is, item);
922  mi = m.insert(mi, item);
923  }
924 }
925 
926 
927 
931 template<typename Stream, typename K, typename Pred, typename A>
932 void Serialize(Stream& os, const std::set<K, Pred, A>& m)
933 {
934  WriteCompactSize(os, m.size());
935  for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it)
936  Serialize(os, (*it));
937 }
938 
939 template<typename Stream, typename K, typename Pred, typename A>
940 void Unserialize(Stream& is, std::set<K, Pred, A>& m)
941 {
942  m.clear();
943  unsigned int nSize = ReadCompactSize(is);
944  typename std::set<K, Pred, A>::iterator it = m.begin();
945  for (unsigned int i = 0; i < nSize; i++)
946  {
947  K key;
948  Unserialize(is, key);
949  it = m.insert(it, key);
950  }
951 }
952 
953 
954 
958 template<typename Stream, typename T> void
959 Serialize(Stream& os, const std::unique_ptr<const T>& p)
960 {
961  Serialize(os, *p);
962 }
963 
964 template<typename Stream, typename T>
965 void Unserialize(Stream& is, std::unique_ptr<const T>& p)
966 {
967  p.reset(new T(deserialize, is));
968 }
969 
970 
971 
975 template<typename Stream, typename T> void
976 Serialize(Stream& os, const std::shared_ptr<const T>& p)
977 {
978  Serialize(os, *p);
979 }
980 
981 template<typename Stream, typename T>
982 void Unserialize(Stream& is, std::shared_ptr<const T>& p)
983 {
984  p = std::make_shared<const T>(deserialize, is);
985 }
986 
991 template <typename Stream, typename... Args>
992 void SerializeMany(Stream& s, const Args&... args)
993 {
994  (::Serialize(s, args), ...);
995 }
996 
997 template <typename Stream, typename... Args>
998 inline void UnserializeMany(Stream& s, Args&&... args)
999 {
1000  (::Unserialize(s, args), ...);
1001 }
1002 
1007  static constexpr bool ForRead() { return false; }
1008 
1009  template<typename Stream, typename... Args>
1010  static void SerReadWriteMany(Stream& s, const Args&... args)
1011  {
1012  ::SerializeMany(s, args...);
1013  }
1014 
1015  template<typename Stream, typename Type, typename Fn>
1016  static void SerRead(Stream& s, Type&&, Fn&&)
1017  {
1018  }
1019 
1020  template<typename Stream, typename Type, typename Fn>
1021  static void SerWrite(Stream& s, Type&& obj, Fn&& fn)
1022  {
1023  fn(s, std::forward<Type>(obj));
1024  }
1025 };
1027  static constexpr bool ForRead() { return true; }
1028 
1029  template<typename Stream, typename... Args>
1030  static void SerReadWriteMany(Stream& s, Args&&... args)
1031  {
1032  ::UnserializeMany(s, args...);
1033  }
1034 
1035  template<typename Stream, typename Type, typename Fn>
1036  static void SerRead(Stream& s, Type&& obj, Fn&& fn)
1037  {
1038  fn(s, std::forward<Type>(obj));
1039  }
1040 
1041  template<typename Stream, typename Type, typename Fn>
1042  static void SerWrite(Stream& s, Type&&, Fn&&)
1043  {
1044  }
1045 };
1046 
1047 /* ::GetSerializeSize implementations
1048  *
1049  * Computing the serialized size of objects is done through a special stream
1050  * object of type SizeComputer, which only records the number of bytes written
1051  * to it.
1052  *
1053  * If your Serialize or SerializationOp method has non-trivial overhead for
1054  * serialization, it may be worthwhile to implement a specialized version for
1055  * SizeComputer, which uses the s.seek() method to record bytes that would
1056  * be written instead.
1057  */
1059 {
1060 protected:
1061  size_t nSize{0};
1062 
1063 public:
1065 
1067  {
1068  this->nSize += src.size();
1069  }
1070 
1072  void seek(size_t _nSize)
1073  {
1074  this->nSize += _nSize;
1075  }
1076 
1077  template<typename T>
1079  {
1080  ::Serialize(*this, obj);
1081  return (*this);
1082  }
1083 
1084  size_t size() const {
1085  return nSize;
1086  }
1087 };
1088 
1089 template<typename I>
1090 inline void WriteVarInt(SizeComputer &s, I n)
1091 {
1092  s.seek(GetSizeOfVarInt<I>(n));
1093 }
1094 
1095 inline void WriteCompactSize(SizeComputer &s, uint64_t nSize)
1096 {
1097  s.seek(GetSizeOfCompactSize(nSize));
1098 }
1099 
1100 template <typename T>
1101 size_t GetSerializeSize(const T& t)
1102 {
1103  return (SizeComputer() << t).size();
1104 }
1105 
1107 template<typename T>
1108 concept ContainsStream = requires(T t) { t.GetStream(); };
1109 
1111 template <typename SubStream, typename Params>
1113 {
1115  // If ParamsStream constructor is passed an lvalue argument, Substream will
1116  // be a reference type, and m_substream will reference that argument.
1117  // Otherwise m_substream will be a substream instance and move from the
1118  // argument. Letting ParamsStream contain a substream instance instead of
1119  // just a reference is useful to make the ParamsStream object self contained
1120  // and let it do cleanup when destroyed, for example by closing files if
1121  // SubStream is a file stream.
1122  SubStream m_substream;
1123 
1124 public:
1125  ParamsStream(SubStream&& substream, const Params& params LIFETIMEBOUND) : m_params{params}, m_substream{std::forward<SubStream>(substream)} {}
1126 
1127  template <typename NestedSubstream, typename Params1, typename Params2, typename... NestedParams>
1128  ParamsStream(NestedSubstream&& s, const Params1& params1 LIFETIMEBOUND, const Params2& params2 LIFETIMEBOUND, const NestedParams&... params LIFETIMEBOUND)
1129  : ParamsStream{::ParamsStream{std::forward<NestedSubstream>(s), params2, params...}, params1} {}
1130 
1131  template <typename U> ParamsStream& operator<<(const U& obj) { ::Serialize(*this, obj); return *this; }
1132  template <typename U> ParamsStream& operator>>(U&& obj) { ::Unserialize(*this, obj); return *this; }
1133  void write(Span<const std::byte> src) { GetStream().write(src); }
1134  void read(Span<std::byte> dst) { GetStream().read(dst); }
1135  void ignore(size_t num) { GetStream().ignore(num); }
1136  bool eof() const { return GetStream().eof(); }
1137  size_t size() const { return GetStream().size(); }
1138 
1140  template <typename P>
1141  const auto& GetParams() const
1142  {
1143  if constexpr (std::is_convertible_v<Params, P>) {
1144  return m_params;
1145  } else {
1146  return m_substream.template GetParams<P>();
1147  }
1148  }
1149 
1151  auto& GetStream()
1152  {
1153  if constexpr (ContainsStream<SubStream>) {
1154  return m_substream.GetStream();
1155  } else {
1156  return m_substream;
1157  }
1158  }
1159  const auto& GetStream() const
1160  {
1161  if constexpr (ContainsStream<SubStream>) {
1162  return m_substream.GetStream();
1163  } else {
1164  return m_substream;
1165  }
1166  }
1167 };
1168 
1174 template <typename Substream, typename Params>
1176 
1181 template <typename Substream, typename Params1, typename Params2, typename... Params>
1182 ParamsStream(Substream&& s, const Params1& params1, const Params2& params2, const Params&... params) ->
1183  ParamsStream<decltype(ParamsStream{std::forward<Substream>(s), params2, params...}), Params1>;
1184 
1186 template <typename Params, typename T>
1188 {
1191 
1192 public:
1193  explicit ParamsWrapper(const Params& params, T& obj) : m_params{params}, m_object{obj} {}
1194 
1195  template <typename Stream>
1196  void Serialize(Stream& s) const
1197  {
1198  ParamsStream ss{s, m_params};
1199  ::Serialize(ss, m_object);
1200  }
1201  template <typename Stream>
1202  void Unserialize(Stream& s)
1203  {
1204  ParamsStream ss{s, m_params};
1205  ::Unserialize(ss, m_object);
1206  }
1207 };
1208 
1218 #define SER_PARAMS_OPFUNC \
1219  \
1224  template <typename T> \
1225  auto operator()(T&& t) const \
1226  { \
1227  return ParamsWrapper{*this, t}; \
1228  }
1229 
1230 #endif // BITCOIN_SERIALIZE_H
#define LIFETIMEBOUND
Definition: attributes.h:16
ArgsManager & args
Definition: bitcoind.cpp:267
const CChainParams & Params()
Return the currently selected parameters.
void Serialize(Stream &s) const
Definition: serialize.h:616
CompactSizeWriter(uint64_t n_in)
Definition: serialize.h:613
Wrapper that overrides the GetParams() function of a stream.
Definition: serialize.h:1113
bool eof() const
Definition: serialize.h:1136
ParamsStream & operator>>(U &&obj)
Definition: serialize.h:1132
ParamsStream(NestedSubstream &&s, const Params1 &params1 LIFETIMEBOUND, const Params2 &params2 LIFETIMEBOUND, const NestedParams &... params LIFETIMEBOUND)
Definition: serialize.h:1128
const auto & GetStream() const
Definition: serialize.h:1159
size_t size() const
Definition: serialize.h:1137
void ignore(size_t num)
Definition: serialize.h:1135
void read(Span< std::byte > dst)
Definition: serialize.h:1134
ParamsStream & operator<<(const U &obj)
Definition: serialize.h:1131
const Params & m_params
Definition: serialize.h:1114
void write(Span< const std::byte > src)
Definition: serialize.h:1133
const auto & GetParams() const
Get reference to stream parameters.
Definition: serialize.h:1141
SubStream m_substream
Definition: serialize.h:1122
auto & GetStream()
Get reference to underlying stream.
Definition: serialize.h:1151
ParamsStream(SubStream &&substream, const Params &params LIFETIMEBOUND)
Definition: serialize.h:1125
Wrapper that serializes objects with the specified parameters.
Definition: serialize.h:1188
const Params & m_params
Definition: serialize.h:1189
void Unserialize(Stream &s)
Definition: serialize.h:1202
void Serialize(Stream &s) const
Definition: serialize.h:1196
ParamsWrapper(const Params &params, T &obj)
Definition: serialize.h:1193
void write(Span< const std::byte > src)
Definition: serialize.h:1066
void seek(size_t _nSize)
Pretend _nSize bytes are written, without specifying them.
Definition: serialize.h:1072
SizeComputer & operator<<(const T &obj)
Definition: serialize.h:1078
size_t nSize
Definition: serialize.h:1061
size_t size() const
Definition: serialize.h:1084
A Span is an object that can refer to a contiguous sequence of objects.
Definition: span.h:98
constexpr std::size_t size() const noexcept
Definition: span.h:187
Simple wrapper class to serialize objects using a formatter; used by Using().
Definition: serialize.h:474
Wrapper(T obj)
Definition: serialize.h:479
T m_object
Definition: serialize.h:475
void Serialize(Stream &s) const
Definition: serialize.h:480
void Unserialize(Stream &s)
Definition: serialize.h:481
Implements a drop-in replacement for std::vector<T> which stores up to N elements directly (without h...
Definition: prevector.h:37
bool empty() const
Definition: prevector.h:300
void clear()
Definition: prevector.h:357
size_type size() const
Definition: prevector.h:296
void resize_uninitialized(size_type new_size)
Definition: prevector.h:400
BSWAP_CONSTEXPR uint32_t be32toh_internal(uint32_t big_endian_32bits)
Definition: endian.h:43
BSWAP_CONSTEXPR uint16_t be16toh_internal(uint16_t big_endian_16bits)
Definition: endian.h:23
BSWAP_CONSTEXPR uint64_t htobe64_internal(uint64_t host_64bits)
Definition: endian.h:53
BSWAP_CONSTEXPR uint16_t htobe16_internal(uint16_t host_16bits)
Definition: endian.h:13
BSWAP_CONSTEXPR uint32_t htole32_internal(uint32_t host_32bits)
Definition: endian.h:38
BSWAP_CONSTEXPR uint16_t htole16_internal(uint16_t host_16bits)
Definition: endian.h:18
BSWAP_CONSTEXPR uint64_t be64toh_internal(uint64_t big_endian_64bits)
Definition: endian.h:63
BSWAP_CONSTEXPR uint16_t le16toh_internal(uint16_t little_endian_16bits)
Definition: endian.h:28
BSWAP_CONSTEXPR uint64_t htole64_internal(uint64_t host_64bits)
Definition: endian.h:58
BSWAP_CONSTEXPR uint64_t le64toh_internal(uint64_t little_endian_64bits)
Definition: endian.h:68
BSWAP_CONSTEXPR uint32_t le32toh_internal(uint32_t little_endian_32bits)
Definition: endian.h:48
BSWAP_CONSTEXPR uint32_t htobe32_internal(uint32_t host_32bits)
Definition: endian.h:33
#define T(expected, seed, data)
size_t GetSerializeSize(const T &t)
Definition: serialize.h:1101
void Serialize(Stream &, V)=delete
constexpr unsigned int GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 byt...
Definition: serialize.h:295
void SerializeMany(Stream &s, const Args &... args)
Support for (un)serializing many things at once.
Definition: serialize.h:992
static const unsigned int MAX_VECTOR_ALLOCATE
Maximum amount of memory (in bytes) to allocate at once when deserializing vectors.
Definition: serialize.h:35
uint8_t ser_readdata8(Stream &s)
Definition: serialize.h:83
I ReadVarInt(Stream &is)
Definition: serialize.h:450
void ser_writedata32be(Stream &s, uint32_t obj)
Definition: serialize.h:73
VarIntMode
Variable-length integers: bytes are a MSB base-128 encoding of the number.
Definition: serialize.h:403
@ NONNEGATIVE_SIGNED
void ser_writedata32(Stream &s, uint32_t obj)
Definition: serialize.h:68
static constexpr uint64_t MAX_SIZE
The maximum size of a serialized object in bytes or number of elements (for eg vectors) when the size...
Definition: serialize.h:32
constexpr deserialize_type deserialize
Definition: serialize.h:49
void ser_writedata16(Stream &s, uint16_t obj)
Definition: serialize.h:58
Out & AsBase(In &x)
Convert any argument to a reference to X, maintaining constness.
Definition: serialize.h:144
concept ContainsStream
Check if type contains a stream by seeing if has a GetStream() method.
Definition: serialize.h:1108
void Unserialize(Stream &, V)=delete
concept CharNotInt8
Definition: serialize.h:253
void UnserializeMany(Stream &s, Args &&... args)
Definition: serialize.h:998
static Wrapper< Formatter, T & > Using(T &&t)
Cause serialization/deserialization of an object to be done using a specified formatter class.
Definition: serialize.h:495
void WriteVarInt(SizeComputer &os, I n)
Definition: serialize.h:1090
void WriteCompactSize(SizeComputer &os, uint64_t nSize)
Definition: serialize.h:1095
concept Unserializable
Definition: serialize.h:756
void ser_writedata16be(Stream &s, uint16_t obj)
Definition: serialize.h:63
uint16_t ser_readdata16(Stream &s)
Definition: serialize.h:89
uint64_t ser_readdata64(Stream &s)
Definition: serialize.h:113
void ser_writedata8(Stream &s, uint8_t obj)
Definition: serialize.h:54
uint64_t ReadCompactSize(Stream &is, bool range_check=true)
Decode a CompactSize-encoded variable-length integer.
Definition: serialize.h:337
ParamsStream(Substream &&, const Params &) -> ParamsStream< Substream, Params >
Explicit template deduction guide is required for single-parameter constructor so Substream&& is trea...
unsigned int GetSizeOfVarInt(I n)
Definition: serialize.h:415
concept Serializable
If none of the specialized versions above matched, default to calling member function.
Definition: serialize.h:747
uint32_t ser_readdata32(Stream &s)
Definition: serialize.h:101
uint16_t ser_readdata16be(Stream &s)
Definition: serialize.h:95
void ser_writedata64(Stream &s, uint64_t obj)
Definition: serialize.h:78
uint32_t ser_readdata32be(Stream &s)
Definition: serialize.h:107
Span< std::byte > AsWritableBytes(Span< T > s) noexcept
Definition: span.h:271
Span< const std::byte > MakeByteSpan(V &&v) noexcept
Definition: span.h:277
Span< const std::byte > AsBytes(Span< T > s) noexcept
Definition: span.h:266
Span< std::byte > MakeWritableByteSpan(V &&v) noexcept
Definition: span.h:282
Support for all macros providing or using the ser_action parameter of the SerializationOps method.
Definition: serialize.h:1006
static void SerReadWriteMany(Stream &s, const Args &... args)
Definition: serialize.h:1010
static constexpr bool ForRead()
Definition: serialize.h:1007
static void SerWrite(Stream &s, Type &&obj, Fn &&fn)
Definition: serialize.h:1021
static void SerRead(Stream &s, Type &&, Fn &&)
Definition: serialize.h:1016
static constexpr bool ForRead()
Definition: serialize.h:1027
static void SerReadWriteMany(Stream &s, Args &&... args)
Definition: serialize.h:1030
static void SerRead(Stream &s, Type &&obj, Fn &&fn)
Definition: serialize.h:1036
static void SerWrite(Stream &s, Type &&, Fn &&)
Definition: serialize.h:1042
constexpr CheckVarIntMode()
Definition: serialize.h:407
void Unser(Stream &s, Tp &tp)
Definition: serialize.h:588
void Ser(Stream &s, Tp tp)
Definition: serialize.h:596
Formatter for integers in CompactSize format.
Definition: serialize.h:564
void Unser(Stream &s, I &v)
Definition: serialize.h:566
void Ser(Stream &s, I v)
Definition: serialize.h:576
Serialization wrapper class for custom integers and enums.
Definition: serialize.h:528
static constexpr uint64_t MAX
Definition: serialize.h:530
void Ser(Stream &s, I v)
Definition: serialize.h:532
void Unser(Stream &s, I &v)
Definition: serialize.h:544
Default formatter.
Definition: serialize.h:770
static void Ser(Stream &s, const T &t)
Definition: serialize.h:772
static void Unser(Stream &s, T &t)
Definition: serialize.h:775
void Unser(Stream &s, std::string &v)
Definition: serialize.h:625
void Ser(Stream &s, const std::string &v)
Definition: serialize.h:636
Serialization wrapper class for integers in VarInt format.
Definition: serialize.h:505
void Ser(Stream &s, I v)
Definition: serialize.h:506
void Unser(Stream &s, I &v)
Definition: serialize.h:511
Formatter to serialize/deserialize vector elements using another formatter.
Definition: serialize.h:657
void Unser(Stream &s, V &v)
Definition: serialize.h:669
void Ser(Stream &s, const V &v)
Definition: serialize.h:659
Dummy data type to identify deserializing constructors.
Definition: serialize.h:48
#define B
Definition: util_tests.cpp:489