generated_message_tctable_impl.h 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303
  1. // Protocol Buffers - Google's data interchange format
  2. // Copyright 2008 Google Inc. All rights reserved.
  3. // https://developers.google.com/protocol-buffers/
  4. //
  5. // Redistribution and use in source and binary forms, with or without
  6. // modification, are permitted provided that the following conditions are
  7. // met:
  8. //
  9. // * Redistributions of source code must retain the above copyright
  10. // notice, this list of conditions and the following disclaimer.
  11. // * Redistributions in binary form must reproduce the above
  12. // copyright notice, this list of conditions and the following disclaimer
  13. // in the documentation and/or other materials provided with the
  14. // distribution.
  15. // * Neither the name of Google Inc. nor the names of its
  16. // contributors may be used to endorse or promote products derived from
  17. // this software without specific prior written permission.
  18. //
  19. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. #ifndef GOOGLE_PROTOBUF_GENERATED_MESSAGE_TCTABLE_IMPL_H__
  31. #define GOOGLE_PROTOBUF_GENERATED_MESSAGE_TCTABLE_IMPL_H__
  32. #include <cstdint>
  33. #include <type_traits>
  34. #include <google/protobuf/parse_context.h>
  35. #include <google/protobuf/extension_set.h>
  36. #include <google/protobuf/generated_message_tctable_decl.h>
  37. #include <google/protobuf/message_lite.h>
  38. #include <google/protobuf/metadata_lite.h>
  39. #include <google/protobuf/port.h>
  40. #include <google/protobuf/wire_format_lite.h>
  41. // Must come last:
  42. #include <google/protobuf/port_def.inc>
  43. namespace google {
  44. namespace protobuf {
  45. class Message;
  46. class UnknownFieldSet;
  47. namespace internal {
  48. // PROTOBUF_TC_PARAM_DECL are the parameters for tailcall functions, it is
  49. // defined in port_def.inc.
  50. //
  51. // Note that this is performance sensitive: changing the parameters will change
  52. // the registers used by the ABI calling convention, which subsequently affects
  53. // register selection logic inside the function.
  54. // PROTOBUF_TC_PARAM_PASS passes values to match PROTOBUF_TC_PARAM_DECL.
  55. #define PROTOBUF_TC_PARAM_PASS msg, ptr, ctx, table, hasbits, data
  56. // PROTOBUF_TC_PARSE_* decide which function is used to parse message-typed
  57. // fields. The guard macros are defined in port_def.inc.
  58. #if PROTOBUF_TC_STATIC_PARSE_SINGULAR1
  59. #define PROTOBUF_TC_PARSE_SINGULAR1(MESSAGE) MESSAGE::Tct_ParseS1
  60. #else
  61. #define PROTOBUF_TC_PARSE_SINGULAR1(MESSAGE) \
  62. ::google::protobuf::internal::TcParser::SingularParseMessage<MESSAGE, uint8_t>
  63. #endif // PROTOBUF_TC_STATIC_PARSE_SINGULAR1
  64. #if PROTOBUF_TC_STATIC_PARSE_SINGULAR2
  65. #define PROTOBUF_TC_PARSE_SINGULAR2(MESSAGE) MESSAGE::Tct_ParseS2
  66. #else
  67. #define PROTOBUF_TC_PARSE_SINGULAR2(MESSAGE) \
  68. ::google::protobuf::internal::TcParser::SingularParseMessage<MESSAGE, uint16_t>
  69. #endif // PROTOBUF_TC_STATIC_PARSE_SINGULAR2
  70. #if PROTOBUF_TC_STATIC_PARSE_REPEATED1
  71. #define PROTOBUF_TC_PARSE_REPEATED1(MESSAGE) MESSAGE::Tct_ParseR1
  72. #else
  73. #define PROTOBUF_TC_PARSE_REPEATED1(MESSAGE) \
  74. ::google::protobuf::internal::TcParser::RepeatedParseMessage<MESSAGE, uint8_t>
  75. #endif // PROTOBUF_TC_STATIC_PARSE_REPEATED1
  76. #if PROTOBUF_TC_STATIC_PARSE_REPEATED2
  77. #define PROTOBUF_TC_PARSE_REPEATED2(MESSAGE) MESSAGE::Tct_ParseR2
  78. #else
  79. #define PROTOBUF_TC_PARSE_REPEATED2(MESSAGE) \
  80. ::google::protobuf::internal::TcParser::RepeatedParseMessage<MESSAGE, uint16_t>
  81. #endif // PROTOBUF_TC_STATIC_PARSE_REPEATED2
  82. #ifndef NDEBUG
  83. template <size_t align>
  84. #ifndef _MSC_VER
  85. [[noreturn]]
  86. #endif
  87. void AlignFail(uintptr_t address) {
  88. GOOGLE_LOG(FATAL) << "Unaligned (" << align << ") access at " << address;
  89. }
  90. extern template void AlignFail<4>(uintptr_t);
  91. extern template void AlignFail<8>(uintptr_t);
  92. #endif
  93. // TcParser implements most of the parsing logic for tailcall tables.
  94. class TcParser final {
  95. public:
  96. static const char* GenericFallback(PROTOBUF_TC_PARAM_DECL);
  97. static const char* GenericFallbackLite(PROTOBUF_TC_PARAM_DECL);
  98. // Dispatch to the designated parse function
  99. inline PROTOBUF_ALWAYS_INLINE static const char* TagDispatch(
  100. PROTOBUF_TC_PARAM_DECL) {
  101. const auto coded_tag = UnalignedLoad<uint16_t>(ptr);
  102. const size_t idx = coded_tag & table->fast_idx_mask;
  103. PROTOBUF_ASSUME((idx & 7) == 0);
  104. auto* fast_entry = table->fast_entry(idx >> 3);
  105. data = fast_entry->bits;
  106. data.data ^= coded_tag;
  107. PROTOBUF_MUSTTAIL return fast_entry->target(PROTOBUF_TC_PARAM_PASS);
  108. }
  109. // We can only safely call from field to next field if the call is optimized
  110. // to a proper tail call. Otherwise we blow through stack. Clang and gcc
  111. // reliably do this optimization in opt mode, but do not perform this in debug
  112. // mode. Luckily the structure of the algorithm is such that it's always
  113. // possible to just return and use the enclosing parse loop as a trampoline.
  114. static const char* ToTagDispatch(PROTOBUF_TC_PARAM_DECL) {
  115. constexpr bool always_return = !PROTOBUF_TAILCALL;
  116. if (always_return || !ctx->DataAvailable(ptr)) {
  117. PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_PASS);
  118. }
  119. PROTOBUF_MUSTTAIL return TagDispatch(PROTOBUF_TC_PARAM_PASS);
  120. }
  121. static const char* ParseLoop(MessageLite* msg, const char* ptr,
  122. ParseContext* ctx,
  123. const TcParseTableBase* table) {
  124. ScopedArenaSwap saved(msg, ctx);
  125. const uint32_t has_bits_offset = table->has_bits_offset;
  126. while (!ctx->Done(&ptr)) {
  127. uint64_t hasbits = 0;
  128. if (has_bits_offset) hasbits = RefAt<uint32_t>(msg, has_bits_offset);
  129. ptr = TagDispatch(msg, ptr, ctx, table, hasbits, {});
  130. if (ptr == nullptr) break;
  131. if (ctx->LastTag() != 1) break; // Ended on terminating tag
  132. }
  133. return ptr;
  134. }
  135. template <typename FieldType, typename TagType>
  136. PROTOBUF_NOINLINE static const char* SingularParseMessage(
  137. PROTOBUF_TC_PARAM_DECL) {
  138. if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
  139. return table->fallback(PROTOBUF_TC_PARAM_PASS);
  140. }
  141. ptr += sizeof(TagType);
  142. hasbits |= (uint64_t{1} << data.hasbit_idx());
  143. auto& field = RefAt<FieldType*>(msg, data.offset());
  144. if (field == nullptr) {
  145. auto arena = ctx->data().arena;
  146. if (Arena::is_arena_constructable<FieldType>::value) {
  147. field = Arena::CreateMessage<FieldType>(arena);
  148. } else {
  149. field = Arena::Create<FieldType>(arena);
  150. }
  151. }
  152. SyncHasbits(msg, hasbits, table);
  153. return ctx->ParseMessage(field, ptr);
  154. }
  155. template <typename FieldType, typename TagType>
  156. PROTOBUF_NOINLINE static const char* RepeatedParseMessage(
  157. PROTOBUF_TC_PARAM_DECL) {
  158. if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
  159. return table->fallback(PROTOBUF_TC_PARAM_PASS);
  160. }
  161. ptr += sizeof(TagType);
  162. auto& field = RefAt<RepeatedPtrField<FieldType>>(msg, data.offset());
  163. SyncHasbits(msg, hasbits, table);
  164. ptr = ctx->ParseMessage(field.Add(), ptr);
  165. return ptr;
  166. }
  167. template <typename LayoutType, typename TagType>
  168. static const char* SingularFixed(PROTOBUF_TC_PARAM_DECL);
  169. template <typename LayoutType, typename TagType>
  170. static const char* RepeatedFixed(PROTOBUF_TC_PARAM_DECL);
  171. template <typename LayoutType, typename TagType>
  172. static const char* PackedFixed(PROTOBUF_TC_PARAM_DECL);
  173. enum VarintDecode { kNoConversion = 0, kZigZag = 1 };
  174. template <typename FieldType, typename TagType, VarintDecode zigzag>
  175. static const char* SingularVarint(PROTOBUF_TC_PARAM_DECL);
  176. template <typename FieldType, typename TagType, VarintDecode zigzag>
  177. static const char* RepeatedVarint(PROTOBUF_TC_PARAM_DECL);
  178. template <typename FieldType, typename TagType, VarintDecode zigzag>
  179. static const char* PackedVarint(PROTOBUF_TC_PARAM_DECL);
  180. enum Utf8Type { kNoUtf8 = 0, kUtf8 = 1, kUtf8ValidateOnly = 2 };
  181. template <typename TagType, Utf8Type utf8>
  182. static const char* SingularString(PROTOBUF_TC_PARAM_DECL);
  183. template <typename TagType, Utf8Type utf8>
  184. static const char* RepeatedString(PROTOBUF_TC_PARAM_DECL);
  185. template <typename T>
  186. static inline T& RefAt(void* x, size_t offset) {
  187. T* target = reinterpret_cast<T*>(static_cast<char*>(x) + offset);
  188. #ifndef NDEBUG
  189. if (PROTOBUF_PREDICT_FALSE(
  190. reinterpret_cast<uintptr_t>(target) % alignof(T) != 0)) {
  191. AlignFail<alignof(T)>(reinterpret_cast<uintptr_t>(target));
  192. }
  193. #endif
  194. return *target;
  195. }
  196. static inline PROTOBUF_ALWAYS_INLINE void SyncHasbits(
  197. MessageLite* msg, uint64_t hasbits, const TcParseTableBase* table) {
  198. const uint32_t has_bits_offset = table->has_bits_offset;
  199. if (has_bits_offset) {
  200. // Only the first 32 has-bits are updated. Nothing above those is stored,
  201. // but e.g. messages without has-bits update the upper bits.
  202. RefAt<uint32_t>(msg, has_bits_offset) = static_cast<uint32_t>(hasbits);
  203. }
  204. }
  205. protected:
  206. static inline PROTOBUF_ALWAYS_INLINE const char* ToParseLoop(
  207. PROTOBUF_TC_PARAM_DECL) {
  208. (void)data;
  209. (void)ctx;
  210. SyncHasbits(msg, hasbits, table);
  211. return ptr;
  212. }
  213. static inline PROTOBUF_ALWAYS_INLINE const char* Error(
  214. PROTOBUF_TC_PARAM_DECL) {
  215. (void)data;
  216. (void)ctx;
  217. (void)ptr;
  218. SyncHasbits(msg, hasbits, table);
  219. return nullptr;
  220. }
  221. class ScopedArenaSwap final {
  222. public:
  223. ScopedArenaSwap(MessageLite* msg, ParseContext* ctx)
  224. : ctx_(ctx), saved_(ctx->data().arena) {
  225. ctx_->data().arena = msg->GetArenaForAllocation();
  226. }
  227. ScopedArenaSwap(const ScopedArenaSwap&) = delete;
  228. ~ScopedArenaSwap() { ctx_->data().arena = saved_; }
  229. private:
  230. ParseContext* const ctx_;
  231. Arena* const saved_;
  232. };
  233. template <class MessageBaseT, class UnknownFieldsT>
  234. static const char* GenericFallbackImpl(PROTOBUF_TC_PARAM_DECL) {
  235. #define CHK_(x) \
  236. if (PROTOBUF_PREDICT_FALSE(!(x))) return nullptr /* NOLINT */
  237. SyncHasbits(msg, hasbits, table);
  238. uint32_t tag;
  239. ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);
  240. CHK_(ptr);
  241. if ((tag & 7) == WireFormatLite::WIRETYPE_END_GROUP || tag == 0) {
  242. ctx->SetLastTag(tag);
  243. return ptr;
  244. }
  245. (void)data;
  246. uint32_t num = tag >> 3;
  247. if (table->extension_range_low <= num &&
  248. num <= table->extension_range_high) {
  249. return RefAt<ExtensionSet>(msg, table->extension_offset)
  250. .ParseField(tag, ptr,
  251. static_cast<const MessageBaseT*>(table->default_instance),
  252. &msg->_internal_metadata_, ctx);
  253. }
  254. return UnknownFieldParse(
  255. tag, msg->_internal_metadata_.mutable_unknown_fields<UnknownFieldsT>(),
  256. ptr, ctx);
  257. #undef CHK_
  258. }
  259. };
  260. // Declare helper functions:
  261. #include <google/protobuf/generated_message_tctable_impl.inc>
  262. } // namespace internal
  263. } // namespace protobuf
  264. } // namespace google
  265. #include <google/protobuf/port_undef.inc>
  266. #endif // GOOGLE_PROTOBUF_GENERATED_MESSAGE_TCTABLE_IMPL_H__