diff --git a/.editorconfig b/.editorconfig index 73050d175..d3df05e77 100644 --- a/.editorconfig +++ b/.editorconfig @@ -20,7 +20,7 @@ indent_style = tab tab_width = 4 # 2 space indentation -[*.{xml,csproj}] +[*.{xml,csproj,props,targets}] indent_style = space indent_size = 2 @@ -42,18 +42,17 @@ csharp_style_var_when_type_is_apparent = true:suggestion csharp_style_var_elsewhere = true:suggestion dotnet_sort_system_directives_first = true csharp_space_between_method_declaration_name_and_open_parenthesis = false -csharp_space_between_method_declaration_parameter_list_parentheses = true +csharp_space_between_method_declaration_parameter_list_parentheses = false csharp_space_between_method_declaration_empty_parameter_list_parentheses = false csharp_space_between_method_call_name_and_opening_parenthesis = false -csharp_space_between_method_call_parameter_list_parentheses = true +csharp_space_between_method_call_parameter_list_parentheses = false csharp_space_between_method_call_empty_parameter_list_parentheses = false -csharp_space_between_parentheses = expressions,type_casts,control_flow_statements csharp_space_after_keywords_in_control_flow_statements = true csharp_space_after_cast = false csharp_space_around_declaration_statements = true csharp_space_before_open_square_brackets = false csharp_space_between_empty_square_brackets = false -csharp_space_between_square_brackets = true +csharp_space_between_square_brackets = false csharp_space_after_colon_in_inheritance_clause = true csharp_space_after_comma = true csharp_space_after_dot = false @@ -62,7 +61,7 @@ csharp_space_before_colon_in_inheritance_clause = true csharp_space_before_comma = false csharp_space_before_dot = false csharp_space_before_semicolon_in_for_statement = false -csharp_space_around_binary_operators = true +csharp_space_around_binary_operators = false csharp_indent_braces = false csharp_indent_block_contents = true csharp_indent_switch_labels = true @@ -75,17 +74,122 @@ csharp_preserve_single_line_statements = true csharp_new_line_before_open_brace = all csharp_new_line_between_query_expression_clauses = true -# Disabled for Unity compatibility until 2017.1 RTM [*.cs] -dotnet_style_null_propagation = false:warning -csharp_style_expression_bodied_methods = false:warning -csharp_style_expression_bodied_constructors = false:warning -csharp_style_expression_bodied_operators = false:warning -csharp_style_expression_bodied_properties = false:warning -csharp_style_expression_bodied_indexers = false:warning -csharp_style_expression_bodied_accessors = false:warning -csharp_style_pattern_matching_over_is_with_cast_check = false:warning -csharp_style_pattern_matching_over_as_with_null_check = false:warning -csharp_style_inlined_variable_declaration = false:warning -csharp_style_throw_expression = false:warning -csharp_style_conditional_delegate_call = false:warning +dotnet_style_null_propagation = true:suggestion +csharp_style_expression_bodied_methods = true:suggestion +csharp_style_expression_bodied_constructors = true:suggestion +csharp_style_expression_bodied_operators = true:suggestion +csharp_style_expression_bodied_properties = true:suggestion +csharp_style_expression_bodied_indexers = true:suggestion +csharp_style_expression_bodied_accessors = true:suggestion +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# From https://github.com/dotnet/roslyn/blob/master/.editorconfig + +# Non-private static fields are PascalCase +dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.symbols = non_private_static_fields +dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.style = non_private_static_field_style + +dotnet_naming_symbols.non_private_static_fields.applicable_kinds = field +dotnet_naming_symbols.non_private_static_fields.applicable_accessibilities = public, protected, internal, protected_internal, private_protected +dotnet_naming_symbols.non_private_static_fields.required_modifiers = static + +dotnet_naming_style.non_private_static_field_style.capitalization = pascal_case + +# Non-private readonly fields are PascalCase +dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.symbols = non_private_readonly_fields +dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.style = non_private_readonly_field_style + +dotnet_naming_symbols.non_private_readonly_fields.applicable_kinds = field +dotnet_naming_symbols.non_private_readonly_fields.applicable_accessibilities = public, protected, internal, protected_internal, private_protected +dotnet_naming_symbols.non_private_readonly_fields.required_modifiers = readonly + +dotnet_naming_style.non_private_readonly_field_style.capitalization = pascal_case + +# Constants are PascalCase +dotnet_naming_rule.constants_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constants_should_be_pascal_case.symbols = constants +dotnet_naming_rule.constants_should_be_pascal_case.style = constant_style + +dotnet_naming_symbols.constants.applicable_kinds = field, local +dotnet_naming_symbols.constants.required_modifiers = const + +dotnet_naming_style.constant_style.capitalization = pascal_case + +# Static fields are camelCase and start with s_ +dotnet_naming_rule.static_fields_should_be_camel_case.severity = suggestion +dotnet_naming_rule.static_fields_should_be_camel_case.symbols = static_fields +dotnet_naming_rule.static_fields_should_be_camel_case.style = static_field_style + +dotnet_naming_symbols.static_fields.applicable_kinds = field +dotnet_naming_symbols.static_fields.required_modifiers = static + +dotnet_naming_style.static_field_style.capitalization = camel_case +dotnet_naming_style.static_field_style.required_prefix = s_ + +# Instance fields are camelCase and start with _ +dotnet_naming_rule.instance_fields_should_be_camel_case.severity = suggestion +dotnet_naming_rule.instance_fields_should_be_camel_case.symbols = instance_fields +dotnet_naming_rule.instance_fields_should_be_camel_case.style = instance_field_style + +dotnet_naming_symbols.instance_fields.applicable_kinds = field + +dotnet_naming_style.instance_field_style.capitalization = camel_case +dotnet_naming_style.instance_field_style.required_prefix = _ + +# Locals and parameters are camelCase +dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion +dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters +dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style + +dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local + +dotnet_naming_style.camel_case_style.capitalization = camel_case + +# Local functions are PascalCase +dotnet_naming_rule.local_functions_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.local_functions_should_be_pascal_case.symbols = local_functions +dotnet_naming_rule.local_functions_should_be_pascal_case.style = local_function_style + +dotnet_naming_symbols.local_functions.applicable_kinds = local_function + +dotnet_naming_style.local_function_style.capitalization = pascal_case + +# By default, name items with PascalCase +dotnet_naming_rule.members_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.members_should_be_pascal_case.symbols = all_members +dotnet_naming_rule.members_should_be_pascal_case.style = pascal_case_style + +dotnet_naming_symbols.all_members.applicable_kinds = * + +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# Blocks are allowed +csharp_prefer_braces = true:silent +csharp_preserve_single_line_blocks = true +csharp_preserve_single_line_statements = true + +# IDE0005: Remove unnecessary usings/imports +dotnet_diagnostic.IDE0005.severity = warning + +# IDE0036: Order modifiers +dotnet_diagnostic.IDE0036.severity = warning + +# IDE0040: Add accessibility modifiers +dotnet_diagnostic.IDE0040.severity = warning + +# IDE0044: Make field readonly +dotnet_diagnostic.IDE0044.severity = warning + +# CONSIDER: Are IDE0051 and IDE0052 too noisy to be warnings for IDE editing scenarios? Should they be made build-only warnings? +# IDE0051: Remove unused private member +dotnet_diagnostic.IDE0051.severity = warning + +# IDE0052: Remove unread private member +dotnet_diagnostic.IDE0052.severity = warning \ No newline at end of file diff --git a/Directory.Build.props b/Directory.Build.props index 06e5a5521..f56a38d8b 100644 Binary files a/Directory.Build.props and b/Directory.Build.props differ diff --git a/Directory.Build.targets b/Directory.Build.targets new file mode 100644 index 000000000..4b45b5873 --- /dev/null +++ b/Directory.Build.targets @@ -0,0 +1,35 @@ + + + + $(DefineConstants);UNITY;AOT + + + $(DefineConstants);LIBRARY + + + $(DefineConstants);FEATURE_BINARY_SERIALIZATION + + + $(DefineConstants);FEATURE_READ_ONLY_COLLECTION;FEATURE_ISET;FEATURE_TAP;FEATURE_EXCEPTION_DISPATCH_INFO;FEATURE_TUPLE + + + + $(DefineConstants);FEATURE_CER;FEATURE_CODE_ACCESS_SECURITY + + + $(DefineConstants);FEATURE_ASYNC_DISPOSABLE;FEATURE_ADVANCED_BIT_CONVERTER;FEATURE_DO_NOT_WRAP_EXCEPTIONS + + + $(DefineConstants);FEATURE_ASYNC_DISPOSABLE;FEATURE_ADVANCED_BIT_CONVERTER;FEATURE_DO_NOT_WRAP_EXCEPTIONS + + + $(DefineConstants);FEATURE_ASYNC_DISPOSABLE;FEATURE_ADVANCED_BIT_CONVERTER;FEATURE_DO_NOT_WRAP_EXCEPTIONS;FEATURE_ENCODING_EXTENSION;FEATURE_UTF8STRING;FEATURE_ADVANCED_OBSOLETE + + + + + <_Parameter1>TargetFramework + <_Parameter2>$(TargetFramework) + + + diff --git a/LICENSE.DotNetFoundation.txt b/LICENSE.DotNetFoundation.txt new file mode 100644 index 000000000..a616ed188 --- /dev/null +++ b/LICENSE.DotNetFoundation.txt @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) .NET Foundation and Contributors + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/doc/dev/JsonCollectionStatemachine.drawio.svg.drawio b/doc/dev/JsonCollectionStatemachine.drawio.svg.drawio new file mode 100644 index 000000000..484b7d2a2 --- /dev/null +++ b/doc/dev/JsonCollectionStatemachine.drawio.svg.drawio @@ -0,0 +1,414 @@ + + + + + + + + + + + + + + +
+
+
match /\s*\[/
+
+
+ + match /\s*\[/ + + + + + + +
+
+
Invalid
+
+
+ + Invalid + + + + + + + + +
+
+
else
+
+
+ + else + + + + + + +
+
+
+ MaybeItem +
+
+
+ + MaybeItem + + + + + + + + +
+
+
Item
+
+
+ + Item + + + + + + + + + + +
+
+
(external)
+
+
+ + (external) + + + + + + + +
+
+
Array
+
+
+ + Array + + + + + + + +
+
+
match /\s*,/
+
+
+ + match /\s*,/ + + + + + +
+
+
else
+
+
+ + else + + + + + + + +
+
+
else
+
+
+ + else + + + + + + + + + +
+
+
match /\s*]/
+
+
+ + match /\s*]/ + + + + + +
+
+
match /\s*]/
+
+
+ + match /\s*]/ + + + + + + + + + +
+
+
+ Object +
+
+
+ + Object + + + + + + + + + + +
+
+
match /\s*\[/
+
+
+ + match /\s*\[/ + + + + + + +
+
+
Invalid
+
+
+ + Invalid + + + + + + + +
+
+
else
+
+
+ + else + + + + + + +
+
+
+ MaybeKey +
+
+
+ + MaybeKey + + + + + + + + +
+
+
Key
+
+
+ + Key + + + + + + + + + + +
+
+
(external)
+
+
+ + (external) + + + + + + + +
+
+
match /\s*,/
+
+
+ + match /\s*,/ + + + + + +
+
+
else
+
+
+ + else + + + + + + + +
+
+
else
+
+
+ + else + + + + + + + + + +
+
+
match /\s*}/
+
+
+ + match /\s*}/ + + + + + +
+
+
match /\s*]/
+
+
+ + match /\s*]/ + + + + + + + + + + +
+
+
Value
+
+
+ + Value + + + + + + + + + + +
+
+
(external)
+
+
+ + (external) + + + + + +
+
+
match /[:=]/
+
+
+ + match /[:=]/ + + + + + + + + + Viewer does not support full SVG 1.1 + + + \ No newline at end of file diff --git a/doc/v2.md b/doc/v2.md new file mode 100644 index 000000000..655032d4e --- /dev/null +++ b/doc/v2.md @@ -0,0 +1,58 @@ +MessagePack for CLI v2 +=== + +Goals +--- + +* Supports .NET Framework 3.5 and 4.8, .NET Core 2.1 and 3.1, Xamarin Android and iOS, UWP, Unity. + * Supproted TFM: + * [ ] `net35` + * [ ] `netstandard1.4` (for .NET 4.8) + * [ ] `netstandard2.0` (for .NET Core 2.1) + * [ ] `netstandard2.1` (for .NET Core 3.1 and after) + * [ ] `MonoAndroid10` (for Xamarin Android AOT) + * [ ] `Xamarin.iOS10` (for Xamarin iOS AOT) + * [ ] `uap` (for .NET Native) + * [ ] (N/A) (for Unity IL2CPP) +* Supports stream based serialization/deserialization for large blobs like v1. +* Supports various format as possible including JSON, CBOF, protbuf, etc. +* Keep high level API compatiblity with v1. +* Improve performance. + +Non-Goals +--- + +* 100% API compatibility with v1. + * Custom `Packer` and custom `Unpacker` will not be supported. +* ABI compatibility with v1. +* Keep performance for compatibility layer espetially custom serializer, IPackable/IUnpackable, and async operation. +* Legacy platform support including Silverlight, .NET Standard 1.x, etc. +* High performance in old platform such as .NET Framework and .NET Core 2.1. +* 100% Unity stability. + +Status +--- + +* [x] Encoder/Decoder layer utlizing `ReadOnlySequence` and `IBufferWriter` +* [x] Simple JSON serializer with some tweak points for design verification. +* [ ] Debugging features +* [ ] Reflection based object serializer +* [ ] Code generator w/ CodDOM or Roslyn +* [ ] ILGenerator based object serializer +* [ ] `SerializationContext` +* [ ] Known serializers +* [ ] Timestamp +* [ ] Multi demensional array +* [ ] Polymorphism +* [ ] v1 compatiblity layer + * [ ] `MessagePackSerializer` + * [ ] `MessagePackObject` + * [ ] `Packer` and `Unpacker` + * [ ] `IPackable` and `IUnpackable` + * other stuff... +* [ ] msgpack for C# compatiblity such as LZ4 support +* [ ] More formats + * [ ] Completes JSON serializer + * [ ] CBOF serializer + * [ ] Protobuf serializer + * other formats... diff --git a/src/Common/CommonAssemblyInfo.cs b/src/Common/CommonAssemblyInfo.cs new file mode 100644 index 000000000..b28a25074 --- /dev/null +++ b/src/Common/CommonAssemblyInfo.cs @@ -0,0 +1,13 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// Universal informations for entire MsgPack for CLI. + +using System; +using System.Runtime.InteropServices; + +[assembly: CLSCompliant(true)] +[assembly: ComVisible(false)] + + diff --git a/src/CommonAssemblyInfo.cs b/src/CommonAssemblyInfo.cs deleted file mode 100644 index 51ac2d17d..000000000 --- a/src/CommonAssemblyInfo.cs +++ /dev/null @@ -1,47 +0,0 @@ -#region -- License Terms -- -// -// MessagePack for CLI -// -// Copyright (C) 2010-2016 FUJIWARA, Yusuke -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#endregion -- License Terms -- - -// Universal informations for entire MsgPack for CLI. - -using System; -using System.Reflection; -using System.Resources; -using System.Runtime.InteropServices; - -[assembly: AssemblyCopyright( "Copyright © FUJIWARA, Yusuke 2010-2017" )] -[assembly: AssemblyProduct( "MessagePack" )] -[assembly: CLSCompliant( true )] -[assembly: ComVisible( false )] -[assembly: NeutralResourcesLanguage( "en-US" )] - -// This version represents API compatibility. -// Major : Represents Major update like re-architecting, remove obsoleted APIs etc. -// Minor : Represents Minor update like adding new feature, obsoleting APIs, fix specification issues, etc. -// Build/Revision : Always 0 since CLI implementations does not care these number, so these changes cause some binding failures. -[assembly: AssemblyVersion( "1.0.0.0" )] - -// This version represents libarary 'version' for human beings. -// Major : Same as AssemblyVersion. -// Minor : Same as AssemblyVersion. -// Build : Bug fixes and improvements, which does not break API contract, but may break some code depends on internal implementation behaviors. -// For example, some programs use reflection to retrieve private fields, analyse human readable exception messages or stack trace, or so. -// Revision : Reserced. It might be used to indicate target platform or patch. -[assembly: AssemblyInformationalVersion( "1.0.0-dev" )] - diff --git a/src/MsgPack.Abstraction/Codecs/CodecFeatures.cs b/src/MsgPack.Abstraction/Codecs/CodecFeatures.cs new file mode 100644 index 000000000..4a0e4f234 --- /dev/null +++ b/src/MsgPack.Abstraction/Codecs/CodecFeatures.cs @@ -0,0 +1,148 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Text; +using MsgPack.Serialization; + +namespace MsgPack.Codecs +{ + /// + /// Represents features of the specific codec. + /// + public sealed class CodecFeatures + { + /// + /// Gets a unique name of the underlying codec. + /// + /// + /// A unique name of the underlying codec which is non-blank string. + /// + public string Name { get; } + + /// + /// Gets a value which indicates the underlying codec supports collection length. + /// + /// + /// true, if the underlyng codec supports collection length; false, otherwise. + /// + /// + /// When this property returns false, following methods should throw . + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public bool CanCountCollectionItems { get; } + + /// + /// Gets a value which indicates the underlying codec supports custom string encoding. + /// + /// + /// true, if the underlyng format supports custom string encoding; false, otherwise. + /// + /// + /// When this property returns false, typed parameters in and methods will be ignored. + /// + public bool CanSpecifyStringEncoding { get; } + + /// + /// Gets a value which indicates the underlying codec supports extension types. + /// + /// + /// true, if the underlyng codec supports extension types; false, otherwise. + /// + /// + /// When this property returns false, following methods should throw . + /// + /// + /// + /// + /// + /// In additon, TExtentionType type parameter should be . + /// + public bool SupportsExtensionTypes { get; } + + /// + /// Gets a value which indicates preferred for the underlying codec. + /// + /// + /// A value which indicates preferred for the underlying codec. + /// + public SerializationMethod PreferredObjectSerializationMethod { get; } + + /// + /// Gets a value which indicates preferred for the underlying codec. + /// + /// + /// A value which indicates preferred for the underlying codec. + /// + public EnumSerializationMethod PreferredEnumSerializationMethod { get; } + + /// + /// Gets a value which indicates preferred for the underlying codec. + /// + /// + /// A value which indicates preferred for the underlying codec. + /// + public DateTimeConversionMethod PreferredDateTimeConversionMethod { get; } + + /// + /// Gets a value which indicates available set as . + /// + /// + /// A value which indicates available set as . + /// + public AvailableSerializationMethods AvailableSerializationMethods { get; } + + /// + /// Gets the default encoding of strings. + /// + /// + /// The default encoding of strings. + /// + public Encoding DefaultStringEncoding { get; } + + /// + /// Gets the precision of fraction portion on ISO 8601 date time string as specified by this codec. + /// + /// + /// The precision of fraction portion on ISO 8601 date time string as specified by this codec. + /// null means that the value will be determined by serialization option or serializer implementation. + /// + /// + /// + /// + public int? Iso8601FractionOfSecondsPrecision { get; } + + /// + /// Gets a separator char for fraction portion by this codec. + /// + /// + /// A separator char for fraction portion. + /// null means that the value will be determined by serialization option or serializer implementation. + /// + public char? Iso8601DecimalSeparator { get; } + + internal CodecFeatures(CodecFeaturesBuilder builder) + { + this.Name = builder.Name; + this.CanCountCollectionItems = builder.CanCountCollectionItems; + this.CanSpecifyStringEncoding = builder.CanSpecifyStringEncoding; + this.SupportsExtensionTypes = builder.SupportsExtensionTypes; + this.PreferredObjectSerializationMethod = builder.PreferredObjectSerializationMethod; + this.PreferredEnumSerializationMethod = builder.PreferredEnumSerializationMethod; + this.PreferredDateTimeConversionMethod = builder.PreferredDateTimeConversionMethod; + this.AvailableSerializationMethods = builder.AvailableSerializationMethods; + this.DefaultStringEncoding = builder.DefaultStringEncoding; + this.Iso8601FractionOfSecondsPrecision = builder.Iso8601FractionOfSecondsPrecision; + this.Iso8601DecimalSeparator = builder.Iso8601DecimalSeparator; + } + } +} diff --git a/src/MsgPack.Abstraction/Codecs/CodecFeaturesBuilder.cs b/src/MsgPack.Abstraction/Codecs/CodecFeaturesBuilder.cs new file mode 100644 index 000000000..53f046b6a --- /dev/null +++ b/src/MsgPack.Abstraction/Codecs/CodecFeaturesBuilder.cs @@ -0,0 +1,290 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Text; +using MsgPack.Internal; +using MsgPack.Serialization; + +namespace MsgPack.Codecs +{ + /// + /// A builder object for . + /// + public sealed class CodecFeaturesBuilder + { + /// + /// Gets a unique name of the underlying codec. + /// + /// + /// A unique name of the underlying codec which is non-blank string. + /// + public string Name { get; } + + /// + /// Gets or sets a value which indicates the underlying codec supports collection length. + /// + /// + /// true, if the underlyng codec supports collection length; false, otherwise. + /// + /// + /// When this property returns false, following methods should throw . + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public bool CanCountCollectionItems { get; set; } + + /// + /// Gets or sets a value which indicates the underlying codec supports custom string encoding. + /// + /// + /// true, if the underlyng format supports custom string encoding; false, otherwise. + /// + /// + /// When this property returns false, typed parameters in and methods will be ignored. + /// + public bool CanSpecifyStringEncoding { get; set; } + + /// + /// Gets or sets a value which indicates the underlying codec supports extension types. + /// + /// + /// true, if the underlyng codec supports extension types; false, otherwise. + /// + /// + /// When this property returns false, following methods should throw . + /// + /// + /// + /// + /// + /// In additon, TExtentionType type parameter should be . + /// + public bool SupportsExtensionTypes { get; set; } + + /// + /// Gets a value which indicates preferred for the underlying codec. + /// + /// + /// A value which indicates preferred for the underlying codec. + /// + public SerializationMethod PreferredObjectSerializationMethod { get; private set; } + + private EnumSerializationMethod _preferredEnumSerializationMethod; + + /// + /// Gets or sets a value which indicates preferred for the underlying codec. + /// + /// + /// A value which indicates preferred for the underlying codec. + /// + /// + /// Setting value is not a defined value. + /// + public EnumSerializationMethod PreferredEnumSerializationMethod + { + get => this._preferredEnumSerializationMethod; + set => this._preferredEnumSerializationMethod = Ensure.Defined(value); + } + + private DateTimeConversionMethod _preferredDateTimeConversionMethod; + + /// + /// Gets or sets a value which indicates preferred for the underlying codec. + /// + /// + /// A value which indicates preferred for the underlying codec. + /// + /// + /// Setting value is not a defined value. + /// + public DateTimeConversionMethod PreferredDateTimeConversionMethod + { + get => this._preferredDateTimeConversionMethod; + set => this._preferredDateTimeConversionMethod = Ensure.Defined(value); + } + + /// + /// Gets a value which indicates available set as . + /// + /// + /// A value which indicates available set as . + /// + public AvailableSerializationMethods AvailableSerializationMethods { get; private set; } + + /// + /// Gets the default encoding of strings. + /// + /// + /// The default encoding of strings. + /// + public Encoding DefaultStringEncoding { get; private set; } = Utf8EncodingNonBomStrict.Instance; + + + /// + /// Gets the precision of fraction portion on ISO 8601 date time string as specified by this codec. + /// + /// + /// The precision of fraction portion on ISO 8601 date time string as specified by this codec. + /// null means that the value will be determined by serialization option or serializer implementation. + /// + /// + /// + /// + public int? Iso8601FractionOfSecondsPrecision { get; private set; } + + /// + /// Gets a separator char for fraction portion for the underlying codec. + /// + /// + /// A separator char for fraction portion. + /// null means that the value will be determined by serialization option or serializer implementation. + /// + public char? Iso8601DecimalSeparator { get; private set; } + + /// + /// Initializes a new instance of object. + /// + /// A unique name of the underlying codec. Note that property will be trimmed value. + /// + /// is null. + /// + /// + /// is empty or contains only white spaces. + /// + public CodecFeaturesBuilder(string name) + { + this.Name = Ensure.NotBlankAndTrimmed(name); + } + + /// + /// Sets preferred and available serialization method for the underlying codec. + /// + /// + /// A value which indicates preferred for the underlying codec. + /// + /// + /// A value which indicates available set as . + /// + /// This instance. + /// + /// value is not a defined value. + /// contains any undefined flag value. + /// + public CodecFeaturesBuilder SetObjectSerializationMethod( + SerializationMethod preferredMethod, + AvailableSerializationMethods availableMethods + ) + { + this.PreferredObjectSerializationMethod = Ensure.Defined(preferredMethod); + this.AvailableSerializationMethods = Ensure.Defined(availableMethods); + return this; + } + + /// + /// Sets the default encoding of strings. + /// + /// The default encoding of strings. + /// This instance. + /// + /// is null. + /// + public CodecFeaturesBuilder SetDefaultStringEncoding(Encoding value) + { + this.DefaultStringEncoding = Ensure.NotNull(value); + return this; + } + + /// + /// Resets the default encoding of strings. + /// + /// This instance. + public CodecFeaturesBuilder ResetDefaultStringEncoding() + { + this.DefaultStringEncoding = Utf8EncodingNonBomStrict.Instance; + return this; + } + + /// + /// Sets the precision of ISO 8601 fraction of seconds portion for the underlying codec. + /// + /// The precision. + /// This instance. + /// + /// is negative value. + /// + /// + /// + /// + public CodecFeaturesBuilder SetIso8601FractionOfSecondsPrecision(int value) + { + this.Iso8601FractionOfSecondsPrecision = Ensure.IsNotLessThan(value, 0); + return this; + } + + /// + /// Resets the precision of ISO 8601 fraction of seconds portion for the underlying codec. + /// + /// This instance. + /// + /// + /// + public CodecFeaturesBuilder ResetIso8601FractionOfSecondsPrecision() + { + this.Iso8601FractionOfSecondsPrecision = null; + return this; + } + + /// + /// Sets the decimal separator of ISO 8601 for the underlying codec. + /// + /// The precision. + /// This instance. + /// + /// is not , nor .. + /// + public CodecFeaturesBuilder SetIso8601DecimalSeparator(char value) + { + switch(value) + { + case ',': + case '.': + { + this.Iso8601DecimalSeparator = value; + break; + } + default: + { + Throw.InvalidIso8601DecimalSeparator(value); + break; // Never reaches + } + } + + return this; + } + + /// + /// Sets the decimal separator of ISO 8601 for the underlying codec. + /// + /// This instance. + public CodecFeaturesBuilder ResetIso8601DecimalSeparator() + { + this.Iso8601DecimalSeparator = null; + return this; + } + + /// + /// Builds new instance of immutable object from this builder instance. + /// + /// New instance of immutable object. + public CodecFeatures Build() => new CodecFeatures(this); + } +} diff --git a/src/MsgPack.Abstraction/Codecs/CodecProvider.cs b/src/MsgPack.Abstraction/Codecs/CodecProvider.cs new file mode 100644 index 000000000..be9460ee4 --- /dev/null +++ b/src/MsgPack.Abstraction/Codecs/CodecProvider.cs @@ -0,0 +1,31 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using MsgPack.Internal; + +namespace MsgPack.Codecs +{ + /// + /// Defines basic features and interfaces of codec provider which provides encoder and decoder. + /// + public abstract class CodecProvider + { + /// + /// Initializes a new instance of object. + /// + protected CodecProvider() { } + + /// + /// Gets an instance of object. + /// + /// An instance of object. + public abstract FormatEncoder GetEncoder(); + + /// + /// Gets an instance of object. + /// + /// An instance of object. + public abstract FormatDecoder GetDecoder(); + } +} diff --git a/src/MsgPack.Abstraction/DecodeException.cs b/src/MsgPack.Abstraction/DecodeException.cs new file mode 100644 index 000000000..87fc53411 --- /dev/null +++ b/src/MsgPack.Abstraction/DecodeException.cs @@ -0,0 +1,69 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +#if FEATURE_BINARY_SERIALIZATION +using System.Runtime.Serialization; +#endif // FEATURE_BINARY_SERIALIZATION + +namespace MsgPack +{ +#warning TODO: -> ParseException + /// + /// Defines common exception for decoding error which is caused by invalid input byte sequence. + /// +#if FEATURE_BINARY_SERIALIZATION + [Serializable] +#endif // FEATURE_BINARY_SERIALIZATION + public abstract class DecodeException : Exception + { + /// + /// Gets the position of the input sequence. + /// + /// + /// The position of the input sequence. + /// + /// + /// This value MAY NOT represents actual position of source byte sequence espetially in asynchronous deserialization + /// because the position stored in this property may reflect the offset in the buffer which is gotten from underlying byte stream rather than the stream position. + /// So, the serializer, which is consumer of the decoder, must provide appropriate exception message with calculated position information with this property. + /// + /// Because of above limitation, this property's value will not included in property nand result. + /// + public long Position { get; } + + protected DecodeException(long position) + : base() + { + this.Position = position; + } + + protected DecodeException(long position, string? message) + : base(message) + { + this.Position = position; + } + + protected DecodeException(long position, string? message, Exception? innerException) + : base(message, innerException) + { + this.Position = position; + } + +#if FEATURE_BINARY_SERIALIZATION + protected DecodeException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + this.Position = info.GetInt64("Position"); + } + + public override void GetObjectData(SerializationInfo info, StreamingContext context) + { + base.GetObjectData(info, context); + info.AddValue("Position", this.Position); + } + +#endif // FEATURE_BINARY_SERIALIZATION + } +} diff --git a/src/MsgPack.Abstraction/Ensure.Enum.cs b/src/MsgPack.Abstraction/Ensure.Enum.cs new file mode 100644 index 000000000..a25224163 --- /dev/null +++ b/src/MsgPack.Abstraction/Ensure.Enum.cs @@ -0,0 +1,77 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; + +namespace MsgPack +{ + internal partial class Ensure + { + public static T Defined(T value, [CallerArgumentExpression("value")] string? paramName = null) + where T : Enum + { + if (EnumValidationHelper.Instance.IsDefined(value)) + { + throw new ArgumentOutOfRangeException(paramName); + } + + return value; + } + + private sealed class EnumValidationHelper + where T : Enum + { + public static readonly EnumValidationHelper Instance = new EnumValidationHelper(); + + private readonly ulong _undefinedFlags; + private readonly ulong[] _constants; + private readonly bool _isFlags; + + private EnumValidationHelper() + { + this._isFlags = typeof(T).IsDefined(typeof(FlagsAttribute)); + var underlyingType = Enum.GetUnderlyingType(typeof(T)); + var values = + (Enum.GetValues(typeof(T)) as object[]) + .Select(v => Convert.ToUInt64(v)) + .ToArray(); + + if (!this._isFlags) + { + Array.Sort(values); + this._constants = values; + this._undefinedFlags = default!; + } + else + { + this._constants = Array.Empty(); + var flags = default(ulong); + foreach(var flag in values) + { + flags |= flag; + } + + this._undefinedFlags = ~flags; + } + } + + public bool IsDefined(T value) + { + var bits = Unsafe.As(ref value); + + if (this._isFlags) + { + return (bits & this._undefinedFlags) == 0; + } + else + { + return Array.BinarySearch(this._constants, bits) >= 0; + } + } + } + } +} diff --git a/src/MsgPack.Abstraction/Ensure.cs b/src/MsgPack.Abstraction/Ensure.cs new file mode 100644 index 000000000..4a9492987 --- /dev/null +++ b/src/MsgPack.Abstraction/Ensure.cs @@ -0,0 +1,124 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace MsgPack +{ + internal static partial class Ensure + { + [return: NotNull] + public static T NotNull([NotNull]T value, [CallerArgumentExpression("value")]string paramName = null!) + { + if (value == null) + { + ThrowArgumentNull(paramName); + } + + return value; + } + + public static string NotNullNorEmpty([NotNull]string? value, [CallerArgumentExpression("value")] string paramName = null!) + { + NotNull(value); + + if (value.Length ==0) + { + ThrowArgumentEmptyString(paramName); + } + + return value; + } + + public static string NotBlank([NotNull] string? value, [CallerArgumentExpression("value")] string paramName = null!) + { + NotNull(value); + + if (String.IsNullOrWhiteSpace(value)) + { + ThrowArgumentBlankString(paramName); + } + + return value; + } + + public static string NotBlankAndTrimmed([NotNull] string? value, [CallerArgumentExpression("value")] string paramName = null!) + { + NotNull(value); + + if (String.IsNullOrWhiteSpace(value)) + { + ThrowArgumentBlankString(paramName); + } + + return value.Trim(); + } + + public static int IsNotNegative(int value, [CallerArgumentExpression("value")]string paramName = null!) + { + if (value < 0) + { + ThrowArgumentOutOfRange(paramName, "Value cannot be negative number."); + } + + return value; + } + + public static T IsNotLessThan(T value, T minInclusive, [CallerArgumentExpression("value")] string paramName = null!) + where T : struct, IComparable + { + if (value.CompareTo(minInclusive) < 0) + { + ThrowArgumentOutOfRange(paramName, $"Value cannot be less than {minInclusive}."); + } + + return value; + } + + public static T IsNotGreaterThan(T value, T maxInclusive, [CallerArgumentExpression("value")] string paramName = null!) + where T : struct, IComparable + { + if (value.CompareTo(maxInclusive) > 0) + { + ThrowArgumentOutOfRange(paramName, $"Value cannot be greater than {maxInclusive}."); + } + + return value; + } + + public static T IsBetween(T value, T minInclusive, T maxInclusive, [CallerArgumentExpression("value")] string paramName = null!) + where T : struct, IComparable + { + if (value.CompareTo(minInclusive) < 0) + { + ThrowArgumentOutOfRange(paramName, $"Value cannot be less than {minInclusive}."); + } + + if (value.CompareTo(maxInclusive) > 0) + { + ThrowArgumentOutOfRange(paramName, $"Value cannot be greater than {maxInclusive}."); + } + + return value; + } + + [DoesNotReturn] + private static void ThrowArgumentNull(string paramName) + => throw new ArgumentNullException(paramName); + + [DoesNotReturn] + private static void ThrowArgumentOutOfRange(string paramName, string message) + => throw new ArgumentOutOfRangeException(paramName, message); + + [DoesNotReturn] + private static void ThrowArgumentEmptyString(string paramName) + => throw new ArgumentException("Value cannot be empty string.", paramName); + + [DoesNotReturn] + private static void ThrowArgumentBlankString(string paramName) + => throw new ArgumentException("Value cannot be empty or blank string.", paramName); + } +} diff --git a/src/MsgPack.Abstraction/ExtensionType.cs b/src/MsgPack.Abstraction/ExtensionType.cs new file mode 100644 index 000000000..08e7cb6e6 --- /dev/null +++ b/src/MsgPack.Abstraction/ExtensionType.cs @@ -0,0 +1,117 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.ComponentModel; + +namespace MsgPack +{ + /// + /// Represents a type of extension. + /// + public readonly struct ExtensionType : IComparable, IEquatable + { + /// + /// Gets raw tag data. + /// + /// Raw tag data. Note that the codec might not allow full length. + /// + /// Valid range and sementics depend on serialization codec. + /// + public long Tag { get; } + + /// + /// Initializes a new instance. + /// This constructor should not be called from application directly. + /// + /// Raw tag data. + [EditorBrowsable(EditorBrowsableState.Never)] + public ExtensionType(long tag) + { + this.Tag = tag; + } + + /// + public bool Equals(ExtensionType other) + => this.Tag == other.Tag; + + /// + public int CompareTo(ExtensionType other) + => this.Tag.CompareTo(other.Tag); + + /// + public override bool Equals(object? obj) + => (obj is ExtensionType other) ? this.Equals(other) : false; + + /// + public override int GetHashCode() + => this.Tag.GetHashCode(); + + /// + /// Determines whether two specified instances of are equal. + /// + /// The first object to compare. + /// The second object to compare. + /// + /// true if and represent the same type; otherwise, false. + /// + public static bool operator ==(ExtensionType left, ExtensionType right) + => left.Tag == right.Tag; + + /// + /// Determines whether two specified instances of are not equal. + /// + /// The first object to compare. + /// The second object to compare. + /// + /// true if and represent the different type; otherwise, false. + /// + public static bool operator !=(ExtensionType left, ExtensionType right) + => left.Tag != right.Tag; + + /// + /// Determines whether one specified object is less than a second specified object. + /// + /// The first object to compare. + /// The second object to compare. + /// + /// true if is less than in their tag; otherwise, false. + /// + public static bool operator <(ExtensionType left, ExtensionType right) + => left.Tag < right.Tag; + + /// + /// Determines whether one specified object is greater than a second specified object. + /// + /// The first object to compare. + /// The second object to compare. + /// + /// true if is greater than in their tag; otherwise, false. + /// + public static bool operator >(ExtensionType left, ExtensionType right) + => left.Tag > right.Tag; + + /// + /// Determines whether one specified object is less than or equal to a second specified object. + /// + /// The first object to compare. + /// The second object to compare. + /// + /// true if is less than or equal to in their tag; otherwise, false. + /// + public static bool operator <=(ExtensionType left, ExtensionType right) + => left.Tag <= right.Tag; + + /// + /// Determines whether one specified object is greater than or equal to a second specified object. + /// + /// The first object to compare. + /// The second object to compare. + /// + /// true if is greater than or equal to in their tag; otherwise, false. + /// + public static bool operator >=(ExtensionType left, ExtensionType right) + => left.Tag >= right.Tag; + } +} diff --git a/src/MsgPack.Abstraction/ExtensionTypeObject.cs b/src/MsgPack.Abstraction/ExtensionTypeObject.cs new file mode 100644 index 000000000..fc2b60731 --- /dev/null +++ b/src/MsgPack.Abstraction/ExtensionTypeObject.cs @@ -0,0 +1,175 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Text; +using MsgPack.Internal; + +namespace MsgPack +{ + /// + /// Represents extension type object data. + /// + /// + /// Valid range and sementics of depend on serialization codec. + /// In addition, some codec does not support extension type at all. + /// + public readonly struct ExtensionTypeObject : IEquatable + { + /// + /// Gets a type of this extension type data. + /// + /// Codec specific type of this extension type data. + public ExtensionType Type { get; } + + /// + /// Gets byte sequence which is body of this extension type data. + /// + /// Byte sequence which is body of this extension type data. + public ReadOnlySequence Body { get; } + + /// + /// Initializes a new instance. + /// + /// Codec specific type of this extension type data. + /// Byte sequence which is body of this extension type data. + public ExtensionTypeObject(ExtensionType type, ReadOnlySequence body) + { + this.Type = type; + this.Body = body; + } + + public override string ToString() + { + const int tokensLength = 3 + 8 + 3 + 20 + 3 + 4 + 4 + 3; + var buffer = StringBuilderCache.Acquire((int)Math.Min(0x2000 + tokensLength, tokensLength + this.Body.Length * 2)); + try + { + this.ToString(buffer, false); + return buffer.ToString(); + } + finally + { + StringBuilderCache.Release(buffer); + } + } + + internal void ToString(StringBuilder buffer, bool isJson) + { + if (isJson) + { + buffer.Append("{ \"TypeCode\": ").Append(this.Type.Tag).Append(", \"Body\": \""); + + var body = this.Body.Slice(0, 0x2000); + + const int byteSpanLength = 96; + const int charSpanLength = byteSpanLength / 3 * 4; + Span charSpan = stackalloc char[charSpanLength]; + Span byteSpan = stackalloc byte[byteSpanLength]; + while (!body.IsEmpty) + { + body.CopyTo(byteSpan); + var byteLength = (int)Math.Min(body.Length, byteSpanLength); + Convert.TryToBase64Chars(byteSpan.Slice(0, byteLength), charSpan, out var charsWritten, Base64FormattingOptions.None); + buffer.Append(charSpan.Slice(0, charsWritten)); + body = body.Slice(byteLength); + } + + buffer.Append("\" }"); + } + else + { + buffer.Append("[").Append(this.Type.Tag).Append("]0x"); + var body = this.Body.Slice(0, 0x5000000); + while (!body.IsEmpty) + { + Binary.ToHexString(body.FirstSpan, buffer); + body = body.Slice(body.First.Length); + } + } + } + + + /// + public override bool Equals(object? obj) + => obj is ExtensionTypeObject other ? this.Equals(other) : false; + + /// + public bool Equals(ExtensionTypeObject other) + { + if (this.Type != other.Type) + { + return false; + } + + if (this.Body.Length != other.Body.Length) + { + return false; + } + + if (this.Body.IsSingleSegment && other.Body.IsSingleSegment) + { + return this.Body.FirstSpan.SequenceEqual(other.Body.FirstSpan); + } + + var thisBody = this.Body; + var otherBody = other.Body; + do + { + var length = Math.Min(thisBody.FirstSpan.Length, otherBody.FirstSpan.Length); + var left = thisBody.FirstSpan.Slice(0, length); + var right = otherBody.FirstSpan.Slice(0, length); + if (!left.SequenceEqual(right)) + { + return false; + } + + thisBody = thisBody.Slice(length); + otherBody = otherBody.Slice(length); + } while (!thisBody.IsEmpty); + + Debug.Assert(otherBody.IsEmpty); + + return true; + } + + /// + public override int GetHashCode() + { + var hashCode = this.Type.GetHashCode(); + var body = this.Body; + while(!body.IsEmpty) + { + hashCode ^= FarmHash.Hash32WithSeed(body.FirstSpan, FarmHash.DefaultSeed); + body = body.Slice(body.FirstSpan.Length); + } + + return hashCode; + } + + /// + /// Determines whether two specified instances of are equal. + /// + /// The first object to compare. + /// The second object to compare. + /// + /// true if and represent the same type; otherwise, false. + /// + public static bool operator ==(ExtensionTypeObject left, ExtensionTypeObject right) + => left.Equals(right); + + /// + /// Determines whether two specified instances of are not equal. + /// + /// The first object to compare. + /// The second object to compare. + /// + /// true if and represent the different type; otherwise, false. + /// + public static bool operator !=(ExtensionTypeObject left, ExtensionTypeObject right) + => !left.Equals(right); + } +} diff --git a/src/MsgPack.Abstraction/InsufficientInputException.cs b/src/MsgPack.Abstraction/InsufficientInputException.cs new file mode 100644 index 000000000..5850afb43 --- /dev/null +++ b/src/MsgPack.Abstraction/InsufficientInputException.cs @@ -0,0 +1,36 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +#if FEATURE_BINARY_SERIALIZATION +using System.Runtime.Serialization; +#endif // FEATURE_BINARY_SERIALIZATION + +namespace MsgPack +{ + /// + /// An exception thrown if there are no more inputs when be requested. + /// +#if FEATURE_BINARY_SERIALIZATION + [Serializable] +#endif // FEATURE_BINARY_SERIALIZATION + public sealed class InsufficientInputException : DecodeException + { + public InsufficientInputException(long position) + : this(position, "There are no more inputs.") { } + + public InsufficientInputException(long position, string? message) + : base(position, message) { } + + public InsufficientInputException(long position, string? message, Exception? innerException) + : base(position, message, innerException) { } + +#if FEATURE_BINARY_SERIALIZATION + + private InsufficientInputException(SerializationInfo info, StreamingContext context) + : base(info, context) { } + +#endif // FEATURE_BINARY_SERIALIZATION + } +} diff --git a/src/MsgPack.Abstraction/Internal/CollectionContext.cs b/src/MsgPack.Abstraction/Internal/CollectionContext.cs new file mode 100644 index 000000000..48947396e --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/CollectionContext.cs @@ -0,0 +1,50 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +namespace MsgPack.Internal +{ + public partial struct CollectionContext + { + public static CollectionContext Default => + new CollectionContext( + OptionsDefaults.MaxArrayLength, + OptionsDefaults.MaxMapCount, + OptionsDefaults.MaxDepth, + currentDepth: 0 + ); + + public int MaxArrayLength { get; } + public int MaxMapCount { get; } + public int MaxDepth { get; } + public int CurrentDepth { get; private set; } + + internal CollectionContext(int maxArrayLength, int maxMapCount, int maxDepth, int currentDepth) + { + this.MaxArrayLength = maxArrayLength; + this.MaxMapCount = maxMapCount; + this.MaxDepth = Ensure.IsNotLessThan(maxDepth, 1); + this.CurrentDepth = currentDepth; + } + + public int IncrementDepth(long position) + { + if (this.CurrentDepth == this.MaxDepth) + { + Throw.DepthExeeded(position, this.MaxDepth); + } + + return this.CurrentDepth++; + } + + public int DecrementDepth() + { + if (this.CurrentDepth == 0) + { + Throw.DepthUnderflow(); + } + + return this.CurrentDepth--; + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/CollectionItemIterator.cs b/src/MsgPack.Abstraction/Internal/CollectionItemIterator.cs new file mode 100644 index 000000000..3e0deb847 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/CollectionItemIterator.cs @@ -0,0 +1,84 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Buffers; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + public struct CollectionItemIterator + { + public delegate bool CollectionEndDetection(ref SequenceReader source, ref long nextItemIndex, long itemsCount, out int requestHint); + + private readonly long _itemsCount; + private long _nextItemIndex; + private readonly CollectionEndDetection _collectionEnds; + + public CollectionItemIterator( + CollectionEndDetection moveNext, + long itemsCount + ) + { + this._collectionEnds = Ensure.NotNull(moveNext); + this._itemsCount = itemsCount; + this._nextItemIndex = 0; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public bool CollectionEnds(ref SequenceReader source) + { + var result = this._collectionEnds(ref source, ref this._nextItemIndex, this._itemsCount, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInputForDetectCollectionEnds(source.Consumed, requestHint); + } + + return result; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public bool CollectionEnds(ref SequenceReader source, out int requestHint) + => this._collectionEnds(ref source, ref this._nextItemIndex, this._itemsCount, out requestHint); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public bool CollectionEnds(in ReadOnlySequence source, out int requestHint) + { + var reader = new SequenceReader(source); + return this.CollectionEnds(ref reader, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void Drain(ref SequenceReader source) + { + if (!this.Drain(ref source, out var requestHint)) + { + Throw.InsufficientInputForDrainCollectionItems(source.Consumed, requestHint); + } + } + +#warning TODO: CancellationToken + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public bool Drain(ref SequenceReader source, out int requestHint) + { + while (!this.CollectionEnds(ref source, out requestHint)) + { + if (requestHint != 0) + { + return false; + } + } + + return true; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public bool Drain(ref ReadOnlySequence source, out int requestHint) + { + var reader = new SequenceReader(source); + var ends = this.Drain(ref reader, out requestHint); + source = source.Slice((int)reader.Consumed); + return ends; + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/CollectionType.cs b/src/MsgPack.Abstraction/Internal/CollectionType.cs new file mode 100644 index 000000000..ad9a9fadd --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/CollectionType.cs @@ -0,0 +1,74 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + /// + /// Represents collection type. + /// + public readonly struct CollectionType : IEquatable + { + public static readonly CollectionType None = default; + public static readonly CollectionType Null = new CollectionType(1); + public static readonly CollectionType Array = new CollectionType(2); + public static readonly CollectionType Map = new CollectionType(3); + + private readonly int _type; + + public bool IsNull + { + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + get => this._type == 1; + } + + public bool IsArray + { + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + get => this._type == 2; + } + + public bool IsMap + { + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + get => this._type == 3; + } + + public bool IsNone + { + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + get => this._type == 0; + } + + private CollectionType(int type) + { + this._type = type; + } + + public override bool Equals(object? obj) + => obj is CollectionType other ? this.Equals(other) : false; + + public bool Equals(CollectionType other) + => this._type == other._type; + + public override int GetHashCode() + => this._type; + + public override string ToString() + => this._type switch + { + 1 => "Array", + 2 => "Map", + _ => String.Empty + }; + + public static bool operator ==(CollectionType left, CollectionType right) + => left.Equals(right); + + public static bool operator !=(CollectionType left, CollectionType right) + => !left.Equals(right); + } +} diff --git a/src/MsgPack.Abstraction/Internal/DecodeItemResult.cs b/src/MsgPack.Abstraction/Internal/DecodeItemResult.cs new file mode 100644 index 000000000..67ca604f3 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/DecodeItemResult.cs @@ -0,0 +1,78 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ +#warning TODO: Decode -> Parse, Encode -> Format + /// + /// Represents a result of . + /// + public readonly struct DecodeItemResult + { + public bool HasValue => this.ElementType != ElementType.None; + public ElementType ElementType { get; } + + // If ElementType is string , this value is not decoded. + public ReadOnlySequence Value { get; } + public CollectionItemIterator CollectionIterator { get; } + public long CollectionLength { get; } + public ExtensionType ExtensionType { get; } + public ReadOnlySequence ExtensionBody => this.Value; + + private DecodeItemResult( + ElementType elementType, + in ReadOnlySequence value = default, + in CollectionItemIterator collectionIterator = default, + long collectionLength = default, + ExtensionType extensionType = default + ) + { + this.ElementType = elementType; + this.Value = value; + this.CollectionIterator = collectionIterator; + this.CollectionLength = collectionLength; + this.ExtensionType = extensionType; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static DecodeItemResult CollectionHeader(ElementType elementType, in CollectionItemIterator iterator, long length = -1) + => new DecodeItemResult(elementType, collectionIterator: iterator, collectionLength: length); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static DecodeItemResult ScalarOrSequence(ElementType elementType, ReadOnlyMemory value) + => ScalarOrSequence(elementType, new ReadOnlySequence(value)); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static DecodeItemResult ScalarOrSequence(ElementType elementType, in ReadOnlySequence value) + => new DecodeItemResult(elementType, value: value); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static DecodeItemResult Null() + => new DecodeItemResult(ElementType.Null); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static DecodeItemResult True() + => new DecodeItemResult(ElementType.True); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static DecodeItemResult False() + => new DecodeItemResult(ElementType.False); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static DecodeItemResult ExtensionTypeObject(ExtensionType extensionType, in ReadOnlySequence body) + => new DecodeItemResult(ElementType.Extension, extensionType: extensionType, value: body); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static DecodeItemResult InsufficientInput() + => new DecodeItemResult(ElementType.None); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static DecodeItemResult EndOfStream() + => new DecodeItemResult(ElementType.None); + } +} diff --git a/src/MsgPack.Abstraction/Internal/ElementType.cs b/src/MsgPack.Abstraction/Internal/ElementType.cs new file mode 100644 index 000000000..4c69521a5 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/ElementType.cs @@ -0,0 +1,25 @@ +using System; + +namespace MsgPack.Internal +{ + public enum ElementType + { + None = 0, + Int32 = 1, + Int64 = 2, + UInt64 = 3, + Single = 4, + Double = 5, + True = 6, + False = 7, + Null = 8, + Array = 0x11, + Map = 0x12, + String = 0x31, + Binary = 0x32, + Extension = 0x41, + Whitespace = 0x51, + Comment = 0x52, + OtherTrivia = 0x5F + } +} diff --git a/src/MsgPack.Abstraction/Internal/FarmHash.IntelCommon.cs b/src/MsgPack.Abstraction/Internal/FarmHash.IntelCommon.cs new file mode 100644 index 000000000..705e763d5 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FarmHash.IntelCommon.cs @@ -0,0 +1,78 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +namespace MsgPack.Internal +{ + internal partial class FarmHash + { + // Common methods for Intel compatible CPU intrinsics and corresponds 64bit methods. + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe ulong Fetch64(byte* p) + => *(ulong*)p; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe Vector128 Fetch128(byte* s) + => Sse2.LoadVector128(s).AsUInt32(); // _mm_loadu_si128 + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Rotate64(ulong val, int shift) + // Avoid shifting by 64: doing so yields an undefined result. + => shift == 0 ? val : ((val >> shift) | (val << (64 - shift))); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Add(Vector128 x, Vector128 y) + => Sse2.Add(x, y); // _mm_add_epi32 + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Xor(Vector128 x, Vector128 y) + => Sse2.Xor(x, y); // _mm_xor_si128 + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Or(Vector128 x, Vector128 y) + => Sse2.Or(x, y); // _mm_or_si128 + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Mul(Vector128 x, Vector128 y) + => Sse41.MultiplyLow(x, y); // _mm_mullo_epi32 + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Mul5(Vector128 x) + => Add( + x, + Sse2.ShiftLeftLogical(x, 2) // _mm_slli_epi32 + ); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 RotateLeft(Vector128 x, byte c) + => Or( + Sse2.ShiftLeftLogical(x, c),// _mm_slli_epi32 + Sse2.ShiftRightLogical(x, (byte)(32 - c)) // _mm_srli_epi32 + ); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Rol17(Vector128 x) + => RotateLeft(x, 17); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Rol19(Vector128 x) + => RotateLeft(x, 19); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Shuffle0321(Vector128 x) + => Sse2.Shuffle(x, (0 << 6) + (3 << 4) + (2 << 2) + (1 << 0)); // _mm_shuffle_epi32 + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Swap(ref ulong left, ref ulong right) + { + var temp = left; + left = right; + right = temp; + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FarmHash.MK.cs b/src/MsgPack.Abstraction/Internal/FarmHash.MK.cs new file mode 100644 index 000000000..43f0a1e96 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FarmHash.MK.cs @@ -0,0 +1,160 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// This file is ported from Google's FarmHash (https://github.com/google/farmhash/) +// Original copyright notice is following: + +// Copyright (c) 2014 Google, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// FarmHash, by Geoff Pike + +namespace MsgPack.Internal +{ + internal partial class FarmHash + { + internal static class MK + { + internal static unsafe uint Hash32Len13to24(byte* s, uint len, uint seed = 0) + { + uint a = Fetch32(s - 4 + (len >> 1)); + uint b = Fetch32(s + 4); + uint c = Fetch32(s + len - 8); + uint d = Fetch32(s + (len >> 1)); + uint e = Fetch32(s); + uint f = Fetch32(s + len - 4); + uint h = d * c1 + len + seed; + a = Rotate32(a, 12) + f; + h = Mur(c, h) + a; + a = Rotate32(a, 3) + c; + h = Mur(e, h) + a; + a = Rotate32(a + f, 12) + d; + h = Mur(b ^ seed, h) + a; + return Fmix(h); + } + + internal static unsafe uint Hash32Len0to4(byte* s, uint len, uint seed = 0) + { + uint b = seed; + uint c = 9; + for (var i = 0; i < len; i++) + { + var v = s[i]; + b = b * c1 + v; + c ^= b; + } + return Fmix(Mur(b, Mur(len, c))); + } + + internal static unsafe uint Hash32Len5to12(byte* s, uint len, uint seed = 0) + { + uint a = len, b = len * 5, c = 9, d = b + seed; + a += Fetch32(s); + b += Fetch32(s + len - 4); + c += Fetch32(s + ((len >> 1) & 4)); + return Fmix(seed ^ Mur(c, Mur(b, Mur(a, d)))); + } + + public static unsafe uint Hash32(byte* s, uint len) + { + if (len <= 24) + { + return len <= 12 ? + (len <= 4 ? Hash32Len0to4(s, len) : Hash32Len5to12(s, len)) : + Hash32Len13to24(s, len); + } + + // len > 24 + uint h = len, g = c1 * len, f = g; + uint a0 = Rotate32(Fetch32(s + len - 4) * c1, 17) * c2; + uint a1 = Rotate32(Fetch32(s + len - 8) * c1, 17) * c2; + uint a2 = Rotate32(Fetch32(s + len - 16) * c1, 17) * c2; + uint a3 = Rotate32(Fetch32(s + len - 12) * c1, 17) * c2; + uint a4 = Rotate32(Fetch32(s + len - 20) * c1, 17) * c2; + h ^= a0; + h = Rotate32(h, 19); + h = h * 5 + 0xe6546b64; + h ^= a2; + h = Rotate32(h, 19); + h = h * 5 + 0xe6546b64; + g ^= a1; + g = Rotate32(g, 19); + g = g * 5 + 0xe6546b64; + g ^= a3; + g = Rotate32(g, 19); + g = g * 5 + 0xe6546b64; + f += a4; + f = Rotate32(f, 19) + 113; + uint iters = (len - 1) / 20; + do + { + uint a = Fetch32(s); + uint b = Fetch32(s + 4); + uint c = Fetch32(s + 8); + uint d = Fetch32(s + 12); + uint e = Fetch32(s + 16); + h += a; + g += b; + f += c; + h = Mur(d, h) + e; + g = Mur(c, g) + a; + f = Mur(b + e * c1, f) + d; + f += g; + g += f; + s += 20; + } while (--iters != 0); + g = Rotate32(g, 11) * c1; + g = Rotate32(g, 17) * c1; + f = Rotate32(f, 11) * c1; + f = Rotate32(f, 17) * c1; + h = Rotate32(h + g, 19); + h = h * 5 + 0xe6546b64; + h = Rotate32(h, 17) * c1; + h = Rotate32(h + f, 19); + h = h * 5 + 0xe6546b64; + h = Rotate32(h, 17) * c1; + return h; + } + + public static unsafe uint Hash32WithSeed(byte* s, uint len, uint seed) + { + if (len <= 24) + { + if (len >= 13) + { + return Hash32Len13to24(s, len, seed * c1); + } + else if (len >= 5) + { + return Hash32Len5to12(s, len, seed); + } + else + { + return Hash32Len0to4(s, len, seed); + } + } + + uint h = Hash32Len13to24(s, 24, seed ^ len); + return Mur(Hash32(s + 24, len - 24) + seed, h); + } + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FarmHash.NA.cs b/src/MsgPack.Abstraction/Internal/FarmHash.NA.cs new file mode 100644 index 000000000..c9f76b8c5 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FarmHash.NA.cs @@ -0,0 +1,238 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// This file is ported from Google's FarmHash (https://github.com/google/farmhash/) +// Original copyright notice is following: + +// Copyright (c) 2014 Google, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// FarmHash, by Geoff Pike + +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + internal partial class FarmHash + { + private static class NA + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Hash128to64((ulong low, ulong high) x) + { + // Murmur-inspired hashing. + const ulong kMul = 0x9ddfea08eb382d69UL; + ulong a = (x.low ^ x.high) * kMul; + a ^= (a >> 47); + ulong b = (x.high ^ a) * kMul; + b ^= (b >> 47); + b *= kMul; + return b; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe ulong ShiftMix(ulong val) + { + return val ^ (val >> 47); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe ulong HashLen16(ulong u, ulong v) + { + return Hash128to64((u, v)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe ulong HashLen16(ulong u, ulong v, ulong mul) + { + // Murmur-inspired hashing. + ulong a = (u ^ v) * mul; + a ^= (a >> 47); + ulong b = (v ^ a) * mul; + b ^= (b >> 47); + b *= mul; + return b; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe ulong HashLen0to16(byte* s, uint len) + { + if (len >= 8) + { + ulong mul = k2 + len * 2; + ulong a = Fetch64(s) + k2; + ulong b = Fetch64(s + len - 8); + ulong c = Rotate64(b, 37) * mul + a; + ulong d = (Rotate64(a, 25) + b) * mul; + return HashLen16(c, d, mul); + } + if (len >= 4) + { + ulong mul = k2 + len * 2; + ulong a = Fetch32(s); + return HashLen16(len + (a << 3), Fetch32(s + len - 4), mul); + } + if (len > 0) + { + byte a = s[0]; + byte b = s[len >> 1]; + byte c = s[len - 1]; + uint y = (uint)a + ((uint)b << 8); + uint z = len + ((uint)c << 2); + return ShiftMix(y * k2 ^ z * k0) * k2; + } + return k2; + } + + // This probably works well for 16-byte strings as well, but it may be overkill + // in that case. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe ulong HashLen17to32(byte* s, uint len) + { + ulong mul = k2 + len * 2; + ulong a = Fetch64(s) * k1; + ulong b = Fetch64(s + 8); + ulong c = Fetch64(s + len - 8) * mul; + ulong d = Fetch64(s + len - 16) * k2; + return HashLen16(Rotate64(a + b, 43) + Rotate64(c, 30) + d, + a + Rotate64(b + k2, 18) + c, mul); + } + + // Return a 16-byte hash for 48 bytes. Quick and dirty. + // Callers do best to use "random-looking" values for a and b. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe (ulong first, ulong second) WeakHashLen32WithSeeds( + ulong w, ulong x, ulong y, ulong z, ulong a, ulong b) + { + a += w; + b = Rotate64(b + a + z, 21); + ulong c = a; + a += x; + a += y; + b += Rotate64(a, 44); + return (a + z, b + c); + } + + // Return a 16-byte hash for s[0] ... s[31], a, and b. Quick and dirty. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe (ulong first, ulong second) WeakHashLen32WithSeeds(byte* s, ulong a, ulong b) + { + return WeakHashLen32WithSeeds(Fetch64(s), + Fetch64(s + 8), + Fetch64(s + 16), + Fetch64(s + 24), + a, + b); + } + + // Return an 8-byte hash for 33 to 64 bytes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe ulong HashLen33to64(byte* s, uint len) + { + ulong mul = k2 + len * 2; + ulong a = Fetch64(s) * k2; + ulong b = Fetch64(s + 8); + ulong c = Fetch64(s + len - 8) * mul; + ulong d = Fetch64(s + len - 16) * k2; + ulong y = Rotate64(a + b, 43) + Rotate64(c, 30) + d; + ulong z = HashLen16(y, a + Rotate64(b + k2, 18) + c, mul); + ulong e = Fetch64(s + 16) * mul; + ulong f = Fetch64(s + 24); + ulong g = (y + Fetch64(s + len - 32)) * mul; + ulong h = (z + Fetch64(s + len - 24)) * mul; + return HashLen16(Rotate64(e + f, 43) + Rotate64(g, 30) + h, + e + Rotate64(f + a, 18) + g, mul); + } + + public static unsafe ulong Hash64(byte* s, uint len) + { + const ulong seed = 81; + if (len <= 32) + { + if (len <= 16) + { + return HashLen0to16(s, len); + } + else + { + return HashLen17to32(s, len); + } + } + else if (len <= 64) + { + return HashLen33to64(s, len); + } + + // For strings over 64 bytes we loop. Internal state consists of + // 56 bytes: v, w, x, y, and z. + ulong x = seed; + ulong y = unchecked(seed * k1) + 113; + ulong z = ShiftMix(y * k2 + 113) * k2; + (ulong first, ulong second) v = (0, 0); + (ulong first, ulong second) w = (0, 0); + x = x * k2 + Fetch64(s); + + // Set end so that after the loop we have 1 to 64 bytes left to process. + byte* end = s + ((len - 1) / 64) * 64; + byte* last64 = end + ((len - 1) & 63) - 63; + Debug.Assert(s + len - 64 == last64); + do + { + x = Rotate64(x + y + v.first + Fetch64(s + 8), 37) * k1; + y = Rotate64(y + v.second + Fetch64(s + 48), 42) * k1; + x ^= w.second; + y += v.first + Fetch64(s + 40); + z = Rotate64(z + w.first, 33) * k1; + v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first); + w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16)); + Swap(ref z, ref x); + s += 64; + } while (s != end); + ulong mul = k1 + ((z & 0xff) << 1); + // Make s point to the last 64 bytes of input. + s = last64; + w.first += ((len - 1) & 63); + v.first += w.first; + w.first += v.first; + x = Rotate64(x + y + v.first + Fetch64(s + 8), 37) * mul; + y = Rotate64(y + v.second + Fetch64(s + 48), 42) * mul; + x ^= w.second * 9; + y += v.first * 9 + Fetch64(s + 40); + z = Rotate64(z + w.first, 33) * mul; + v = WeakHashLen32WithSeeds(s, v.second * mul, x + w.first); + w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16)); + Swap(ref z, ref x); + return HashLen16(HashLen16(v.first, w.first, mul) + ShiftMix(y) * k0 + z, + HashLen16(v.second, w.second, mul) + x, + mul); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe ulong Hash64WithSeed(byte* s, uint len, ulong seed) + => Hash64WithSeeds(s, len, k2, seed); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe ulong Hash64WithSeeds(byte* s, uint len, ulong seed0, ulong seed1) + => HashLen16(Hash64(s, len) - seed0, seed1); + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FarmHash.NT.cs b/src/MsgPack.Abstraction/Internal/FarmHash.NT.cs new file mode 100644 index 000000000..36d671bd5 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FarmHash.NT.cs @@ -0,0 +1,43 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// This file is ported from Google's FarmHash (https://github.com/google/farmhash/) +// Original copyright notice is following: + +// Copyright (c) 2014 Google, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// FarmHash, by Geoff Pike + +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + internal partial class FarmHash + { + internal static class NT + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe uint Hash32WithSeed(byte* s, uint len, uint seed) + => (uint)(TE.Hash64WithSeed(s, len, seed) & 0xFFFFFFFF); + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FarmHash.SA.cs b/src/MsgPack.Abstraction/Internal/FarmHash.SA.cs new file mode 100644 index 000000000..0bfb2ad24 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FarmHash.SA.cs @@ -0,0 +1,211 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// This file is ported from Google's FarmHash (https://github.com/google/farmhash/) +// Original copyright notice is following: + +// Copyright (c) 2014 Google, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// FarmHash, by Geoff Pike + +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +using __m128i = System.Runtime.Intrinsics.Vector128; + +namespace MsgPack.Internal +{ + internal partial class FarmHash + { + internal static class SA + { + public static unsafe uint Hash32(byte* s, uint len) + { + const uint seed = 81; + if (len <= 24) + { + return len <= 12 ? + (len <= 4 ? + MK.Hash32Len0to4(s, len) : + MK.Hash32Len5to12(s, len)) : + MK.Hash32Len13to24(s, len); + } + + if (len < 40) + { + uint a = len, b = unchecked(seed * c2), c = a + b; + a += Fetch32(s + len - 4); + b += Fetch32(s + len - 20); + c += Fetch32(s + len - 16); + uint d = a; + a = Rotate32(a, 21); + a = Mur(a, Mur(b, Mur(c, d))); + a += Fetch32(s + len - 12); + b += Fetch32(s + len - 8); + d += a; + a += d; + b = Mur(b, d) * c2; + a = Sse42.Crc32(a, b + c); // _mm_crc32_u32 + return MK.Hash32Len13to24(s, (len + 1) / 2, a) + b; + } + + __m128i cc1 = Vector128.Create(c1); // _mm_set1_epi32 + __m128i cc2 = Vector128.Create(c2); // _mm_set1_epi32 + __m128i h = Vector128.Create(seed); // _mm_set1_epi32 + __m128i g = Vector128.Create(unchecked(c1 * seed)); // _mm_set1_epi32 + __m128i f = g; + __m128i k = Vector128.Create(0xe6546b64); // _mm_set1_epi32 + if (len < 80) + { + __m128i a = Fetch128(s); + __m128i b = Fetch128(s + 16); + __m128i c = Fetch128(s + (len - 15) / 2); + __m128i d = Fetch128(s + len - 32); + __m128i e = Fetch128(s + len - 16); + h = Add(h, a); + g = Add(g, b); + g = Shuffle0321(g); + f = Add(f, c); + __m128i be = Add(b, Mul(e, cc1)); + h = Add(h, f); + f = Add(f, h); + h = Add(Add(k, Mul5(Rol19(Xor(Mul(Rol17(Mul(d, cc1)), cc2), (h))))), e); + k = Xor(k, Ssse3.Shuffle(g.AsByte(), f.AsByte()).AsUInt32()); // _mm_shuffle_epi8 + g = Add(Xor(c, g), a); + f = Add(Xor(be, f), d); + k = Add(k, be); + k = Add(k, Ssse3.Shuffle(f.AsByte(), h.AsByte()).AsUInt32()); // _mm_shuffle_epi8 + f = Add(f, g); + g = Add(g, f); + g = Add( + Vector128.Create(len), // _mm_set1_epi32 + Mul(g, cc1) + ); + } + else + { + // len >= 80 + // The following is loosely modelled after farmhashmk::Hash32. + uint iters = (len - 1) / 80; + len -= iters * 80; + + while (iters-- != 0) + { + // + __m128i a = Fetch128(s); + __m128i b = Fetch128(s + 16); + __m128i c = Fetch128(s + 32); + __m128i d = Fetch128(s + 48); + __m128i e = Fetch128(s + 64); + h = Add(h, a); + g = Add(g, b); + g = Shuffle0321(g); + f = Add(f, c); + __m128i be = Add(b, Mul(e, cc1)); + h = Add(h, f); + f = Add(f, h); + h = Add(Add(k, Mul5(Rol19(Xor(Mul(Rol17(Mul(d, cc1)), cc2), (h))))), e); + k = Xor(k, Ssse3.Shuffle(g.AsByte(), f.AsByte()).AsUInt32()); // _mm_shuffle_epi8 + g = Add(Xor(c, g), a); + f = Add(Xor(be, f), d); + k = Add(k, be); + k = Add(k, Ssse3.Shuffle(f.AsByte(), h.AsByte()).AsUInt32()); // _mm_shuffle_epi8 + f = Add(f, g); + g = Add(g, f); + f = Mul(f, cc1); + // + s += 80; + } + + if (len != 0) + { + h = Add(h, Vector128.Create(len)); // _mm_set1_epi32 + s = s + len - 80; + // + __m128i a = Fetch128(s); + __m128i b = Fetch128(s + 16); + __m128i c = Fetch128(s + 32); + __m128i d = Fetch128(s + 48); + __m128i e = Fetch128(s + 64); + h = Add(h, a); + g = Add(g, b); + g = Shuffle0321(g); + f = Add(f, c); + __m128i be = Add(b, Mul(e, cc1)); + h = Add(h, f); + f = Add(f, h); + h = Add(Add(k, Mul5(Rol19(Xor(Mul(Rol17(Mul(d, cc1)), cc2), (h))))), e); + k = Xor(k, Ssse3.Shuffle(g.AsByte(), f.AsByte()).AsUInt32()); // _mm_shuffle_epi8 + g = Add(Xor(c, g), a); + f = Add(Xor(be, f), d); + k = Add(k, be); + k = Add(k, Ssse3.Shuffle(f.AsByte(), h.AsByte()).AsUInt32()); // _mm_shuffle_epi8 + f = Add(f, g); + g = Add(g, f); + f = Mul(f, cc1); + // + } + } + + g = Shuffle0321(g); + k = Xor(k, g); + f = Mul(f, cc1); + k = Mul(k, cc2); + g = Mul(g, cc1); + h = Mul(h, cc2); + k = Add(k, Ssse3.Shuffle(g.AsByte(), f.AsByte()).AsUInt32()); // _mm_shuffle_epi8 + h = Add(h, f); + f = Add(f, h); + g = Add(g, k); + k = Add(k, g); + k = Xor(k, Ssse3.Shuffle(f.AsByte(), h.AsByte()).AsUInt32()); // _mm_shuffle_epi8 + var buf = stackalloc __m128i[4]; + buf[0] = f; + buf[1] = g; + buf[2] = k; + buf[3] = h; + s = (byte*)(buf); + uint x = Fetch32(s); + uint y = Fetch32(s + 4); + uint z = Fetch32(s + 8); + x = Sse42.Crc32(x, Fetch32(s + 12)); // _mm_crc32_u32 + y = Sse42.Crc32(y, Fetch32(s + 16)); // _mm_crc32_u32 + z = Sse42.Crc32(z * c1, Fetch32(s + 20)); // _mm_crc32_u32 + x = Sse42.Crc32(x, Fetch32(s + 24)); // _mm_crc32_u32 + y = Sse42.Crc32(y * c1, Fetch32(s + 28)); // _mm_crc32_u32 + uint o = y; + z = Sse42.Crc32(z, Fetch32(s + 32)); // _mm_crc32_u32 + x = Sse42.Crc32(x * c1, Fetch32(s + 36)); // _mm_crc32_u32 + y = Sse42.Crc32(y, Fetch32(s + 40)); // _mm_crc32_u32 + z = Sse42.Crc32(z * c1, Fetch32(s + 44)); // _mm_crc32_u32 + x = Sse42.Crc32(x, Fetch32(s + 48)); // _mm_crc32_u32 + y = Sse42.Crc32(y * c1, Fetch32(s + 52)); // _mm_crc32_u32 + z = Sse42.Crc32(z, Fetch32(s + 56)); // _mm_crc32_u32 + x = Sse42.Crc32(x, Fetch32(s + 60)); // _mm_crc32_u32 + return (o - x + y - z) * c1; + } + + public static unsafe uint Hash32WithSeed(byte* s, uint len, uint seed, uint h) + => Sse42.Crc32(Hash32(s + 24, len - 24) + seed, h); // _mm_crc32_u32 + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FarmHash.SU.cs b/src/MsgPack.Abstraction/Internal/FarmHash.SU.cs new file mode 100644 index 000000000..7eb408425 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FarmHash.SU.cs @@ -0,0 +1,239 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// This file is ported from Google's FarmHash (https://github.com/google/farmhash/) +// Original copyright notice is following: + +// Copyright (c) 2014 Google, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// FarmHash, by Geoff Pike + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +using __m128i = System.Runtime.Intrinsics.Vector128; + +namespace MsgPack.Internal +{ + internal partial class FarmHash + { + internal static class SU + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Swap(ref __m128i left, ref __m128i right) + { + var temp = left; + left = right; + right = temp; + } + + public static unsafe uint Hash32(byte* s, uint len) + { + const uint seed = 81; + if (len <= 24) + { + return len <= 12 ? + (len <= 4 ? + MK.Hash32Len0to4(s, len) : + MK.Hash32Len5to12(s, len)) : + MK.Hash32Len13to24(s, len); + } + + if (len < 40) + { + uint a = len, b = unchecked(seed * c2), c = a + b; + a += Fetch32(s + len - 4); + b += Fetch32(s + len - 20); + c += Fetch32(s + len - 16); + uint d = a; + a = Rotate32(a, 21); + a = Mur(a, Mur(b, Sse42.Crc32(c, d))); // _mm_crc32_u32 + a += Fetch32(s + len - 12); + b += Fetch32(s + len - 8); + d += a; + a += d; + b = Mur(b, d) * c2; + a = Sse42.Crc32(a, b + c); // _mm_crc32_u32 + return MK.Hash32Len13to24(s, (len + 1) / 2, a) + b; + } + + // Murk + // Add(k, Mul5(Rol19(Xor(Mul(Rol17(Mul(★, cc1)), cc2),(h))))) + // Add(k, Mul5(Rot19(Xor(Mul(Rot17(Mul(d, cc1)), cc2), (h))))) + + + __m128i cc1 = Vector128.Create(c1); // _mm_set1_epi32 + __m128i cc2 = Vector128.Create(c2); // _mm_set1_epi32 + __m128i h = Vector128.Create(seed); // _mm_set1_epi32 + __m128i g = Vector128.Create(unchecked(c1 * seed)); // _mm_set1_epi32 + __m128i f = g; + __m128i k = Vector128.Create(0xe6546b64); // _mm_set1_epi32 + __m128i q; + if (len < 80) + { + __m128i a = Fetch128(s); + __m128i b = Fetch128(s + 16); + __m128i c = Fetch128(s + (len - 15) / 2); + __m128i d = Fetch128(s + len - 32); + __m128i e = Fetch128(s + len - 16); + h = Add(h, a); + g = Add(g, b); + q = g; + g = Shuffle0321(g); + f = Add(f, c); + __m128i be = Add(b, Mul(e, cc1)); + h = Add(h, f); + f = Add(f, h); + h = Add(Add(k, Mul5(Rol19(Xor(Mul(Rol17(Mul(d, cc1)), cc2), (h))))), e); + k = Xor(k, Ssse3.Shuffle(g.AsByte(), f.AsByte()).AsUInt32()); // _mm_shuffle_epi8 + g = Add(Xor(c, g), a); + f = Add(Xor(be, f), d); + k = Add(k, be); + k = Add(k, Ssse3.Shuffle(f.AsByte(), h.AsByte()).AsUInt32()); // _mm_shuffle_epi8 + f = Add(f, g); + g = Add(g, f); + g = Add( + Vector128.Create(len), // _mm_set1_epi32 + Mul(g, cc1) + ); + } + else + { + // len >= 80 + // The following is loosely modelled after farmhashmk::Hash32. + uint iters = (len - 1) / 80; + len -= iters * 80; + q = g; + while (iters-- != 0) + { + // + __m128i a = Fetch128(s); + __m128i b = Fetch128(s + 16); + __m128i c = Fetch128(s + 32); + __m128i d = Fetch128(s + 48); + __m128i e = Fetch128(s + 64); + h = Add(h, a); + g = Add(g, b); + g = Shuffle0321(g); + f = Add(f, c); + __m128i be = Add(b, Mul(e, cc1)); + h = Add(h, f); + f = Add(f, h); + h = Add(h, d); + q = Add(q, e); + h = Rol17(h); + h = Mul(h, cc1); + k = Xor(k, Ssse3.Shuffle(g.AsByte(), f.AsByte()).AsUInt32()); //_mm_shuffle_epi8 + g = Add(Xor(c, g), a); + f = Add(Xor(be, f), d); + Swap(ref f, ref q); + q = Aes.InverseMixColumns(q.AsByte()).AsUInt32(); // _mm_aesimc_si128 + k = Add(k, be); + k = Add(k, Ssse3.Shuffle(f.AsByte(), h.AsByte()).AsUInt32()); + f = Add(f, g); + g = Add(g, f); + f = Mul(f, cc1); + // + s += 80; + } + + if (len != 0) + { + h = Add(h, Vector128.Create(len)); // _mm_set1_epi32 + s = s + len - 80; + // + __m128i a = Fetch128(s); + __m128i b = Fetch128(s + 16); + __m128i c = Fetch128(s + 32); + __m128i d = Fetch128(s + 48); + __m128i e = Fetch128(s + 64); + h = Add(h, a); + g = Add(g, b); + g = Shuffle0321(g); + f = Add(f, c); + __m128i be = Add(b, Mul(e, cc1)); + h = Add(h, f); + f = Add(f, h); + h = Add(h, d); + q = Add(q, e); + h = Rol17(h); + h = Mul(h, cc1); + k = Xor(k, Ssse3.Shuffle(g.AsByte(), f.AsByte()).AsUInt32()); //_mm_shuffle_epi8 + g = Add(Xor(c, g), a); + f = Add(Xor(be, f), d); + Swap(ref f, ref q); + q = Aes.InverseMixColumns(q.AsByte()).AsUInt32(); // _mm_aesimc_si128 + k = Add(k, be); + k = Add(k, Ssse3.Shuffle(f.AsByte(), h.AsByte()).AsUInt32()); + f = Add(f, g); + g = Add(g, f); + f = Mul(f, cc1); + // + } + } + + g = Shuffle0321(g); + k = Xor(k, g); + k = Xor(k, q); + h = Xor(h, q); + f = Mul(f, cc1); + k = Mul(k, cc2); + g = Mul(g, cc1); + h = Mul(h, cc2); + k = Add(k, Ssse3.Shuffle(g.AsByte(), f.AsByte()).AsUInt32()); // _mm_shuffle_epi8 + h = Add(h, f); + f = Add(f, h); + g = Add(g, k); + k = Add(k, g); + k = Xor(k, Ssse3.Shuffle(f.AsByte(), h.AsByte()).AsUInt32()); // _mm_shuffle_epi8 + var buf = stackalloc __m128i[4]; + buf[0] = f; + buf[1] = g; + buf[2] = k; + buf[3] = h; + s = (byte*)buf; + uint x = Fetch32(s); + uint y = Fetch32(s + 4); + uint z = Fetch32(s + 8); + x = Sse42.Crc32(x, Fetch32(s + 12)); // _mm_crc32_u32 + y = Sse42.Crc32(y, Fetch32(s + 16)); // _mm_crc32_u32 + z = Sse42.Crc32(z * c1, Fetch32(s + 20)); // _mm_crc32_u32 + x = Sse42.Crc32(x, Fetch32(s + 24)); // _mm_crc32_u32 + y = Sse42.Crc32(y * c1, Fetch32(s + 28)); // _mm_crc32_u32 + uint o = y; + z = Sse42.Crc32(z, Fetch32(s + 32)); // _mm_crc32_u32 + x = Sse42.Crc32(x * c1, Fetch32(s + 36)); // _mm_crc32_u32 + y = Sse42.Crc32(y, Fetch32(s + 40)); // _mm_crc32_u32 + z = Sse42.Crc32(z * c1, Fetch32(s + 44)); // _mm_crc32_u32 + x = Sse42.Crc32(x, Fetch32(s + 48)); // _mm_crc32_u32 + y = Sse42.Crc32(y * c1, Fetch32(s + 52)); // _mm_crc32_u32 + z = Sse42.Crc32(z, Fetch32(s + 56)); // _mm_crc32_u32 + x = Sse42.Crc32(x, Fetch32(s + 60)); // _mm_crc32_u32 + return (o - x + y - z) * c1; + } + + public static unsafe uint Hash32WithSeed(byte* s, uint len, uint seed, uint h) + => Sse42.Crc32(Hash32(s + 24, len - 24) + seed, h); // _mm_crc32_u32 + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FarmHash.TE.cs b/src/MsgPack.Abstraction/Internal/FarmHash.TE.cs new file mode 100644 index 000000000..942fbf305 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FarmHash.TE.cs @@ -0,0 +1,258 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// This file is ported from Google's FarmHash (https://github.com/google/farmhash/) +// Original copyright notice is following: + +// Copyright (c) 2014 Google, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// FarmHash, by Geoff Pike + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +using __m128i = System.Runtime.Intrinsics.Vector128; + +namespace MsgPack.Internal +{ + internal partial class FarmHash + { + private static class TE + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe __m128i Fetch128(byte* s) + => Sse2.LoadVector128(s).AsUInt64(); // _mm_loadu_si128 + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static __m128i Add(__m128i x, __m128i y) + => Sse2.Add(x, y); // _mm_add_epi64 + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static __m128i Xor(__m128i x, __m128i y) + => Sse2.Xor(x, y); // _mm_xor_si128 + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static __m128i Mul(__m128i x, __m128i y) + => Sse41.MultiplyLow(x.AsUInt32(), y.AsUInt32()).AsUInt64(); // _mm_mullo_epi32 + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static __m128i Shuf(__m128i x, __m128i y) + => Ssse3.Shuffle(x.AsByte(), y.AsByte()).AsUInt64(); // _mm_shuffle_epi8 + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void swap(ref __m128i left, ref __m128i right) + { + var temp = left; + left = right; + right = temp; + } + + // Requires n >= 256. Requires SSE4.1. Should be slightly faster if the + // compiler uses AVX instructions (e.g., use the -mavx flag with GCC). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe ulong Hash64Long(byte* s, uint n, ulong seed0, ulong seed1) + { + __m128i kShuf = + Vector128.Create(4, 11, 10, 5, 8, 15, 6, 9, 12, 2, 14, 13, 0, 7, 3, 1).AsUInt64(); // _mm_set_epi8 + __m128i kMult = + Vector128.Create(0xbd, 0xd6, 0x33, 0x39, 0x45, 0x54, 0xfa, 0x03, + 0x34, 0x3e, 0x33, 0xed, 0xcc, 0x9e, 0x2d, 0x51).AsUInt64(); // _mm_set_epi8 + ulong seed2 = (seed0 + 113) * (seed1 + 9); + ulong seed3 = (Rotate64(seed0, 23) + 27) * (Rotate64(seed1, 30) + 111); + __m128i d0 = Sse2.X64.ConvertScalarToVector128UInt64(seed0); // _mm_cvtsi64_si128 + __m128i d1 = Sse2.X64.ConvertScalarToVector128UInt64(seed1); // _mm_cvtsi64_si128 + __m128i d2 = Shuf(kShuf, d0); + __m128i d3 = Shuf(kShuf, d1); + __m128i d4 = Xor(d0, d1); + __m128i d5 = Xor(d1, d2); + __m128i d6 = Xor(d2, d4); + __m128i d7 = Vector128.Create(seed2 >> 32); // _mm_set1_epi32 + __m128i d8 = Mul(kMult, d2); + __m128i d9 = Vector128.Create(seed3 >> 32); // _mm_set1_epi32 + __m128i d10 = Vector128.Create(seed3); // _mm_set1_epi32 + __m128i d11 = Add(d2, Vector128.Create(seed2)); // _mm_set1_epi32 + // size_t == ulong here because this code should be run under X64 process. + byte* end = s + (n & ~(/* size_t */ ulong)(255)); + do + { + __m128i z; + z = Fetch128(s); + d0 = Add(d0, z); + d1 = Shuf(kShuf, d1); + d2 = Xor(d2, d0); + d4 = Xor(d4, z); + d4 = Xor(d4, d1); + swap(ref d0, ref d6); + z = Fetch128(s + 16); + d5 = Add(d5, z); + d6 = Shuf(kShuf, d6); + d8 = Shuf(kShuf, d8); + d7 = Xor(d7, d5); + d0 = Xor(d0, z); + d0 = Xor(d0, d6); + swap(ref d5, ref d11); + z = Fetch128(s + 32); + d1 = Add(d1, z); + d2 = Shuf(kShuf, d2); + d4 = Shuf(kShuf, d4); + d5 = Xor(d5, z); + d5 = Xor(d5, d2); + swap(ref d10, ref d4); + z = Fetch128(s + 48); + d6 = Add(d6, z); + d7 = Shuf(kShuf, d7); + d0 = Shuf(kShuf, d0); + d8 = Xor(d8, d6); + d1 = Xor(d1, z); + d1 = Add(d1, d7); + z = Fetch128(s + 64); + d2 = Add(d2, z); + d5 = Shuf(kShuf, d5); + d4 = Add(d4, d2); + d6 = Xor(d6, z); + d6 = Xor(d6, d11); + swap(ref d8, ref d2); + z = Fetch128(s + 80); + d7 = Xor(d7, z); + d8 = Shuf(kShuf, d8); + d1 = Shuf(kShuf, d1); + d0 = Add(d0, d7); + d2 = Add(d2, z); + d2 = Add(d2, d8); + swap(ref d1, ref d7); + z = Fetch128(s + 96); + d4 = Shuf(kShuf, d4); + d6 = Shuf(kShuf, d6); + d8 = Mul(kMult, d8); + d5 = Xor(d5, d11); + d7 = Xor(d7, z); + d7 = Add(d7, d4); + swap(ref d6, ref d0); + z = Fetch128(s + 112); + d8 = Add(d8, z); + d0 = Shuf(kShuf, d0); + d2 = Shuf(kShuf, d2); + d1 = Xor(d1, d8); + d10 = Xor(d10, z); + d10 = Xor(d10, d0); + swap(ref d11, ref d5); + z = Fetch128(s + 128); + d4 = Add(d4, z); + d5 = Shuf(kShuf, d5); + d7 = Shuf(kShuf, d7); + d6 = Add(d6, d4); + d8 = Xor(d8, z); + d8 = Xor(d8, d5); + swap(ref d4, ref d10); + z = Fetch128(s + 144); + d0 = Add(d0, z); + d1 = Shuf(kShuf, d1); + d2 = Add(d2, d0); + d4 = Xor(d4, z); + d4 = Xor(d4, d1); + z = Fetch128(s + 160); + d5 = Add(d5, z); + d6 = Shuf(kShuf, d6); + d8 = Shuf(kShuf, d8); + d7 = Xor(d7, d5); + d0 = Xor(d0, z); + d0 = Xor(d0, d6); + swap(ref d2, ref d8); + z = Fetch128(s + 176); + d1 = Add(d1, z); + d2 = Shuf(kShuf, d2); + d4 = Shuf(kShuf, d4); + d5 = Mul(kMult, d5); + d5 = Xor(d5, z); + d5 = Xor(d5, d2); + swap(ref d7, ref d1); + z = Fetch128(s + 192); + d6 = Add(d6, z); + d7 = Shuf(kShuf, d7); + d0 = Shuf(kShuf, d0); + d8 = Add(d8, d6); + d1 = Xor(d1, z); + d1 = Xor(d1, d7); + swap(ref d0, ref d6); + z = Fetch128(s + 208); + d2 = Add(d2, z); + d5 = Shuf(kShuf, d5); + d4 = Xor(d4, d2); + d6 = Xor(d6, z); + d6 = Xor(d6, d9); + swap(ref d5, ref d11); + z = Fetch128(s + 224); + d7 = Add(d7, z); + d8 = Shuf(kShuf, d8); + d1 = Shuf(kShuf, d1); + d0 = Xor(d0, d7); + d2 = Xor(d2, z); + d2 = Xor(d2, d8); + swap(ref d10, ref d4); + z = Fetch128(s + 240); + d3 = Add(d3, z); + d4 = Shuf(kShuf, d4); + d6 = Shuf(kShuf, d6); + d7 = Mul(kMult, d7); + d5 = Add(d5, d3); + d7 = Xor(d7, z); + d7 = Xor(d7, d4); + swap(ref d3, ref d9); + s += 256; + } while (s != end); + d6 = Add(Mul(kMult, d6), Sse2.X64.ConvertScalarToVector128UInt64(n)); // _mm_cvtsi64_si128 + if (n % 256 != 0) + { + d7 = Add(Sse2.Shuffle(d8.AsUInt32(), (0 << 6) + (3 << 4) + (2 << 2) + (1 << 0)).AsUInt64(), d7); // _mm_shuffle_epi32 + d8 = Add(Mul(kMult, d8), Sse2.X64.ConvertScalarToVector128UInt64(XO.Hash64(s, n % 256))); // _mm_cvtsi64_si128 + } + var t = stackalloc __m128i[8]; + d0 = Mul(kMult, Shuf(kShuf, Mul(kMult, d0))); + d3 = Mul(kMult, Shuf(kShuf, Mul(kMult, d3))); + d9 = Mul(kMult, Shuf(kShuf, Mul(kMult, d9))); + d1 = Mul(kMult, Shuf(kShuf, Mul(kMult, d1))); + d0 = Add(d11, d0); + d3 = Xor(d7, d3); + d9 = Add(d8, d9); + d1 = Add(d10, d1); + d4 = Add(d3, d4); + d5 = Add(d9, d5); + d6 = Xor(d1, d6); + d2 = Add(d0, d2); + t[0] = d0; + t[1] = d3; + t[2] = d9; + t[3] = d1; + t[4] = d4; + t[5] = d5; + t[6] = d6; + t[7] = d2; + return XO.Hash64((byte*)t, /* sizeof(t) */ 8); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe ulong Hash64WithSeed(byte* s, uint len, uint seed) + => len >= 512 ? + Hash64Long(s, len, k1, seed) : + XO.Hash64WithSeed(s, len, seed); + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FarmHash.UO.cs b/src/MsgPack.Abstraction/Internal/FarmHash.UO.cs new file mode 100644 index 000000000..ae087da0a --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FarmHash.UO.cs @@ -0,0 +1,155 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// This file is ported from Google's FarmHash (https://github.com/google/farmhash/) +// Original copyright notice is following: + +// Copyright (c) 2014 Google, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// FarmHash, by Geoff Pike + +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + internal partial class FarmHash + { + private static class UO + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe ulong H(ulong x, ulong y, ulong mul, int r) + { + ulong a = (x ^ y) * mul; + a ^= (a >> 47); + ulong b = (y ^ a) * mul; + return Rotate64(b, r) * mul; + } + + private static unsafe ulong Hash64WithSeeds(byte* s, uint len, + ulong seed0, ulong seed1) + { + if (len <= 64) + { + return NA.Hash64WithSeeds(s, len, seed0, seed1); + } + + // For strings over 64 bytes we loop. Internal state consists of + // 64 bytes: u, v, w, x, y, and z. + ulong x = seed0; + ulong y = seed1 * k2 + 113; + ulong z = NA.ShiftMix(y * k2) * k2; + (ulong first, ulong second) v = (seed0, seed1); + (ulong first, ulong second) w = (0, 0); + ulong u = x - z; + x *= k2; + ulong mul = k2 + (u & 0x82); + + // Set end so that after the loop we have 1 to 64 bytes left to process. + byte* end = s + ((len - 1) / 64) * 64; + byte* last64 = end + ((len - 1) & 63) - 63; + Debug.Assert(s + len - 64 == last64); + do + { + ulong a0 = Fetch64(s); + ulong a1 = Fetch64(s + 8); + ulong a2 = Fetch64(s + 16); + ulong a3 = Fetch64(s + 24); + ulong a4 = Fetch64(s + 32); + ulong a5 = Fetch64(s + 40); + ulong a6 = Fetch64(s + 48); + ulong a7 = Fetch64(s + 56); + x += a0 + a1; + y += a2; + z += a3; + v.first += a4; + v.second += a5 + a1; + w.first += a6; + w.second += a7; + + x = Rotate64(x, 26); + x *= 9; + y = Rotate64(y, 29); + z *= mul; + v.first = Rotate64(v.first, 33); + v.second = Rotate64(v.second, 30); + w.first ^= x; + w.first *= 9; + z = Rotate64(z, 32); + z += w.second; + w.second += z; + z *= 9; + Swap(ref u, ref y); + + z += a0 + a6; + v.first += a2; + v.second += a3; + w.first += a4; + w.second += a5 + a6; + x += a1; + y += a7; + + y += v.first; + v.first += x - y; + v.second += w.first; + w.first += v.second; + w.second += x - y; + x += w.second; + w.second = Rotate64(w.second, 34); + Swap(ref u, ref z); + s += 64; + } while (s != end); + // Make s point to the last 64 bytes of input. + s = last64; + u *= 9; + v.second = Rotate64(v.second, 28); + v.first = Rotate64(v.first, 20); + w.first += ((len - 1) & 63); + u += y; + y += u; + x = Rotate64(y - x + v.first + Fetch64(s + 8), 37) * mul; + y = Rotate64(y ^ v.second ^ Fetch64(s + 48), 42) * mul; + x ^= w.second * 9; + y += v.first + Fetch64(s + 40); + z = Rotate64(z + w.first, 33) * mul; + v = NA.WeakHashLen32WithSeeds(s, v.second * mul, x + w.first); + w = NA.WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16)); + return H(NA.HashLen16(v.first + x, w.first ^ y, mul) + z - u, + H(v.second + y, w.second + z, k2, 30) ^ x, + k2, + 31); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe ulong Hash64WithSeed(byte* s, uint len, ulong seed) + => len <= 64 ? + NA.Hash64WithSeed(s, len, seed) : + Hash64WithSeeds(s, len, 0, seed); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe ulong Hash64(byte* s, uint len) + => len <= 64 ? + NA.Hash64(s, len) : + Hash64WithSeeds(s, len, 81, 0); + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FarmHash.XO.cs b/src/MsgPack.Abstraction/Internal/FarmHash.XO.cs new file mode 100644 index 000000000..4b29fa36b --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FarmHash.XO.cs @@ -0,0 +1,112 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// This file is ported from Google's FarmHash (https://github.com/google/farmhash/) +// Original copyright notice is following: + +// Copyright (c) 2014 Google, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// FarmHash, by Geoff Pike + +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + internal partial class FarmHash + { + private static class XO + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe ulong H32(byte* s, uint len, ulong mul, + ulong seed0 = 0, ulong seed1 = 0) + { + ulong a = Fetch64(s) * k1; + ulong b = Fetch64(s + 8); + ulong c = Fetch64(s + len - 8) * mul; + ulong d = Fetch64(s + len - 16) * k2; + ulong u = Rotate64(a + b, 43) + Rotate64(c, 30) + d + seed0; + ulong v = a + Rotate64(b + k2, 18) + c + seed1; + a = NA.ShiftMix((u ^ v) * mul); + b = NA.ShiftMix((v ^ a) * mul); + return b; + } + + // Return an 8-byte hash for 33 to 64 bytes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe ulong HashLen33to64(byte* s, uint len) + { + ulong mul0 = k2 - 30; + ulong mul1 = k2 - 30 + 2 * len; + ulong h0 = H32(s, 32, mul0); + ulong h1 = H32(s + len - 32, 32, mul1); + return ((h1 * mul1) + h0) * mul1; + } + + // Return an 8-byte hash for 65 to 96 bytes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe ulong HashLen65to96(byte* s, uint len) + { + ulong mul0 = k2 - 114; + ulong mul1 = k2 - 114 + 2 * len; + ulong h0 = H32(s, 32, mul0); + ulong h1 = H32(s + 32, 32, mul1); + ulong h2 = H32(s + len - 32, 32, mul1, h0, h1); + return (h2 * 9 + (h0 >> 17) + (h1 >> 21)) * mul1; + } + + internal static unsafe ulong Hash64(byte* s, uint len) + { + if (len <= 32) + { + if (len <= 16) + { + return NA.HashLen0to16(s, len); + } + else + { + return NA.HashLen17to32(s, len); + } + } + else if (len <= 64) + { + return HashLen33to64(s, len); + } + else if (len <= 96) + { + return HashLen65to96(s, len); + } + else if (len <= 256) + { + return NA.Hash64(s, len); + } + else + { + return UO.Hash64(s, len); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe ulong Hash64WithSeed(byte* s, uint len, ulong seed) + => UO.Hash64WithSeed(s, len, seed); + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FarmHash.cs b/src/MsgPack.Abstraction/Internal/FarmHash.cs new file mode 100644 index 000000000..63865d04b --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FarmHash.cs @@ -0,0 +1,147 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// This file is ported from Google's FarmHash (https://github.com/google/farmhash/) +// Original copyright notice is following: + +// Copyright (c) 2014 Google, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// FarmHash, by Geoff Pike + +using System; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +using RandomNumberGenerator = System.Security.Cryptography.RandomNumberGenerator; + +namespace MsgPack.Internal +{ + internal static partial class FarmHash + { + private const uint c1 = 0xcc9e2d51; + private const uint c2 = 0x1b873593; + private const ulong k0 = 0xc3a5c85c97cb3127UL; + private const ulong k1 = 0xb492b66fbe98f273UL; + private const ulong k2 = 0x9ae16a3b2f90404fUL; + + /// + /// Hash function for a byte array. + /// May change from time to time, may differ on different platforms. + /// + /// Target byte span. + /// Non-cryptgraphic, temporal hash value suitable as hash code. + public static unsafe int Hash32WithSeed(ReadOnlySpan bytes, uint seed) + { + var len = (uint)bytes.Length; + unchecked + { + fixed (byte* s = bytes) + { + if (Sse41.IsSupported && RuntimeInformation.ProcessArchitecture == Architecture.X64) + { + return (int)NT.Hash32WithSeed(s, len, seed); + } + + if (Sse42.IsSupported) + { + // farmhashsa::Hash32WithSeed and farmshahsu::Hash32WithSeed is same for less than or equal to 24bytes, + // so inline here to reduce code size. + if (len <= 24) + { + if (len >= 13) + { + return (int)MK.Hash32Len13to24(s, len, seed * c1); + } + else if (len >= 5) + { + return (int)MK.Hash32Len5to12(s, len, seed); + } + else + { + return (int)MK.Hash32Len0to4(s, len, seed); + } + } + + // Calculation of h is also common. + uint h = MK.Hash32Len13to24(s, 24, seed ^ len); + + if (Aes.IsSupported) + { + return (int)SU.Hash32WithSeed(s, (uint)bytes.Length, seed, h); + } + else + { + return (int)SA.Hash32WithSeed(s, (uint)bytes.Length, seed, h); + } + } + + return (int)MK.Hash32WithSeed(s, (uint)bytes.Length, seed); + } + } + } + + public static readonly uint DefaultSeed = InitializeRandomSeed(); + + private static uint InitializeRandomSeed() + { + using var rng = RandomNumberGenerator.Create(); + Span bytes = stackalloc byte[sizeof(uint)]; + rng.GetBytes(bytes); + return BinaryPrimitives.ReadUInt32LittleEndian(bytes); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private static unsafe uint Fetch32(byte* p) + => *(uint*)p; + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private static uint Rotate32(uint val, int shift) + // Avoid shifting by 32: doing so yields an undefined result. + => shift == 0 ? val : ((val >> shift) | (val << (32 - shift))); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private static uint Fmix(uint h) + { + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + return h; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private static uint Mur(uint a, uint h) + { + // Helper from Murmur3 for combining two 32-bit values. + a *= c1; + a = Rotate32(a, 17); + a *= c2; + h ^= a; + h = Rotate32(h, 19); + return h * 5 + 0xe6546b64; + } + } +} + diff --git a/src/MsgPack.Abstraction/Internal/FormatDecoder.Primitives.cs b/src/MsgPack.Abstraction/Internal/FormatDecoder.Primitives.cs new file mode 100644 index 000000000..9e7144635 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FormatDecoder.Primitives.cs @@ -0,0 +1,876 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + partial class FormatDecoder + { + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Byte DecodeByte(ref SequenceReader source) + { + var result = this.DecodeByte(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Byte), requestHint); + } + + return result; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract Byte DecodeByte(ref SequenceReader source, out int requestHint); + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Byte? DecodeNullableByte(ref SequenceReader source) + { + var result = this.DecodeNullableByte(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Byte), requestHint); + } + + return result; + } + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract Byte? DecodeNullableByte(ref SequenceReader source, out int requestHint); + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Int16 DecodeInt16(ref SequenceReader source) + { + var result = this.DecodeInt16(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Int16), requestHint); + } + + return result; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract Int16 DecodeInt16(ref SequenceReader source, out int requestHint); + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Int16? DecodeNullableInt16(ref SequenceReader source) + { + var result = this.DecodeNullableInt16(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Int16), requestHint); + } + + return result; + } + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract Int16? DecodeNullableInt16(ref SequenceReader source, out int requestHint); + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Int32 DecodeInt32(ref SequenceReader source) + { + var result = this.DecodeInt32(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Int32), requestHint); + } + + return result; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract Int32 DecodeInt32(ref SequenceReader source, out int requestHint); + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Int32? DecodeNullableInt32(ref SequenceReader source) + { + var result = this.DecodeNullableInt32(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Int32), requestHint); + } + + return result; + } + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract Int32? DecodeNullableInt32(ref SequenceReader source, out int requestHint); + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Int64 DecodeInt64(ref SequenceReader source) + { + var result = this.DecodeInt64(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Int64), requestHint); + } + + return result; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract Int64 DecodeInt64(ref SequenceReader source, out int requestHint); + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Int64? DecodeNullableInt64(ref SequenceReader source) + { + var result = this.DecodeNullableInt64(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Int64), requestHint); + } + + return result; + } + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract Int64? DecodeNullableInt64(ref SequenceReader source, out int requestHint); + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public SByte DecodeSByte(ref SequenceReader source) + { + var result = this.DecodeSByte(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(SByte), requestHint); + } + + return result; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract SByte DecodeSByte(ref SequenceReader source, out int requestHint); + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public SByte? DecodeNullableSByte(ref SequenceReader source) + { + var result = this.DecodeNullableSByte(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(SByte), requestHint); + } + + return result; + } + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract SByte? DecodeNullableSByte(ref SequenceReader source, out int requestHint); + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public UInt16 DecodeUInt16(ref SequenceReader source) + { + var result = this.DecodeUInt16(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(UInt16), requestHint); + } + + return result; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract UInt16 DecodeUInt16(ref SequenceReader source, out int requestHint); + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public UInt16? DecodeNullableUInt16(ref SequenceReader source) + { + var result = this.DecodeNullableUInt16(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(UInt16), requestHint); + } + + return result; + } + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract UInt16? DecodeNullableUInt16(ref SequenceReader source, out int requestHint); + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public UInt32 DecodeUInt32(ref SequenceReader source) + { + var result = this.DecodeUInt32(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(UInt32), requestHint); + } + + return result; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract UInt32 DecodeUInt32(ref SequenceReader source, out int requestHint); + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public UInt32? DecodeNullableUInt32(ref SequenceReader source) + { + var result = this.DecodeNullableUInt32(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(UInt32), requestHint); + } + + return result; + } + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract UInt32? DecodeNullableUInt32(ref SequenceReader source, out int requestHint); + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public UInt64 DecodeUInt64(ref SequenceReader source) + { + var result = this.DecodeUInt64(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(UInt64), requestHint); + } + + return result; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract UInt64 DecodeUInt64(ref SequenceReader source, out int requestHint); + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public UInt64? DecodeNullableUInt64(ref SequenceReader source) + { + var result = this.DecodeNullableUInt64(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(UInt64), requestHint); + } + + return result; + } + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract UInt64? DecodeNullableUInt64(ref SequenceReader source, out int requestHint); + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Single DecodeSingle(ref SequenceReader source) + { + var result = this.DecodeSingle(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Single), requestHint); + } + + return result; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract Single DecodeSingle(ref SequenceReader source, out int requestHint); + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Single? DecodeNullableSingle(ref SequenceReader source) + { + var result = this.DecodeNullableSingle(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Single), requestHint); + } + + return result; + } + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract Single? DecodeNullableSingle(ref SequenceReader source, out int requestHint); + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Double DecodeDouble(ref SequenceReader source) + { + var result = this.DecodeDouble(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Double), requestHint); + } + + return result; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract Double DecodeDouble(ref SequenceReader source, out int requestHint); + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Double? DecodeNullableDouble(ref SequenceReader source) + { + var result = this.DecodeNullableDouble(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Double), requestHint); + } + + return result; + } + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract Double? DecodeNullableDouble(ref SequenceReader source, out int requestHint); + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Boolean DecodeBoolean(ref SequenceReader source) + { + var result = this.DecodeBoolean(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Boolean), requestHint); + } + + return result; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract Boolean DecodeBoolean(ref SequenceReader source, out int requestHint); + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Boolean? DecodeNullableBoolean(ref SequenceReader source) + { + var result = this.DecodeNullableBoolean(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(Boolean), requestHint); + } + + return result; + } + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract Boolean? DecodeNullableBoolean(ref SequenceReader source, out int requestHint); + + } +} diff --git a/src/MsgPack.Abstraction/Internal/FormatDecoder.Primitives.tt b/src/MsgPack.Abstraction/Internal/FormatDecoder.Primitives.tt new file mode 100644 index 000000000..f31bf57a4 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FormatDecoder.Primitives.tt @@ -0,0 +1,117 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ output extension=".cs" #> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + partial class FormatDecoder + { +<# +foreach (var outputType in new [] { + "Byte", + "Int16", + "Int32", + "Int64", + "SByte", + "UInt16", + "UInt32", + "UInt64", + "Single", + "Double", + "Boolean" +}) +{ +#> + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public <#= outputType #> Decode<#= outputType #>(ref SequenceReader source) + { + var result = this.Decode<#= outputType #>(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(<#= outputType #>), requestHint); + } + + return result; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract <#= outputType #> Decode<#= outputType #>(ref SequenceReader source, out int requestHint); + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public <#= outputType #>? DecodeNullable<#= outputType #>(ref SequenceReader source) + { + var result = this.DecodeNullable<#= outputType #>(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(<#= outputType #>), requestHint); + } + + return result; + } + + /// + /// Encodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract <#= outputType #>? DecodeNullable<#= outputType #>(ref SequenceReader source, out int requestHint); + +<# +} +#> + } +} diff --git a/src/MsgPack.Abstraction/Internal/FormatDecoder.Strings.cs b/src/MsgPack.Abstraction/Internal/FormatDecoder.Strings.cs new file mode 100644 index 000000000..87f7ad1e5 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FormatDecoder.Strings.cs @@ -0,0 +1,548 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +#nullable enable + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; + +namespace MsgPack.Internal +{ + partial class FormatDecoder + { + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// to cancel long running operation. This value can be omitted. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public String DecodeString(ref SequenceReader source, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + var result = this.DecodeString(ref source, out var requestHint, encoding, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInputForString(source.Consumed, typeof(String), encoding, requestHint); + } + + return result!; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// to cancel long running operation. This value can be omitted. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract String? DecodeString(ref SequenceReader source, out int requestHint, Encoding? encoding = null, CancellationToken cancellationToken = default); + + /// + /// Decodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public String? DecodeNullableString(ref SequenceReader source, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + var result = this.DecodeNullableString(ref source, out var requestHint, encoding, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInputForString(source.Consumed, typeof(String), encoding, requestHint); + } + + return result; + } + + /// + /// Decodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract String? DecodeNullableString(ref SequenceReader source, out int requestHint, Encoding? encoding = null, CancellationToken cancellationToken = default); + + /// + /// Decodes the value from specified sequence to of . + /// + /// SequenceReader<byte>. + /// of which will store the decoded binary. + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// to cancel long running operation. This value can be omitted. + /// + /// The length of the decoded value. + /// This value can be 0 for empty string. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public int DecodeString(ref SequenceReader source, Span buffer, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + var result = this.DecodeString(ref source, buffer, out var requestHint, encoding, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInputForString(source.Consumed, typeof(String), encoding, requestHint); + } + + return result!; + } + + /// + /// Decodes the value from specified sequence to of . + /// + /// SequenceReader<byte>. + /// of which will store the decoded binary. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// to cancel long running operation. This value can be omitted. + /// + /// The length of the decoded value. + /// This value can be 0 for empty string. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract int DecodeString(ref SequenceReader source, Span buffer, out int requestHint, Encoding? encoding = null, CancellationToken cancellationToken = default); + + /// + /// Decodes the value or null from specified sequence to of . + /// + /// SequenceReader<byte>. + /// of which will store the decoded binary. + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// + /// The length of the decoded value; + /// null when the underlying value is null. + /// This value can be 0 for empty string. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public int? DecodeNullableString(ref SequenceReader source, Span buffer, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + var result = this.DecodeNullableString(ref source, buffer, out var requestHint, encoding, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInputForString(source.Consumed, typeof(String), encoding, requestHint); + } + + return result; + } + + /// + /// Decodes the value or null from specified sequence to of . + /// + /// SequenceReader<byte>. + /// of which will store the decoded binary. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// + /// The length of the decoded value; + /// null when the underlying value is null. + /// This value can be 0 for empty string. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract int? DecodeNullableString(ref SequenceReader source, Span buffer, out int requestHint, Encoding? encoding = null, CancellationToken cancellationToken = default); + + +#if FEATURE_UTF8STRING + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// to cancel long running operation. This value can be omitted. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Utf8String DecodeUtf8String(ref SequenceReader source, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + var result = this.DecodeUtf8String(ref source, out var requestHint, encoding, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInputForUtf8String(source.Consumed, typeof(Utf8String), encoding, requestHint); + } + + return result!; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// to cancel long running operation. This value can be omitted. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract Utf8String? DecodeUtf8String(ref SequenceReader source, out int requestHint, Encoding? encoding = null, CancellationToken cancellationToken = default); + + /// + /// Decodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Utf8String? DecodeNullableUtf8String(ref SequenceReader source, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + var result = this.DecodeNullableUtf8String(ref source, out var requestHint, encoding, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInputForUtf8String(source.Consumed, typeof(Utf8String), encoding, requestHint); + } + + return result; + } + + /// + /// Decodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract Utf8String? DecodeNullableUtf8String(ref SequenceReader source, out int requestHint, Encoding? encoding = null, CancellationToken cancellationToken = default); + + /// + /// Decodes the value from specified sequence to . + /// + /// SequenceReader<byte>. + /// which will store the decoded binary. + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// to cancel long running operation. This value can be omitted. + /// + /// The length of the decoded value. + /// This value can be 0 for empty string. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public int DecodeUtf8String(ref SequenceReader source, Utf8Span buffer, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + var result = this.DecodeUtf8String(ref source, buffer, out var requestHint, encoding, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInputForUtf8String(source.Consumed, typeof(Utf8String), encoding, requestHint); + } + + return result!; + } + + /// + /// Decodes the value from specified sequence to . + /// + /// SequenceReader<byte>. + /// which will store the decoded binary. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// to cancel long running operation. This value can be omitted. + /// + /// The length of the decoded value. + /// This value can be 0 for empty string. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract int DecodeUtf8String(ref SequenceReader source, Utf8Span buffer, out int requestHint, Encoding? encoding = null, CancellationToken cancellationToken = default); + + /// + /// Decodes the value or null from specified sequence to . + /// + /// SequenceReader<byte>. + /// which will store the decoded binary. + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// + /// The length of the decoded value; + /// null when the underlying value is null. + /// This value can be 0 for empty string. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public int? DecodeNullableUtf8String(ref SequenceReader source, Utf8Span buffer, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + var result = this.DecodeNullableUtf8String(ref source, buffer, out var requestHint, encoding, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInputForUtf8String(source.Consumed, typeof(Utf8String), encoding, requestHint); + } + + return result; + } + + /// + /// Decodes the value or null from specified sequence to . + /// + /// SequenceReader<byte>. + /// which will store the decoded binary. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM. + /// + /// The length of the decoded value; + /// null when the underlying value is null. + /// This value can be 0 for empty string. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract int? DecodeNullableUtf8String(ref SequenceReader source, Utf8Span buffer, out int requestHint, Encoding? encoding = null, CancellationToken cancellationToken = default); + + +#endif // FEATURE_UTF8STRING + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// to cancel long running operation. This value can be omitted. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public byte[] DecodeBinary(ref SequenceReader source, CancellationToken cancellationToken = default) + { + var result = this.DecodeBinary(ref source, out var requestHint, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(byte[]), requestHint); + } + + return result!; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// to cancel long running operation. This value can be omitted. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract byte[]? DecodeBinary(ref SequenceReader source, out int requestHint, CancellationToken cancellationToken = default); + + /// + /// Decodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public byte[]? DecodeNullableBinary(ref SequenceReader source, CancellationToken cancellationToken = default) + { + var result = this.DecodeNullableBinary(ref source, out var requestHint, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(byte[]), requestHint); + } + + return result; + } + + /// + /// Decodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract byte[]? DecodeNullableBinary(ref SequenceReader source, out int requestHint, CancellationToken cancellationToken = default); + + /// + /// Decodes the value from specified sequence to of . + /// + /// SequenceReader<byte>. + /// of which will store the decoded binary. + /// to cancel long running operation. This value can be omitted. + /// + /// The length of the decoded value. + /// This value can be 0 for empty string. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public int DecodeBinary(ref SequenceReader source, Span buffer, CancellationToken cancellationToken = default) + { + var result = this.DecodeBinary(ref source, buffer, out var requestHint, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(byte[]), requestHint); + } + + return result!; + } + + /// + /// Decodes the value from specified sequence to of . + /// + /// SequenceReader<byte>. + /// of which will store the decoded binary. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// to cancel long running operation. This value can be omitted. + /// + /// The length of the decoded value. + /// This value can be 0 for empty string. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract int DecodeBinary(ref SequenceReader source, Span buffer, out int requestHint, CancellationToken cancellationToken = default); + + /// + /// Decodes the value or null from specified sequence to of . + /// + /// SequenceReader<byte>. + /// of which will store the decoded binary. + /// + /// The length of the decoded value; + /// null when the underlying value is null. + /// This value can be 0 for empty string. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public int? DecodeNullableBinary(ref SequenceReader source, Span buffer, CancellationToken cancellationToken = default) + { + var result = this.DecodeNullableBinary(ref source, buffer, out var requestHint, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInput(source.Consumed, typeof(byte[]), requestHint); + } + + return result; + } + + /// + /// Decodes the value or null from specified sequence to of . + /// + /// SequenceReader<byte>. + /// of which will store the decoded binary. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// The length of the decoded value; + /// null when the underlying value is null. + /// This value can be 0 for empty string. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract int? DecodeNullableBinary(ref SequenceReader source, Span buffer, out int requestHint, CancellationToken cancellationToken = default); + + } +} diff --git a/src/MsgPack.Abstraction/Internal/FormatDecoder.Strings.tt b/src/MsgPack.Abstraction/Internal/FormatDecoder.Strings.tt new file mode 100644 index 000000000..b949abac6 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FormatDecoder.Strings.tt @@ -0,0 +1,301 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ output extension=".cs" #> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +#nullable enable + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; + +namespace MsgPack.Internal +{ + partial class FormatDecoder + { +<# +foreach (var spec in new [] { + new + { + Name = "String", Type = "String", SpanType = "Span", ThrowSuffix = "ForString", SpanTypeDoc = " of ", + ExtraParameters = "Encoding? encoding = null, ", + ExtraParameterDocs = new [] { "Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM." }, + ExtraArguments = "encoding, ", IfDef = String.Empty + }, + new + { + Name = "Utf8String", Type = "Utf8String", SpanType = "Utf8Span", ThrowSuffix = "ForUtf8String", SpanTypeDoc = "", + ExtraParameters = "Encoding? encoding = null, ", + ExtraParameterDocs = new [] { "Specify charactor encoding. This value can be omitted, and default is UTF-8 without BOM." }, + ExtraArguments = "encoding, ", IfDef = "FEATURE_UTF8STRING" + }, + new + { + Name = "Binary", Type = "byte[]", SpanType = "Span", ThrowSuffix = String.Empty, SpanTypeDoc = " of ", + ExtraParameters = String.Empty, + ExtraParameterDocs = Array.Empty(), + ExtraArguments = String.Empty, IfDef = String.Empty + }, +}) +{ + if (!String.IsNullOrEmpty(spec.IfDef)) + { +#> + +#if <#= spec.IfDef #> + +<# + } +#> + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. +<# + foreach (var doc in spec.ExtraParameterDocs) + { +#> + /// <#= doc #> +<# + } +#> + /// to cancel long running operation. This value can be omitted. + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public <#= spec.Type #> Decode<#= spec.Name #>(ref SequenceReader source, <#= spec.ExtraParameters #>CancellationToken cancellationToken = default) + { + var result = this.Decode<#= spec.Name #>(ref source, out var requestHint, <#= spec.ExtraArguments #>cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInput<#= spec.ThrowSuffix #>(source.Consumed, typeof(<#= spec.Type #>), <#= spec.ExtraArguments #>requestHint); + } + + return result!; + } + + /// + /// Decodes value from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// +<# + foreach (var doc in spec.ExtraParameterDocs) + { +#> + /// <#= doc #> +<# + } +#> + /// to cancel long running operation. This value can be omitted. + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract <#= spec.Type #>? Decode<#= spec.Name #>(ref SequenceReader source, out int requestHint, <#= spec.ExtraParameters #>CancellationToken cancellationToken = default); + + /// + /// Decodes value or null from specified sequence. + /// + /// SequenceReader<byte>. +<# + foreach (var doc in spec.ExtraParameterDocs) + { +#> + /// <#= doc #> +<# + } +#> + /// + /// Decoded value if this method succeeds to decode value. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public <#= spec.Type #>? DecodeNullable<#= spec.Name #>(ref SequenceReader source, <#= spec.ExtraParameters #>CancellationToken cancellationToken = default) + { + var result = this.DecodeNullable<#= spec.Name #>(ref source, out var requestHint, <#= spec.ExtraArguments #>cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInput<#= spec.ThrowSuffix #>(source.Consumed, typeof(<#= spec.Type #>), <#= spec.ExtraArguments #>requestHint); + } + + return result; + } + + /// + /// Decodes value or null from specified sequence. + /// + /// SequenceReader<byte>. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// +<# + foreach (var doc in spec.ExtraParameterDocs) + { +#> + /// <#= doc #> +<# + } +#> + /// + /// Decoded value if this method succeeds to decode value. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract <#= spec.Type #>? DecodeNullable<#= spec.Name #>(ref SequenceReader source, out int requestHint, <#= spec.ExtraParameters #>CancellationToken cancellationToken = default); + + /// + /// Decodes the value from specified sequence to <#= spec.SpanTypeDoc #>. + /// + /// SequenceReader<byte>. + /// <#= spec.SpanTypeDoc #> which will store the decoded binary. +<# + foreach (var doc in spec.ExtraParameterDocs) + { +#> + /// <#= doc #> +<# + } +#> + /// to cancel long running operation. This value can be omitted. + /// + /// The length of the decoded value. + /// This value can be 0 for empty string. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public int Decode<#= spec.Name #>(ref SequenceReader source, <#= spec.SpanType #> buffer, <#= spec.ExtraParameters #>CancellationToken cancellationToken = default) + { + var result = this.Decode<#= spec.Name #>(ref source, buffer, out var requestHint, <#= spec.ExtraArguments #>cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInput<#= spec.ThrowSuffix #>(source.Consumed, typeof(<#= spec.Type #>), <#= spec.ExtraArguments #>requestHint); + } + + return result!; + } + + /// + /// Decodes the value from specified sequence to <#= spec.SpanTypeDoc #>. + /// + /// SequenceReader<byte>. + /// <#= spec.SpanTypeDoc #> which will store the decoded binary. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// +<# + foreach (var doc in spec.ExtraParameterDocs) + { +#> + /// <#= doc #> +<# + } +#> + /// to cancel long running operation. This value can be omitted. + /// + /// The length of the decoded value. + /// This value can be 0 for empty string. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + public abstract int Decode<#= spec.Name #>(ref SequenceReader source, <#= spec.SpanType #> buffer, out int requestHint, <#= spec.ExtraParameters #>CancellationToken cancellationToken = default); + + /// + /// Decodes the value or null from specified sequence to <#= spec.SpanTypeDoc #>. + /// + /// SequenceReader<byte>. + /// <#= spec.SpanTypeDoc #> which will store the decoded binary. +<# + foreach (var doc in spec.ExtraParameterDocs) + { +#> + /// <#= doc #> +<# + } +#> + /// + /// The length of the decoded value; + /// null when the underlying value is null. + /// This value can be 0 for empty string. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public int? DecodeNullable<#= spec.Name #>(ref SequenceReader source, <#= spec.SpanType #> buffer, <#= spec.ExtraParameters #>CancellationToken cancellationToken = default) + { + var result = this.DecodeNullable<#= spec.Name #>(ref source, buffer, out var requestHint, <#= spec.ExtraArguments #>cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInput<#= spec.ThrowSuffix #>(source.Consumed, typeof(<#= spec.Type #>), <#= spec.ExtraArguments #>requestHint); + } + + return result; + } + + /// + /// Decodes the value or null from specified sequence to <#= spec.SpanTypeDoc #>. + /// + /// SequenceReader<byte>. + /// <#= spec.SpanTypeDoc #> which will store the decoded binary. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// +<# + foreach (var doc in spec.ExtraParameterDocs) + { +#> + /// <#= doc #> +<# + } +#> + /// + /// The length of the decoded value; + /// null when the underlying value is null. + /// This value can be 0 for empty string. + /// Note that the value of this return is not defined when is 0. + /// + /// contains valid byte sequence for the underlying format. + /// The underlying format value is not compatible to type. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract int? DecodeNullable<#= spec.Name #>(ref SequenceReader source, <#= spec.SpanType #> buffer, out int requestHint, <#= spec.ExtraParameters #>CancellationToken cancellationToken = default); + +<# + if (!String.IsNullOrEmpty(spec.IfDef)) + { +#> + +#endif // <#= spec.IfDef #> + +<# + } +} +#> + } +} diff --git a/src/MsgPack.Abstraction/Internal/FormatDecoder.cs b/src/MsgPack.Abstraction/Internal/FormatDecoder.cs new file mode 100644 index 000000000..684ea87ef --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FormatDecoder.cs @@ -0,0 +1,258 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace MsgPack.Internal +{ + /// + /// Defines an interface and basic functionarity of stateless . + /// + /// + /// The is stateless, so caller (serializer, writer, etc.) can cache the instance for performance. + /// + public abstract partial class FormatDecoder + { + public FormatDecoderOptions Options { get; } + + protected FormatDecoder(FormatDecoderOptions options) + { + this.Options = Ensure.NotNull(options); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void Skip(ref SequenceReader source, in CollectionContext collectionContext, CancellationToken cancellationToken = default) + { + this.Skip(ref source, collectionContext, out var requestHint, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInputForSkip(source.Consumed, requestHint); + } + } + + public abstract void Skip(ref SequenceReader source, in CollectionContext collectionContext, out int requestHint, CancellationToken cancellationToken = default); + + public void DecodeItem(ref SequenceReader source, out DecodeItemResult result, CancellationToken cancellationToken = default) + { + this.DecodeItem(ref source, out result, out var requestHint, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInputForAnyItem(source.Consumed, requestHint); + } + } + + public abstract void DecodeItem(ref SequenceReader source, out DecodeItemResult result, out int requestHint, CancellationToken cancellationToken = default); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void GetRawString(ref SequenceReader source, out ReadOnlySpan rawString, CancellationToken cancellationToken = default) + { + if (!this.GetRawString(ref source, out rawString, out var requestHint, cancellationToken)) + { + Throw.InsufficientInputForRawString(source.Consumed, requestHint); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public bool TryDecodeNull(ref SequenceReader source) + { + if (this.TryDecodeNull(ref source, out var requestHint)) + { + return true; + } + + if (requestHint != 0) + { + Throw.InsufficientInputForNull(source.Consumed, requestHint); + } + + return false; + } + + public abstract bool TryDecodeNull(ref SequenceReader source, out int requestHint); + + + public abstract bool GetRawString(ref SequenceReader source, out ReadOnlySpan rawString, out int requestHint, CancellationToken cancellationToken = default); + + /// + /// Decodes current data as array or map header, and returns the items count if known. + /// + /// Reader of source byte sequence. If and only if this method succeeds, the reader will be advanced. + /// Items count if known; -1 if underlying format does not contain any count information; 0 if underlying format is not an array nor a map. + /// + /// for array, for map (dictionary). + /// This method does not return anything else, but may throw an exception. + /// + /// The decoded value is not an array nor a map. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public CollectionType DecodeArrayOrMapHeader(ref SequenceReader source, out int itemsCount) + { + var result = this.DecodeArrayOrMapHeader(ref source, out itemsCount, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInputForDecodeArrayOrMapHeader(source.Consumed, requestHint); + } + + return result; + } + + /// + /// Decodes current data as array or map header, and returns the items count if known. + /// + /// Reader of source byte sequence. If and only if this method succeeds, the reader will be advanced. + /// Items count if known; -1 if underlying format does not contain any count information; 0 if underlying format is not an array nor a map. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// for array, for map (dictionary), or if there were not enough bytes to decode. + /// This method does not return anything else, but may throw an exception. + /// + /// The decoded value is not an array nor a map. + public abstract CollectionType DecodeArrayOrMapHeader(ref SequenceReader source, out int itemsCount, out int requestHint); + + /// + /// Decodes current data as array header, and returns the items count if known. + /// + /// Reader of source byte sequence. If and only if this method succeeds, the reader will be advanced. + /// + /// Items count if known; -1 if underlying format does not contain any count information. + /// Note that 0 is valid value when the array is empty. + /// + /// The decoded value is not an array. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public int DecodeArrayHeader(ref SequenceReader source) + { + var result = this.DecodeArrayHeader(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInputForDecodeArrayHeader(source.Consumed, requestHint); + } + + return result; + } + + /// + /// Decodes current data as array header, and returns the items count if known. + /// + /// Reader of source byte sequence. If and only if this method succeeds, the reader will be advanced. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Items count if known; -1 if underlying format does not contain any count information. + /// Note that 0 is valid value when the array is empty. + /// + /// The decoded value is not an array. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public abstract int DecodeArrayHeader(ref SequenceReader source, out int requestHint); + + /// + /// Decodes current data as map header, and returns the items count if known. + /// + /// Reader of source byte sequence. If and only if this method succeeds, the reader will be advanced. + /// + /// Items count if known; -1 if underlying format does not contain any count information. + /// Note that 0 is valid value when the map is empty. + /// + /// The decoded value is not a map. + /// does not contain enough bytes to decode. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public int DecodeMapHeader(ref SequenceReader source) + { + var result = this.DecodeMapHeader(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInputForDecodeMapHeader(source.Consumed, requestHint); + } + + return result; + } + + /// + /// Decodes current data as map header, and returns the items count if known. + /// + /// Reader of source byte sequence. If and only if this method succeeds, the reader will be advanced. + /// + /// 0 if this method succeeds to decode value; Positive integer when does not contain enough bytes to decode, and required memory bytes hint is stored. + /// Note that -1 represents unknown size. If so, caller must supply new buffer with most efficient size. + /// + /// + /// Items count if known; -1 if underlying format does not contain any count information. + /// Note that 0 is valid value when the map is empty. + /// + /// The decoded value is not a map. + public abstract int DecodeMapHeader(ref SequenceReader source, out int requestHint); + + public virtual void DecodeExtension(ref SequenceReader source, out ExtensionTypeObject result, out int requestHint, CancellationToken cancellationToken = default) + { + Throw.ExtensionsIsNotSupported(); + // never + result = default; + requestHint = -1; + } + + public CollectionType DecodeArrayOrMap(ref SequenceReader source, out CollectionItemIterator iterator) + { + var result = this.DecodeArrayOrMap(ref source, out iterator, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInputForDecodeArrayOrMapHeader(source.Consumed, requestHint); + } + + return result; + } + + public abstract CollectionType DecodeArrayOrMap(ref SequenceReader source, out CollectionItemIterator iterator, out int requestHint); + + public CollectionItemIterator DecodeArray(ref SequenceReader source) + { + var result = this.DecodeArray(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInputForDecodeArrayHeader(source.Consumed, requestHint); + } + + return result; + } + + public abstract CollectionItemIterator DecodeArray(ref SequenceReader source, out int requestHint); + + public CollectionItemIterator DecodeMap(ref SequenceReader source) + { + var result = this.DecodeMap(ref source, out var requestHint); + if (requestHint != 0) + { + Throw.InsufficientInputForDecodeMapHeader(source.Consumed, requestHint); + } + + return result; + } + + public abstract CollectionItemIterator DecodeMap(ref SequenceReader source, out int requestHint); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void Drain(ref SequenceReader source, in CollectionContext collectionContext, long itemsCount, CancellationToken cancellationToken = default) + { + if (itemsCount <= 0) + { + return; + } + + this.Drain(ref source, collectionContext, itemsCount, out var requestHint, cancellationToken); + if (requestHint != 0) + { + Throw.InsufficientInputForDecodeMapHeader(source.Consumed, requestHint); + } + } + + public abstract void Drain(ref SequenceReader source, in CollectionContext collectionContext, long itemsCount, out int requestHint, CancellationToken cancellationToken = default); + } +} diff --git a/src/MsgPack.Abstraction/Internal/FormatDecoderOptions.cs b/src/MsgPack.Abstraction/Internal/FormatDecoderOptions.cs new file mode 100644 index 000000000..6fa191e60 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FormatDecoderOptions.cs @@ -0,0 +1,51 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Buffers; +using MsgPack.Codecs; + +namespace MsgPack.Internal +{ + public abstract class FormatDecoderOptions + { + public bool CanTreatRealAsInteger { get; } + + public int CancellationSupportThreshold { get; } + + public ArrayPool ByteBufferPool { get; } + + public ArrayPool CharBufferPool { get; } + + public int MaxByteBufferLength { get; } + + public int MaxCharBufferLength { get; } + + public int MaxNumberLengthInBytes { get; } + + public int MaxStringLengthInBytes { get; } + + public int MaxBinaryLengthInBytes { get; } + + public bool ClearsBuffer { get; } + + public CodecFeatures Features { get; } + + protected FormatDecoderOptions(FormatDecoderOptionsBuilder builder, CodecFeatures features) + { + builder = Ensure.NotNull(builder); + + this.CanTreatRealAsInteger = builder.CanTreatRealAsInteger; + this.CancellationSupportThreshold = builder.CancellationSupportThreshold; + this.ClearsBuffer = builder.ClearsBuffer; + this.ByteBufferPool = builder.ByteBufferPool; + this.CharBufferPool = builder.CharBufferPool; + this.MaxByteBufferLength = builder.MaxByteBufferLength; + this.MaxCharBufferLength = builder.MaxCharBufferLength; + this.MaxNumberLengthInBytes = builder.MaxNumberLengthInBytes; + this.MaxStringLengthInBytes = builder.MaxStringLengthInBytes; + this.MaxBinaryLengthInBytes = builder.MaxBinaryLengthInBytes; + this.Features = Ensure.NotNull(features); + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FormatDecoderOptionsBuilder.cs b/src/MsgPack.Abstraction/Internal/FormatDecoderOptionsBuilder.cs new file mode 100644 index 000000000..44b584d1a --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FormatDecoderOptionsBuilder.cs @@ -0,0 +1,94 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Buffers; + +namespace MsgPack.Internal +{ +#warning Remove unused buffer max pr rename to initial buffer length if appropriate. + public abstract class FormatDecoderOptionsBuilder + { + public bool CanTreatRealAsInteger { get; set; } = OptionsDefaults.CanTreatRealAsInteger; + + private int _cancellationSupportThreshold = OptionsDefaults.CancellationSupportThreshold; + + public int CancellationSupportThreshold + { + get => this._cancellationSupportThreshold; + set => this._cancellationSupportThreshold = Ensure.IsNotLessThan(value, 1); + } + + private int _maxNumberLengthInBytes = OptionsDefaults.MaxNumberLengthInBytes; + + public int MaxNumberLengthInBytes + { + get => this._maxNumberLengthInBytes; + set => this._maxNumberLengthInBytes = Ensure.IsBetween(value, 1, OptionsDefaults.MaxSingleByteCollectionLength); + } + + private int _maxStringLengthInBytes = OptionsDefaults.MaxStringLengthInBytes; + + public int MaxStringLengthInBytes + { + get => this._maxStringLengthInBytes; + set => this._maxStringLengthInBytes = Ensure.IsBetween(value, 1, OptionsDefaults.MaxMultiByteCollectionLength); + } + + private int _naxBinaryLengthInBytes = OptionsDefaults.MaxBinaryLengthInBytes; + + public int MaxBinaryLengthInBytes + { + get => this._naxBinaryLengthInBytes; + set => this._naxBinaryLengthInBytes = Ensure.IsBetween(value, 1, OptionsDefaults.MaxSingleByteCollectionLength); + } + + private int _maxByteBufferLength = OptionsDefaults.MaxByteBufferLength; + + public int MaxByteBufferLength + { + get => this._maxByteBufferLength; + set => this._maxByteBufferLength = Ensure.IsNotLessThan(value, 4); + } + + private int _maxCharBufferLength = OptionsDefaults.MaxCharBufferLength; + + public int MaxCharBufferLength + { + get => this._maxCharBufferLength; + set => this._maxCharBufferLength = Ensure.IsNotLessThan(value, 2); + } + + private ArrayPool _byteBufferPool = OptionsDefaults.ByteBufferPool; + + public ArrayPool ByteBufferPool + { + get => this._byteBufferPool; + set => this._byteBufferPool = Ensure.NotNull(value); + } + + private ArrayPool _charBufferPool = OptionsDefaults.CharBufferPool; + + public ArrayPool CharBufferPool + { + get => this._charBufferPool; + set => this._charBufferPool = Ensure.NotNull(value); + } + + public bool ClearsBuffer { get; set; } = OptionsDefaults.ClearsBufferOnReturn; + + protected FormatDecoderOptionsBuilder() { } + + public FormatDecoderOptionsBuilder WithoutBufferClear() + { + this.ClearsBuffer = false; + return this; + } + + public FormatDecoderOptionsBuilder ProhibitTreatRealAsInteger() + { + this.CanTreatRealAsInteger = false; + return this; + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FormatEncoder.Primitives.cs b/src/MsgPack.Abstraction/Internal/FormatEncoder.Primitives.cs new file mode 100644 index 000000000..0c4701a06 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FormatEncoder.Primitives.cs @@ -0,0 +1,228 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + partial class FormatEncoder + { + /// + /// Encodes value to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + /// is null. + /// The underlying format does not suppor this type. + public abstract void EncodeInt32(Int32 value, IBufferWriter buffer); + + /// + /// Encodes value or null to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void EncodeInt32(Int32? value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + if (value == null) + { + this.EncodeNull(buffer); + } + else + { + this.EncodeInt32(value.GetValueOrDefault(), buffer); + } + } + /// + /// Encodes value to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + /// is null. + /// The underlying format does not suppor this type. + public abstract void EncodeInt64(Int64 value, IBufferWriter buffer); + + /// + /// Encodes value or null to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void EncodeInt64(Int64? value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + if (value == null) + { + this.EncodeNull(buffer); + } + else + { + this.EncodeInt64(value.GetValueOrDefault(), buffer); + } + } + /// + /// Encodes value to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + /// is null. + /// The underlying format does not suppor this type. + public abstract void EncodeUInt32(UInt32 value, IBufferWriter buffer); + + /// + /// Encodes value or null to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void EncodeUInt32(UInt32? value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + if (value == null) + { + this.EncodeNull(buffer); + } + else + { + this.EncodeUInt32(value.GetValueOrDefault(), buffer); + } + } + /// + /// Encodes value to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + /// is null. + /// The underlying format does not suppor this type. + public abstract void EncodeUInt64(UInt64 value, IBufferWriter buffer); + + /// + /// Encodes value or null to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void EncodeUInt64(UInt64? value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + if (value == null) + { + this.EncodeNull(buffer); + } + else + { + this.EncodeUInt64(value.GetValueOrDefault(), buffer); + } + } + /// + /// Encodes value to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + /// is null. + /// The underlying format does not suppor this type. + public abstract void EncodeSingle(Single value, IBufferWriter buffer); + + /// + /// Encodes value or null to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void EncodeSingle(Single? value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + if (value == null) + { + this.EncodeNull(buffer); + } + else + { + this.EncodeSingle(value.GetValueOrDefault(), buffer); + } + } + /// + /// Encodes value to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + /// is null. + /// The underlying format does not suppor this type. + public abstract void EncodeDouble(Double value, IBufferWriter buffer); + + /// + /// Encodes value or null to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void EncodeDouble(Double? value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + if (value == null) + { + this.EncodeNull(buffer); + } + else + { + this.EncodeDouble(value.GetValueOrDefault(), buffer); + } + } + /// + /// Encodes value to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + /// is null. + /// The underlying format does not suppor this type. + public abstract void EncodeBoolean(Boolean value, IBufferWriter buffer); + + /// + /// Encodes value or null to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void EncodeBoolean(Boolean? value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + if (value == null) + { + this.EncodeNull(buffer); + } + else + { + this.EncodeBoolean(value.GetValueOrDefault(), buffer); + } + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FormatEncoder.Primitives.tt b/src/MsgPack.Abstraction/Internal/FormatEncoder.Primitives.tt new file mode 100644 index 000000000..939f76871 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FormatEncoder.Primitives.tt @@ -0,0 +1,65 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ output extension=".cs" #> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + partial class FormatEncoder + { +<# +foreach (var inputType in new [] { + "Int32", + "Int64", + "UInt32", + "UInt64", + "Single", + "Double", + "Boolean" +}) +{ +#> + /// + /// Encodes value to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + /// is null. + /// The underlying format does not suppor this type. + public abstract void Encode<#= inputType #>(<#= inputType #> value, IBufferWriter buffer); + + /// + /// Encodes value or null to specified buffer. + /// The implementation will choose most compact format. + /// + /// Value to be encoded. + /// IBufferWriter<byte>. + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void Encode<#= inputType #>(<#= inputType #>? value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + if (value == null) + { + this.EncodeNull(buffer); + } + else + { + this.Encode<#= inputType #>(value.GetValueOrDefault(), buffer); + } + } +<# +} +#> + } +} diff --git a/src/MsgPack.Abstraction/Internal/FormatEncoder.cs b/src/MsgPack.Abstraction/Internal/FormatEncoder.cs new file mode 100644 index 000000000..928c1e26c --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FormatEncoder.cs @@ -0,0 +1,186 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; + +namespace MsgPack.Internal +{ + /// + /// Defines an interface and basic functionarity of stateless . + /// + /// + /// The is stateless, so caller (serializer, reader, etc.) can cache the instance for performance. + /// + public abstract partial class FormatEncoder + { + public FormatEncoderOptions Options { get; } + + protected FormatEncoder(FormatEncoderOptions options) + { + this.Options = Ensure.NotNull(options); + } + + public abstract void EncodeNull(IBufferWriter buffer); + +#if FEATURE_UTF8STRING + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void EncodeString(Utf8String? value, IBufferWriter buffer) + => this.EncodeString(value.AsBytes(), buffer); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void EncodeString(Utf8Span value, IBufferWriter buffer) + => this.EncodeString(value.AsBytes(), buffer); +#endif // FEATURE_UTF8STRING + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void EncodeString(string? value, IBufferWriter buffer, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + if (value == null) + { + this.EncodeNull(buffer); + return; + } + + this.EncodeString(value.AsSpan(), buffer, encoding, cancellationToken); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void EncodeString(StringBuilder? value, IBufferWriter buffer, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + if (value == null) + { + this.EncodeNull(buffer); + return; + } + + this.EncodeString(value.ToSequence(), buffer, encoding, cancellationToken); + } + + public abstract void EncodeString(ReadOnlySpan value, IBufferWriter buffer, Encoding? encoding = null, CancellationToken cancellationToken = default); + + public abstract void EncodeString(in ReadOnlySequence value, IBufferWriter buffer, Encoding? encoding = null, CancellationToken cancellationToken = default); + + public abstract void EncodeString(ReadOnlySpan encodedValue, int charLength, IBufferWriter buffer, CancellationToken cancellationToken = default); + + public abstract void EncodeString(in ReadOnlySequence encodedValue, int charLength, IBufferWriter buffer, CancellationToken cancellationToken = default); + + public abstract void EncodeRawString(ReadOnlySpan rawString, int charLength, IBufferWriter buffer, CancellationToken cancellationToken = default); + + public abstract void EncodeBinary(ReadOnlySpan value, IBufferWriter buffer, CancellationToken cancellationToken = default); + + public abstract void EncodeBinary(in ReadOnlySequence value, IBufferWriter buffer, CancellationToken cancellationToken = default); + + public abstract void EncodeArrayStart(int length, IBufferWriter buffer, in CollectionContext collectionContext); + + public abstract void EncodeArrayEnd(int length, IBufferWriter buffer, in CollectionContext collectionContext); + + public abstract void EncodeArrayItemStart(int index, IBufferWriter buffer, in CollectionContext collectionContext); + + public abstract void EncodeArrayItemEnd(int index, IBufferWriter buffer, in CollectionContext collectionContext); + + public abstract void EncodeMapStart(int length, IBufferWriter buffer, in CollectionContext collectionContext); + + public abstract void EncodeMapEnd(int length, IBufferWriter buffer, in CollectionContext collectionContext); + + public abstract void EncodeMapKeyStart(int index, IBufferWriter buffer, in CollectionContext collectionContext); + + public abstract void EncodeMapKeyEnd(int index, IBufferWriter buffer, in CollectionContext collectionContext); + + public abstract void EncodeMapValueStart(int index, IBufferWriter buffer, in CollectionContext collectionContext); + + public abstract void EncodeMapValueEnd(int index, IBufferWriter buffer, in CollectionContext collectionContext); + + public virtual void EncodeExtension(ExtensionType typeCode, ReadOnlySpan serializedValue, IBufferWriter buffer) + => Throw.ExtensionsIsNotSupported(); + + public virtual void EncodeExtension(ExtensionType typeCode, in ReadOnlySequence serializedValue, IBufferWriter buffer) + => Throw.ExtensionsIsNotSupported(); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void WriteRaw(ReadOnlySpan value, IBufferWriter buffer, CancellationToken cancellationToken = default) + { + if (value.Length <= this.Options.CancellationSupportThreshold) + { + buffer.Write(value); + } + else + { + WriteRawSlow(buffer, value, buffer.GetSpan(), cancellationToken); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void WriteRawSlow(IBufferWriter buffer, in ReadOnlySpan source, Span destination, CancellationToken cancellationToken) + { + // https://github.com/dotnet/runtime/blob/af1db3eccbc238745e1d163458c92c1bfa650fbd/src/libraries/System.Memory/src/System/Buffers/BuffersExtensions.cs#L133 + ReadOnlySpan input = source; + while (true) + { + if (destination.IsEmpty) + { + ThrowBufferWriterReturnsEmptyBuffer(nameof(buffer)); + } + + var writeSize = Math.Min(destination.Length, input.Length); + input.Slice(0, writeSize).CopyTo(destination); + buffer.Advance(writeSize); + input = input.Slice(writeSize); + if (input.Length > 0) + { + destination = buffer.GetSpan(); + + cancellationToken.ThrowIfCancellationRequested(); + continue; + } + + return; + } + } + + private static void ThrowBufferWriterReturnsEmptyBuffer(string paramName) + => throw new ArgumentOutOfRangeException(paramName); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void WriteRaw(in ReadOnlySequence value, IBufferWriter buffer, CancellationToken cancellationToken = default) + { + buffer = Ensure.NotNull(buffer); + + if (value.Length <= this.Options.MaxByteBufferLength) + { + var length32 = unchecked((int)value.Length); + var span = buffer.GetSpan(length32); + value.CopyTo(span); + buffer.Advance(length32); + return; + } + + WriteRawSlow(value, buffer, cancellationToken); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void WriteRawSlow(in ReadOnlySequence value, IBufferWriter buffer, CancellationToken cancellationToken = default) + { + var reader = new SequenceReader(value); + while (!reader.End) + { + cancellationToken.ThrowIfCancellationRequested(); + + var sink = buffer.GetSpan(); + if (sink.IsEmpty) + { + ThrowBufferWriterReturnsEmptyBuffer(nameof(buffer)); + } + + var source = reader.UnreadSpan.Slice(0, Math.Min(sink.Length, reader.UnreadSpan.Length)); + source.CopyTo(sink); + buffer.Advance(source.Length); + reader.Advance(source.Length); + } + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FormatEncoderOptions.cs b/src/MsgPack.Abstraction/Internal/FormatEncoderOptions.cs new file mode 100644 index 000000000..53da6645f --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FormatEncoderOptions.cs @@ -0,0 +1,39 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Buffers; +using MsgPack.Codecs; + +namespace MsgPack.Internal +{ + public abstract class FormatEncoderOptions + { + public int CancellationSupportThreshold { get; } + + public int MaxByteBufferLength { get; } + + public int MaxCharBufferLength { get; } + + public ArrayPool ByteBufferPool { get; } + + public ArrayPool CharBufferPool { get; } + + public bool ClearsBuffer { get; } + + public CodecFeatures Features { get; } + + protected FormatEncoderOptions(FormatEncoderOptionsBuilder builder, CodecFeatures features) + { + builder = Ensure.NotNull(builder); + + this.CancellationSupportThreshold = builder.CancellationSupportThreshold; + this.MaxByteBufferLength = builder.MaxByteBufferLength; + this.MaxCharBufferLength = builder.MaxCharBufferLength; + this.ByteBufferPool = builder.ByteBufferPool; + this.CharBufferPool = builder.CharBufferPool; + this.ClearsBuffer = builder.ClearsBuffer; + this.Features = Ensure.NotNull(features); + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/FormatEncoderOptionsBuilder.cs b/src/MsgPack.Abstraction/Internal/FormatEncoderOptionsBuilder.cs new file mode 100644 index 000000000..d31f70499 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/FormatEncoderOptionsBuilder.cs @@ -0,0 +1,61 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Buffers; + +namespace MsgPack.Internal +{ + public abstract class FormatEncoderOptionsBuilder + { + private int _cancellationSupportThreshold = OptionsDefaults.CancellationSupportThreshold; + + public int CancellationSupportThreshold + { + get => this._cancellationSupportThreshold; + set => this._cancellationSupportThreshold = Ensure.IsNotLessThan(value, 1); + } + + private int _maxByteBufferLength = OptionsDefaults.MaxByteBufferLength; + + public int MaxByteBufferLength + { + get => this._maxByteBufferLength; + set => this._maxByteBufferLength = Ensure.IsNotLessThan(value, 4); + } + + private int _maxCharBufferLength = OptionsDefaults.MaxCharBufferLength; + + public int MaxCharBufferLength + { + get => this._maxCharBufferLength; + set => this._maxCharBufferLength = Ensure.IsNotLessThan(value, 2); + } + + private ArrayPool _byteBufferPool = OptionsDefaults.ByteBufferPool; + + public ArrayPool ByteBufferPool + { + get => this._byteBufferPool; + set => this._byteBufferPool = Ensure.NotNull(value); + } + + private ArrayPool _charBufferPool = OptionsDefaults.CharBufferPool; + + public ArrayPool CharBufferPool + { + get => this._charBufferPool; + set => this._charBufferPool = Ensure.NotNull(value); + } + + public bool ClearsBuffer { get; set; } = OptionsDefaults.ClearsBufferOnReturn; + + protected FormatEncoderOptionsBuilder() { } + + public FormatEncoderOptionsBuilder WithoutBufferClear() + { + this.ClearsBuffer = false; + return this; + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/MethodImplOptionsShim.cs b/src/MsgPack.Abstraction/Internal/MethodImplOptionsShim.cs new file mode 100644 index 000000000..07bf05070 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/MethodImplOptionsShim.cs @@ -0,0 +1,22 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + /// + /// Defines compatibility shims for legacy runtime. + /// + internal static class MethodImplOptionsShim + { + /// + /// Tells JIT to inline aggressively. + /// + /// + /// This value is not defined in .NET Framework 3.5, but the runtime will ignore this flag value. + /// + public const MethodImplOptions AggressiveInlining = (MethodImplOptions)256; + } +} diff --git a/src/MsgPack.Abstraction/Internal/OptionsDefaults.cs b/src/MsgPack.Abstraction/Internal/OptionsDefaults.cs new file mode 100644 index 000000000..bdb6492ba --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/OptionsDefaults.cs @@ -0,0 +1,32 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Buffers; +using MsgPack.Serialization; + +namespace MsgPack.Internal +{ +#warning TODO: tuning default as backward compatible + internal static class OptionsDefaults + { + public const int MaxSingleByteCollectionLength = 0x7FFFFFC7; + public const int MaxMultiByteCollectionLength = 0X7FEFFFFF; + + public static readonly int CancellationSupportThreshold = 128 * 1024 * 1024; // About 0.1 sec in desktop, more for IoT + public static readonly int MaxNumberLengthInBytes = 32; + public static readonly int MaxStringLengthInBytes = 256 * 1024 * 1024; + public static readonly int MaxBinaryLengthInBytes = 256 * 1024 * 1024; + public static readonly int MaxByteBufferLength = 2 * 1024 * 1024; + public static readonly int MaxCharBufferLength = 2 * 1024 * 1024; + public static readonly int MaxArrayLength = 1024 * 1024; + public static readonly int MaxMapCount = 1024 * 1024; + public static readonly int MaxPropertyKeyLength = 256; + public static readonly int MaxDepth = 100; + public static readonly ArrayPool ByteBufferPool = ArrayPool.Shared; + public static readonly ArrayPool CharBufferPool = ArrayPool.Shared; + public static readonly bool ClearsBufferOnReturn = false; + public static readonly bool CanTreatRealAsInteger = true; + public static readonly SerializationMethod? PreferredSerializationMethod = null; + } +} diff --git a/src/MsgPack.Abstraction/Internal/ReadOnlyStreamSequence.cs b/src/MsgPack.Abstraction/Internal/ReadOnlyStreamSequence.cs new file mode 100644 index 000000000..58334cd6a --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/ReadOnlyStreamSequence.cs @@ -0,0 +1,131 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace MsgPack.Internal +{ + /// + /// Wraps and provides via internal buffer. + /// + /// + /// User of this object must be call when this sequence used. + /// Note that the will NOT be closed when is called. + /// + public sealed class ReadOnlyStreamSequence : IDisposable + { + private readonly ArrayPool _arrayPool; + private readonly bool _clearsBuffer; + private readonly Stream _stream; // DO NOT Dispose this in this class. + private byte[] _array; + + /// + /// Gets the position of the head of the on underlying . + /// + /// + /// The position of the head of the on underlying . + /// + public long Position { get; private set; } + + /// + /// Gets internal buffer as which holds last fetched data. + /// + public ReadOnlyMemory Memory { get; private set; } + + /// + /// Gets internal buffer as which holds last fetched data. + /// + public ReadOnlySequence Sequence => new ReadOnlySequence(this.Memory); + + internal ReadOnlyStreamSequence(Stream stream, ArrayPool arrayPool, int bufferSize, bool clearsBuffer) + { + this._stream = stream; + this._arrayPool = arrayPool; + this._array = arrayPool.Rent(bufferSize); + this.Memory = this._array; + this._clearsBuffer = clearsBuffer; + } + + /// + public void Dispose() + { + var array = Interlocked.Exchange(ref this._array, null!); + if (array != null) + { + this._arrayPool.Return(this._array, this._clearsBuffer); + } + } + + /// + /// Fetches first data from underlying and fills internal buffer. + /// Remaining data in the current buffer will be included new buffer. + /// + /// to cancel read operation. + /// Async operation state. + /// This object is already disposed. + public ValueTask FetchAsync(CancellationToken cancellationToken) + => this.FetchAsync(0, cancellationToken); + + /// + /// Fetches next data from underlying and fills internal buffer. + /// Remaining data in the current buffer will be included new buffer. + /// + /// + /// Hint size to fetch from . + /// 0 is valid value which means that the implementation fetches bytes from to fill out current buffer. + /// Note that it is not guaranteed that contains this length because underlying may not have enough data. + /// + /// to cancel read operation. + /// Async operation state. + /// This object is already disposed. + public async ValueTask FetchAsync(int requestHint, CancellationToken cancellationToken) + { + if (this._array == null) + { + Throw.ObjectDisposed(this.ToString()); + // never + return; + } + + var existingLength = this.Memory.Length; + if (existingLength + requestHint > this._array.Length) + { + // realloc + var newArray = this._arrayPool.Rent(Math.Max(this._array.Length * 2, existingLength + requestHint)); + this.Memory.CopyTo(newArray); + this._arrayPool.Return(this._array, this._clearsBuffer); + this._array = newArray; + } + else + { + // compact + this.Memory.CopyTo(this._array); + } + + var readLength = await this._stream.ReadAsync(this._array.AsMemory(existingLength), cancellationToken).ConfigureAwait(false); + this.Memory = this._array.AsMemory(existingLength + readLength); + } + + /// + /// Advances internal buffer position. + /// + /// Length to be advanced. + /// is greater than length of . + /// This object is already disposed. + public void Advance(long length) + { + if (this._array == null) + { + Throw.ObjectDisposed(this.ToString()); + } + + this.Memory = this.Memory.Slice((int)Ensure.IsNotGreaterThan(length, (long)this.Memory.Length)); + this.Position += length; + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/StreamBufferWriter.cs b/src/MsgPack.Abstraction/Internal/StreamBufferWriter.cs new file mode 100644 index 000000000..5b94e0035 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/StreamBufferWriter.cs @@ -0,0 +1,113 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.IO; +using System.Threading.Tasks; + +namespace MsgPack.Internal +{ +#warning TODO: CodeGen + public sealed class StreamBufferWriter : IBufferWriter, IAsyncDisposable + { + private readonly Stream _underlying; + private readonly bool _ownsStream; + + private readonly ArrayPool _arrayPool; + private readonly bool _cleansBuffer; + + private byte[] _buffer = null!; + private int _offset; + + private ValueTask _flushTask; + private byte[]? _flushingBuffer; + + public int Remaining => this._buffer.Length - this._offset; + + public StreamBufferWriter( + Stream underlying, + bool ownsStream, + ArrayPool arrayPool, + bool cleansBuffer + ) + { + this._underlying = underlying; + this._ownsStream = ownsStream; + this._arrayPool = arrayPool; + this._cleansBuffer = cleansBuffer; + } + + public async ValueTask DisposeAsync() + { + await this._flushTask.ConfigureAwait(false); + this.CleanUpAsyncWork(); + await this.FlushToStreamAsync().ConfigureAwait(false); + await this._underlying.FlushAsync().ConfigureAwait(false); + this._arrayPool.Return(this._buffer, this._cleansBuffer); + this._buffer = null!; + this._offset = 0; + if (this._ownsStream) + { + await this._underlying.DisposeAsync(); + } + } + + public void Advance(int count) + => this._offset++; + + public Memory GetMemory(int sizeHint = 0) + { + this.EnsureBuffer(sizeHint); + return this._buffer.AsMemory(this._offset); + } + + public Span GetSpan(int sizeHint = 0) + { + this.EnsureBuffer(sizeHint); + return this._buffer.AsSpan(this._offset); + } + + private void EnsureBuffer(int sizeHint) + { + if (this._buffer == null) + { + throw new ObjectDisposedException(this.GetType().FullName); + } + + if (this.Remaining > 0 && this.Remaining >= sizeHint) + { + return; + } + + if (!this._flushTask.IsCompletedSuccessfully) + { + this._flushTask.ConfigureAwait(false).GetAwaiter().GetResult(); + this.CleanUpAsyncWork(); + } + + this._flushTask = this.FlushToStreamAsync(); + if (!this._flushTask.IsCompletedSuccessfully) + { + this._flushingBuffer = this._buffer; + this._buffer = this._arrayPool.Rent(this._buffer.Length); + } + + this._offset = 0; + } + + private ValueTask FlushToStreamAsync() + => this._underlying.WriteAsync(this._buffer.AsMemory(0, this._offset)); + + private void CleanUpAsyncWork() + { + if (this._flushingBuffer != null) + { + this._arrayPool.Return(this._flushingBuffer!, this._cleansBuffer); + this._flushingBuffer = null; + this._flushTask = default; + } + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/StringBuilderExtensions.cs b/src/MsgPack.Abstraction/Internal/StringBuilderExtensions.cs new file mode 100644 index 000000000..9b3e380ec --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/StringBuilderExtensions.cs @@ -0,0 +1,60 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Text; + +namespace MsgPack.Internal +{ + internal static partial class StringBuilderExtensions + { + public static ReadOnlySequence ToSequence(this StringBuilder builder) + { + var runningIndex = 0L; + var firstSegment = default(StringBuilderChunkSegment); + var lastSegment = default(StringBuilderChunkSegment); + foreach(var chunk in builder.GetChunks()) + { + if (chunk.IsEmpty) + { + continue; + } + + var segment = new StringBuilderChunkSegment(chunk, runningIndex); + runningIndex += chunk.Length; + if (lastSegment == null) + { + firstSegment = segment; + } + else + { + lastSegment.SetNext(segment); + } + + lastSegment = segment; + } + + if (lastSegment == null) + { + return ReadOnlySequence.Empty; + } + + return new ReadOnlySequence(firstSegment!, 0, lastSegment, lastSegment.Memory.Length - 1); + } + + private sealed class StringBuilderChunkSegment : ReadOnlySequenceSegment + { + public void SetNext(StringBuilderChunkSegment next) + => this.Next = next; + + public StringBuilderChunkSegment(ReadOnlyMemory chunk, long runningIndex) + { + this.Memory = chunk; + this.RunningIndex = runningIndex; + } + } + } +} diff --git a/src/MsgPack.Abstraction/Internal/Utf8EncodingNonBom.cs b/src/MsgPack.Abstraction/Internal/Utf8EncodingNonBom.cs new file mode 100644 index 000000000..0cf4999fc --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/Utf8EncodingNonBom.cs @@ -0,0 +1,21 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Text; + +namespace MsgPack.Internal +{ + /// + /// Defines sealed UTF-8 class which cannot emit BOM. + /// + /// + /// Sealed class typed static constant is important for JIT devirtualization optimization. + /// + internal sealed class Utf8EncodingNonBom : UTF8Encoding + { + public static Utf8EncodingNonBom Instance { get; } = new Utf8EncodingNonBom(); + + private Utf8EncodingNonBom() : base(encoderShouldEmitUTF8Identifier: false) { } + } +} diff --git a/src/MsgPack.Abstraction/Internal/Utf8EncodingNonBomStrict.cs b/src/MsgPack.Abstraction/Internal/Utf8EncodingNonBomStrict.cs new file mode 100644 index 000000000..b30665ab6 --- /dev/null +++ b/src/MsgPack.Abstraction/Internal/Utf8EncodingNonBomStrict.cs @@ -0,0 +1,24 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Text; + +namespace MsgPack.Internal +{ + /// + /// Defines sealed UTF-8 class which cannot emit BOM and throws Exception. + /// + /// + /// Sealed class typed static constant is important for JIT devirtualization optimization. + /// + internal sealed class Utf8EncodingNonBomStrict : UTF8Encoding + { + public static Utf8EncodingNonBomStrict Instance { get; } = new Utf8EncodingNonBomStrict(); + + private Utf8EncodingNonBomStrict() : base(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true) + { + this.DecoderFallback = DecoderFallback.ExceptionFallback; + } + } +} diff --git a/src/MsgPack.Abstraction/LimitExceededException.cs b/src/MsgPack.Abstraction/LimitExceededException.cs new file mode 100644 index 000000000..cf2ff5a03 --- /dev/null +++ b/src/MsgPack.Abstraction/LimitExceededException.cs @@ -0,0 +1,33 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +#if FEATURE_BINARY_SERIALIZATION +using System.Runtime.Serialization; +#endif // FEATURE_BINARY_SERIALIZATION + +namespace MsgPack +{ +#if FEATURE_BINARY_SERIALIZATION + [Serializable] +#endif // FEATURE_BINARY_SERIALIZATION + public sealed class LimitExceededException : DecodeException + { + public LimitExceededException(long potision) + : this(potision, "Some limit is exeeded.") { } + + public LimitExceededException(long potision, string? message) + : base(potision, message) { } + + public LimitExceededException(long potision, string? message, Exception? innerException) + : base(potision, message, innerException) { } + +#if FEATURE_BINARY_SERIALIZATION + + private LimitExceededException(SerializationInfo info, StreamingContext context) + : base(info, context) { } + +#endif // FEATURE_BINARY_SERIALIZATION + } +} diff --git a/src/MsgPack.Abstraction/MessageFormatException.cs b/src/MsgPack.Abstraction/MessageFormatException.cs new file mode 100644 index 000000000..722616721 --- /dev/null +++ b/src/MsgPack.Abstraction/MessageFormatException.cs @@ -0,0 +1,33 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +#if FEATURE_BINARY_SERIALIZATION +using System.Runtime.Serialization; +#endif // FEATURE_BINARY_SERIALIZATION + +namespace MsgPack +{ +#if FEATURE_BINARY_SERIALIZATION + [Serializable] +#endif // FEATURE_BINARY_SERIALIZATION + public sealed class MessageFormatException : DecodeException + { + public MessageFormatException(long position) + : this(position, "Input has invalid byte sequence.") { } + + public MessageFormatException(long position, string? message) + : base(position, message) { } + + public MessageFormatException(long position, string? message, Exception? innerException) + : base(position, message, innerException) { } + +#if FEATURE_BINARY_SERIALIZATION + + private MessageFormatException(SerializationInfo info, StreamingContext context) + : base(info, context) { } + +#endif // FEATURE_BINARY_SERIALIZATION + } +} diff --git a/src/MsgPack.Abstraction/MessagePackObject.cs b/src/MsgPack.Abstraction/MessagePackObject.cs new file mode 100644 index 000000000..8983d02e7 --- /dev/null +++ b/src/MsgPack.Abstraction/MessagePackObject.cs @@ -0,0 +1,1765 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +#nullable enable + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace MsgPack +{ + /// + /// Represents deserialized object of MsgPack. + /// + [StructLayout(LayoutKind.Auto)] + public partial struct MessagePackObject : IEquatable + { + #region -- Constructors -- + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + public MessagePackObject(Boolean value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = value ? (ulong)1 : 0; + this._handleOrTypeCode = BooleanTypeCode; + } + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + public MessagePackObject(Byte value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = value; + this._handleOrTypeCode = ByteTypeCode; + } + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + [CLSCompliant(false)] + public MessagePackObject(SByte value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = unchecked((ulong)value); + this._handleOrTypeCode = SByteTypeCode; + } + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + public MessagePackObject(Int16 value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = unchecked((ulong)value); + this._handleOrTypeCode = Int16TypeCode; + } + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + [CLSCompliant(false)] + public MessagePackObject(UInt16 value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = value; + this._handleOrTypeCode = UInt16TypeCode; + } + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + public MessagePackObject(Int32 value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = unchecked((ulong)value); + this._handleOrTypeCode = Int32TypeCode; + } + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + [CLSCompliant(false)] + public MessagePackObject(UInt32 value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = value; + this._handleOrTypeCode = UInt32TypeCode; + } + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + public MessagePackObject(Int64 value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = unchecked((ulong)value); + this._handleOrTypeCode = Int64TypeCode; + } + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + [CLSCompliant(false)] + public MessagePackObject(UInt64 value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = value; + this._handleOrTypeCode = UInt64TypeCode; + } + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + public MessagePackObject(Single value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = unchecked((ulong)Binary.ToBits(value)); + this._handleOrTypeCode = SingleTypeCode; + } + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + public MessagePackObject(Double value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = unchecked((ulong)BitConverter.DoubleToInt64Bits(value)); + this._handleOrTypeCode = DoubleTypeCode; + } + /// + /// Initializes a new instance of the [] type which wraps [] instance with specified manner. + /// + /// A bytes array to be wrapped. + /// + /// This constructor invokes with false, that means if you pass tha bytes array which is valid utf-8, resulting object can be , + /// and its should be . + /// + public MessagePackObject(String value) + : this(value, false) { } + + /// + /// Initializes a new instance of the [] type which wraps [] instance with specified manner. + /// + /// A bytes array to be wrapped. + /// true if always should be binary; false, otherwise. + /// + /// When the is true, then resulting object represents binary even if the is valid utf-8 sequence, + /// that is, its should be []. + /// On the other hand, when contrast, the is false, and if the is valid utf-8, + /// then the resulting object can be , + /// and its should be . + /// + public MessagePackObject(String value, bool isBinary) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + if (value == null) + { + this._handleOrTypeCode = null; + } + else + { + this._handleOrTypeCode = new MessagePackString(value); + } + } + /// + /// Initializes a new instance of the [] type which wraps [] instance with specified manner. + /// + /// A bytes array to be wrapped. + /// + /// This constructor invokes with false, that means if you pass tha bytes array which is valid utf-8, resulting object can be , + /// and its should be . + /// + public MessagePackObject(Byte[] value) + : this(value, false) { } + + /// + /// Initializes a new instance of the [] type which wraps [] instance with specified manner. + /// + /// A bytes array to be wrapped. + /// true if always should be binary; false, otherwise. + /// + /// When the is true, then resulting object represents binary even if the is valid utf-8 sequence, + /// that is, its should be []. + /// On the other hand, when contrast, the is false, and if the is valid utf-8, + /// then the resulting object can be , + /// and its should be . + /// + public MessagePackObject(Byte[] value, bool isBinary) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + if (value == null) + { + this._handleOrTypeCode = null; + } + else + { + this._handleOrTypeCode = new MessagePackString(value, isBinary); + } + } + /// + /// Initializes a new instance of the [] type which wraps [] instance with specified manner. + /// + /// A bytes array to be wrapped. + /// + /// This constructor invokes with false, that means if you pass tha bytes array which is valid utf-8, resulting object can be , + /// and its should be . + /// + public MessagePackObject(ReadOnlyMemory value) + : this(value, false) { } + + /// + /// Initializes a new instance of the [] type which wraps [] instance with specified manner. + /// + /// A bytes array to be wrapped. + /// true if always should be binary; false, otherwise. + /// + /// When the is true, then resulting object represents binary even if the is valid utf-8 sequence, + /// that is, its should be []. + /// On the other hand, when contrast, the is false, and if the is valid utf-8, + /// then the resulting object can be , + /// and its should be . + /// + public MessagePackObject(ReadOnlyMemory value, bool isBinary) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._handleOrTypeCode = new MessagePackString(value, false); + } + /// + /// Initializes a new instance of the [] type which wraps [] instance with specified manner. + /// + /// A bytes array to be wrapped. + /// + /// This constructor invokes with false, that means if you pass tha bytes array which is valid utf-8, resulting object can be , + /// and its should be . + /// + public MessagePackObject(ReadOnlyMemory value) + : this(value, false) { } + + /// + /// Initializes a new instance of the [] type which wraps [] instance with specified manner. + /// + /// A bytes array to be wrapped. + /// true if always should be binary; false, otherwise. + /// + /// When the is true, then resulting object represents binary even if the is valid utf-8 sequence, + /// that is, its should be []. + /// On the other hand, when contrast, the is false, and if the is valid utf-8, + /// then the resulting object can be , + /// and its should be . + /// + public MessagePackObject(ReadOnlyMemory value, bool isBinary) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._handleOrTypeCode = new MessagePackString(value); + } + +#pragma warning disable 0618 // obsolete + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + public MessagePackObject(MessagePackExtendedTypeObject value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = value.TypeCode; + this._handleOrTypeCode = value.Body; + } + +#pragma warning restore 0618 // obsolete + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. + public MessagePackObject(ExtensionTypeObject value) + { + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + this._value = unchecked((ulong)value.Type.Tag); + this._handleOrTypeCode = value.Body; + } + + #endregion -- Constructors -- + + #region -- Primitive Type Conversion Methods -- + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + public Boolean AsBoolean() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = this._handleOrTypeCode as ValueTypeCode; + if (typeCode == null || typeCode.TypeCode != MessagePackValueTypeCode.Boolean) + { + ThrowInvalidTypeAs(this); + } + + Debug.Assert(typeCode != null); + + return this._value != 0; + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + public Byte AsByte() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = this._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(this); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + const long minValue = (long)Byte.MinValue; + const long maxValue = (long)Byte.MaxValue; + + long asInt64 = unchecked((long)this._value); + if(asInt64 < minValue || maxValue < asInt64) + { + ThrowInvalidTypeAs(this); + } + + return unchecked((Byte)asInt64); + } + else + { + const ulong maxValue = unchecked((ulong)Byte.MaxValue); + if(maxValue < this._value) + { + ThrowInvalidTypeAs(this); + } + + return (Byte)this._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (Byte)BitConverter.Int64BitsToDouble(unchecked((long)this._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (Byte)Binary.FromBits(unchecked((int)this._value)); + } + else + { + ThrowInvalidTypeAs(this); + return default(Byte); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + [CLSCompliant(false)] + public SByte AsSByte() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = this._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(this); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + const long minValue = (long)SByte.MinValue; + const long maxValue = (long)SByte.MaxValue; + + long asInt64 = unchecked((long)this._value); + if(asInt64 < minValue || maxValue < asInt64) + { + ThrowInvalidTypeAs(this); + } + + return unchecked((SByte)asInt64); + } + else + { + const ulong maxValue = unchecked((ulong)SByte.MaxValue); + if(maxValue < this._value) + { + ThrowInvalidTypeAs(this); + } + + return (SByte)this._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (SByte)BitConverter.Int64BitsToDouble(unchecked((long)this._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (SByte)Binary.FromBits(unchecked((int)this._value)); + } + else + { + ThrowInvalidTypeAs(this); + return default(SByte); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + public Int16 AsInt16() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = this._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(this); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + const long minValue = (long)Int16.MinValue; + const long maxValue = (long)Int16.MaxValue; + + long asInt64 = unchecked((long)this._value); + if(asInt64 < minValue || maxValue < asInt64) + { + ThrowInvalidTypeAs(this); + } + + return unchecked((Int16)asInt64); + } + else + { + const ulong maxValue = unchecked((ulong)Int16.MaxValue); + if(maxValue < this._value) + { + ThrowInvalidTypeAs(this); + } + + return (Int16)this._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (Int16)BitConverter.Int64BitsToDouble(unchecked((long)this._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (Int16)Binary.FromBits(unchecked((int)this._value)); + } + else + { + ThrowInvalidTypeAs(this); + return default(Int16); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + [CLSCompliant(false)] + public UInt16 AsUInt16() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = this._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(this); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + const ulong maxValue = unchecked((ulong)UInt16.MaxValue); + + if (maxValue < this._value) + { + // Overflow or negative. + ThrowInvalidTypeAs(this); + } + + return unchecked((UInt16 )this._value); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (UInt16)BitConverter.Int64BitsToDouble(unchecked((long)this._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (UInt16)Binary.FromBits(unchecked((int)this._value)); + } + else + { + ThrowInvalidTypeAs(this); + return default(UInt16); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + public Int32 AsInt32() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = this._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(this); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + const long minValue = (long)Int32.MinValue; + const long maxValue = (long)Int32.MaxValue; + + long asInt64 = unchecked((long)this._value); + if(asInt64 < minValue || maxValue < asInt64) + { + ThrowInvalidTypeAs(this); + } + + return unchecked((Int32)asInt64); + } + else + { + const ulong maxValue = unchecked((ulong)Int32.MaxValue); + if(maxValue < this._value) + { + ThrowInvalidTypeAs(this); + } + + return (Int32)this._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (Int32)BitConverter.Int64BitsToDouble(unchecked((long)this._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (Int32)Binary.FromBits(unchecked((int)this._value)); + } + else + { + ThrowInvalidTypeAs(this); + return default(Int32); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + [CLSCompliant(false)] + public UInt32 AsUInt32() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = this._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(this); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + const ulong maxValue = unchecked((ulong)UInt32.MaxValue); + + if (maxValue < this._value) + { + // Overflow or negative. + ThrowInvalidTypeAs(this); + } + + return unchecked((UInt32 )this._value); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (UInt32)BitConverter.Int64BitsToDouble(unchecked((long)this._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (UInt32)Binary.FromBits(unchecked((int)this._value)); + } + else + { + ThrowInvalidTypeAs(this); + return default(UInt32); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + public Int64 AsInt64() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = this._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(this); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + return unchecked((long)this._value); + } + else + { + const ulong maxValue = unchecked((ulong)Int64.MaxValue); + if(maxValue < this._value) + { + ThrowInvalidTypeAs(this); + } + + return (Int64)this._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (Int64)BitConverter.Int64BitsToDouble(unchecked((long)this._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (Int64)Binary.FromBits(unchecked((int)this._value)); + } + else + { + ThrowInvalidTypeAs(this); + return default(Int64); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + [CLSCompliant(false)] + public UInt64 AsUInt64() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = this._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(this); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + if (Int64.MaxValue < this._value) + { + // Negative. + ThrowInvalidTypeAs(this); + } + } + + return this._value; + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (UInt64)BitConverter.Int64BitsToDouble(unchecked((long)this._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (UInt64)Binary.FromBits(unchecked((int)this._value)); + } + else + { + ThrowInvalidTypeAs(this); + return default(UInt64); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + public Single AsSingle() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = this._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(this); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + return (float)(unchecked((long)this._value)); + } + else + { + return (float)this._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (Single)BitConverter.Int64BitsToDouble(unchecked((long)this._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (Single)Binary.FromBits(unchecked((int)this._value)); + } + else + { + ThrowInvalidTypeAs(this); + return default(float); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + public Double AsDouble() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = this._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(this); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + return (double)(unchecked((long)this._value)); + } + else + { + return (double)this._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (Double)BitConverter.Int64BitsToDouble(unchecked((long)this._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (Double)Binary.FromBits(unchecked((int)this._value)); + } + else + { + ThrowInvalidTypeAs(this); + return default(double); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + public String? AsString() + { + VerifyUnderlyingRawType(this, null); + + if(this._handleOrTypeCode == null) + { + // nil + return null; + } + + var asString = this._handleOrTypeCode as MessagePackString; + Debug.Assert(asString != null); + return asString.GetString(); + } + + /// + /// Convert this instance to [] instance. + /// + /// [] instance corresponds to this instance. + public Byte[]? AsBinary() + { + VerifyUnderlyingRawType(this, null); + + if(this._handleOrTypeCode == null) + { + // nil + return null; + } + + var asString = this._handleOrTypeCode as MessagePackString; + Debug.Assert(asString != null); + return asString.GetByteArray(); + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + public ReadOnlyMemory AsReadOnlyMemoryOfByte() + { + if(this.IsNil) + { + ThrowCannotBeNilAs>(); + } + VerifyUnderlyingRawType>(this, null); + + var asString = this._handleOrTypeCode as MessagePackString; + Debug.Assert(asString != null); + return asString.GetByteMemory(); + } + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + public ReadOnlyMemory AsReadOnlyMemoryOfChar() + { + if(this.IsNil) + { + ThrowCannotBeNilAs>(); + } + VerifyUnderlyingRawType>(this, null); + + var asString = this._handleOrTypeCode as MessagePackString; + Debug.Assert(asString != null); + return asString.GetCharMemory(); + } + +#pragma warning disable 0618 // obsolete + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + public MessagePackExtendedTypeObject AsMessagePackExtendedTypeObject() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + VerifyUnderlyingType(this, null); + + return MessagePackExtendedTypeObject.Unpack(unchecked((byte)this._value), (this._handleOrTypeCode as byte[])!); + } + +#pragma warning restore 0618 // obsolete + + /// + /// Convert this instance to instance. + /// + /// instance corresponds to this instance. + public ExtensionTypeObject AsExtensionTypeObject() + { + if(this.IsNil) + { + ThrowCannotBeNilAs(); + } + VerifyUnderlyingType(this, null); + Debug.Assert(this._handleOrTypeCode != null); + + return new ExtensionTypeObject(new ExtensionType(unchecked((long)this._value)), (ReadOnlySequence)this._handleOrTypeCode); + } + + #endregion -- Primitive Type Conversion Methods -- + + #region -- Conversion Operator Overloads -- + + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(Boolean value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(Byte value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + [CLSCompliant(false)] + public static implicit operator MessagePackObject(SByte value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(Int16 value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + [CLSCompliant(false)] + public static implicit operator MessagePackObject(UInt16 value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(Int32 value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + [CLSCompliant(false)] + public static implicit operator MessagePackObject(UInt32 value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(Int64 value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + [CLSCompliant(false)] + public static implicit operator MessagePackObject(UInt64 value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(Single value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(Double value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(String value) + => new MessagePackObject(value); + + /// + /// Convert []instance to instance. + /// + /// [] instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(Byte[] value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(ReadOnlyMemory value) + => new MessagePackObject(value); + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(ReadOnlyMemory value) + => new MessagePackObject(value); + +#pragma warning disable 0618 // obsolete + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(MessagePackExtendedTypeObject value) + => new MessagePackObject(value); + +#pragma warning restore 0618 // obsolete + + /// + /// Convert instance to instance. + /// + /// instance. + /// instance corresponds to . + public static implicit operator MessagePackObject(ExtensionTypeObject value) + => new MessagePackObject(value); + + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + public static explicit operator Boolean(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = value._handleOrTypeCode as ValueTypeCode; + if (typeCode == null || typeCode.TypeCode != MessagePackValueTypeCode.Boolean) + { + ThrowInvalidTypeAs(value); + } + + Debug.Assert(typeCode != null); + + return value._value != 0; + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + public static explicit operator Byte(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = value._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(value); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + const long minValue = (long)Byte.MinValue; + const long maxValue = (long)Byte.MaxValue; + + long asInt64 = unchecked((long)value._value); + if(asInt64 < minValue || maxValue < asInt64) + { + ThrowInvalidTypeAs(value); + } + + return unchecked((Byte)asInt64); + } + else + { + const ulong maxValue = unchecked((ulong)Byte.MaxValue); + if(maxValue < value._value) + { + ThrowInvalidTypeAs(value); + } + + return (Byte)value._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (Byte)BitConverter.Int64BitsToDouble(unchecked((long)value._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (Byte)Binary.FromBits(unchecked((int)value._value)); + } + else + { + ThrowInvalidTypeAs(value); + return default(Byte); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + [CLSCompliant(false)] + public static explicit operator SByte(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = value._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(value); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + const long minValue = (long)SByte.MinValue; + const long maxValue = (long)SByte.MaxValue; + + long asInt64 = unchecked((long)value._value); + if(asInt64 < minValue || maxValue < asInt64) + { + ThrowInvalidTypeAs(value); + } + + return unchecked((SByte)asInt64); + } + else + { + const ulong maxValue = unchecked((ulong)SByte.MaxValue); + if(maxValue < value._value) + { + ThrowInvalidTypeAs(value); + } + + return (SByte)value._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (SByte)BitConverter.Int64BitsToDouble(unchecked((long)value._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (SByte)Binary.FromBits(unchecked((int)value._value)); + } + else + { + ThrowInvalidTypeAs(value); + return default(SByte); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + public static explicit operator Int16(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = value._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(value); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + const long minValue = (long)Int16.MinValue; + const long maxValue = (long)Int16.MaxValue; + + long asInt64 = unchecked((long)value._value); + if(asInt64 < minValue || maxValue < asInt64) + { + ThrowInvalidTypeAs(value); + } + + return unchecked((Int16)asInt64); + } + else + { + const ulong maxValue = unchecked((ulong)Int16.MaxValue); + if(maxValue < value._value) + { + ThrowInvalidTypeAs(value); + } + + return (Int16)value._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (Int16)BitConverter.Int64BitsToDouble(unchecked((long)value._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (Int16)Binary.FromBits(unchecked((int)value._value)); + } + else + { + ThrowInvalidTypeAs(value); + return default(Int16); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + [CLSCompliant(false)] + public static explicit operator UInt16(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = value._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(value); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + const ulong maxValue = unchecked((ulong)UInt16.MaxValue); + + if (maxValue < value._value) + { + // Overflow or negative. + ThrowInvalidTypeAs(value); + } + + return unchecked((UInt16 )value._value); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (UInt16)BitConverter.Int64BitsToDouble(unchecked((long)value._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (UInt16)Binary.FromBits(unchecked((int)value._value)); + } + else + { + ThrowInvalidTypeAs(value); + return default(UInt16); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + public static explicit operator Int32(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = value._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(value); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + const long minValue = (long)Int32.MinValue; + const long maxValue = (long)Int32.MaxValue; + + long asInt64 = unchecked((long)value._value); + if(asInt64 < minValue || maxValue < asInt64) + { + ThrowInvalidTypeAs(value); + } + + return unchecked((Int32)asInt64); + } + else + { + const ulong maxValue = unchecked((ulong)Int32.MaxValue); + if(maxValue < value._value) + { + ThrowInvalidTypeAs(value); + } + + return (Int32)value._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (Int32)BitConverter.Int64BitsToDouble(unchecked((long)value._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (Int32)Binary.FromBits(unchecked((int)value._value)); + } + else + { + ThrowInvalidTypeAs(value); + return default(Int32); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + [CLSCompliant(false)] + public static explicit operator UInt32(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = value._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(value); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + const ulong maxValue = unchecked((ulong)UInt32.MaxValue); + + if (maxValue < value._value) + { + // Overflow or negative. + ThrowInvalidTypeAs(value); + } + + return unchecked((UInt32 )value._value); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (UInt32)BitConverter.Int64BitsToDouble(unchecked((long)value._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (UInt32)Binary.FromBits(unchecked((int)value._value)); + } + else + { + ThrowInvalidTypeAs(value); + return default(UInt32); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + public static explicit operator Int64(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = value._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(value); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + return unchecked((long)value._value); + } + else + { + const ulong maxValue = unchecked((ulong)Int64.MaxValue); + if(maxValue < value._value) + { + ThrowInvalidTypeAs(value); + } + + return (Int64)value._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (Int64)BitConverter.Int64BitsToDouble(unchecked((long)value._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (Int64)Binary.FromBits(unchecked((int)value._value)); + } + else + { + ThrowInvalidTypeAs(value); + return default(Int64); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + [CLSCompliant(false)] + public static explicit operator UInt64(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = value._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(value); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + if (Int64.MaxValue < value._value) + { + // Negative. + ThrowInvalidTypeAs(value); + } + } + + return value._value; + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (UInt64)BitConverter.Int64BitsToDouble(unchecked((long)value._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (UInt64)Binary.FromBits(unchecked((int)value._value)); + } + else + { + ThrowInvalidTypeAs(value); + return default(UInt64); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + public static explicit operator Single(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = value._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(value); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + return (float)(unchecked((long)value._value)); + } + else + { + return (float)value._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (Single)BitConverter.Int64BitsToDouble(unchecked((long)value._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (Single)Binary.FromBits(unchecked((int)value._value)); + } + else + { + ThrowInvalidTypeAs(value); + return default(float); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + public static explicit operator Double(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + var typeCode = value._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(value); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + return (double)(unchecked((long)value._value)); + } + else + { + return (double)value._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (Double)BitConverter.Int64BitsToDouble(unchecked((long)value._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (Double)Binary.FromBits(unchecked((int)value._value)); + } + else + { + ThrowInvalidTypeAs(value); + return default(double); // Never reaches + } + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + public static explicit operator String?(MessagePackObject value) + { + VerifyUnderlyingRawType(value, "value"); + + if(value._handleOrTypeCode == null) + { + // nil + return null; + } + + var asString = value._handleOrTypeCode as MessagePackString; + Debug.Assert(asString != null); + return asString.GetString(); + } + + /// + /// Convert this instance to [] instance. + /// + /// instance. + /// [] instance corresponds to . + public static explicit operator Byte[]?(MessagePackObject value) + { + VerifyUnderlyingRawType(value, "value"); + + if(value._handleOrTypeCode == null) + { + // nil + return null; + } + + var asString = value._handleOrTypeCode as MessagePackString; + Debug.Assert(asString != null); + return asString.GetByteArray(); + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + public static explicit operator ReadOnlyMemory(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs>(); + } + VerifyUnderlyingRawType>(value, "value"); + + var asString = value._handleOrTypeCode as MessagePackString; + Debug.Assert(asString != null); + return asString.GetByteMemory(); + } + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + public static explicit operator ReadOnlyMemory(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs>(); + } + VerifyUnderlyingRawType>(value, "value"); + + var asString = value._handleOrTypeCode as MessagePackString; + Debug.Assert(asString != null); + return asString.GetCharMemory(); + } + +#pragma warning disable 0618 // obsolete + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + public static explicit operator MessagePackExtendedTypeObject(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + VerifyUnderlyingType(value, "value"); + + return MessagePackExtendedTypeObject.Unpack(unchecked((byte)value._value), (value._handleOrTypeCode as byte[])!); + } + +#pragma warning restore 0618 // obsolete + + /// + /// Convert this instance to instance. + /// + /// instance. + /// instance corresponds to . + public static explicit operator ExtensionTypeObject(MessagePackObject value) + { + if(value.IsNil) + { + ThrowCannotBeNilAs(); + } + VerifyUnderlyingType(value, "value"); + Debug.Assert(value._handleOrTypeCode != null); + + return new ExtensionTypeObject(new ExtensionType(unchecked((long)value._value)), (ReadOnlySequence)value._handleOrTypeCode); + } + + #endregion -- Conversion Operator Overloads -- + } +} diff --git a/src/MsgPack.Abstraction/MessagePackObject.tt b/src/MsgPack.Abstraction/MessagePackObject.tt new file mode 100644 index 000000000..399a58025 --- /dev/null +++ b/src/MsgPack.Abstraction/MessagePackObject.tt @@ -0,0 +1,723 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ output extension=".cs" #> +<#@ assembly Name="System.Core" #> +<#@ import namespace="System" #> +<#@ import namespace="System.Collections.Generic" #> +<#@ import namespace="System.IO" #> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +#nullable enable + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +<# +var typeName = "MessagePackObject"; +var types = + new Dictionary // value: isStringOrBinary + { + { typeof(bool), false }, + { typeof(byte), false }, + { typeof(sbyte), false }, + { typeof(short), false }, + { typeof(ushort), false }, + { typeof(int), false }, + { typeof(uint), false }, + { typeof(long), false }, + { typeof(ulong), false }, + { typeof(float), false }, + { typeof(double), false }, + { typeof(string), true }, + { typeof(byte[]), true }, + { "ReadOnlyMemory", true }, + { "ReadOnlyMemory", true }, + { "MessagePackExtendedTypeObject", false }, + { "ExtensionTypeObject", false }, + }; +Func IsNotCLSCompliant = + input => + { + var type = (input as Type) ?? typeof(object); + return + type == typeof(sbyte) + || type == typeof(ushort) + || type == typeof(uint) + || type == typeof(ulong); + }; +#> + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace MsgPack +{ + /// + /// Represents deserialized object of MsgPack. + /// + [StructLayout(LayoutKind.Auto)] + public partial struct <#= typeName #> : IEquatable<<#= typeName #>> + { + #region -- Constructors -- +<# +foreach (var type in types) +{ + var t = type.Key; + var isBinary = type.Value; + + if (t == "MessagePackExtendedTypeObject") + { +#> + +#pragma warning disable 0618 // obsolete +<# + } + + if (isBinary) + { +#> + /// + /// Initializes a new instance of the [] type which wraps [] instance with specified manner. + /// + /// A bytes array to be wrapped. + /// + /// This constructor invokes with false, that means if you pass tha bytes array which is valid utf-8, resulting object can be , + /// and its should be . + /// + public <#= typeName #>(<#= GetTypeName(t) #> value) + : this(value, false) { } + + /// + /// Initializes a new instance of the [] type which wraps [] instance with specified manner. + /// + /// A bytes array to be wrapped. + /// true if always should be binary; false, otherwise. + /// + /// When the is true, then resulting object represents binary even if the is valid utf-8 sequence, + /// that is, its should be []. + /// On the other hand, when contrast, the is false, and if the is valid utf-8, + /// then the resulting object can be , + /// and its should be . + /// + public <#= typeName #>(<#= GetTypeName(t) #> value, bool isBinary) +<# + } + else + { +#> + /// + /// Initializes a new instance of the type which wraps instance. + /// + /// A value to be wrapped. +<# + if (IsNotCLSCompliant(t)) + { +#> + [CLSCompliant(false)] +<# + } +#> + public <#= typeName #>(<#= GetTypeName(t) #> value) +<# + } +#> + { +<# + if (t.ToString() == typeof(byte[]).ToString()) + { +#> + // trick: Avoid long boilerplate initialization. + this = new MessagePackObject(); + if (value == null) + { + this._handleOrTypeCode = null; + } + else + { + this._handleOrTypeCode = new MessagePackString(value, isBinary); + } +<# + } + else + { + this.GenerateConstructor(typeName, "this", t); + } +#> + } +<# + if (t == "MessagePackExtendedTypeObject") + { +#> + +#pragma warning restore 0618 // obsolete +<# + } +} +#> + + #endregion -- Constructors -- + + #region -- Primitive Type Conversion Methods -- +<# +foreach(var t in types.Keys ) +{ + if (t == "MessagePackExtendedTypeObject") + { +#> + +#pragma warning disable 0618 // obsolete +<# + } +#> + + /// + /// Convert this instance to <#= t.ToString() == typeof( byte[] ).ToString() ? "[]" : "" #> instance. + /// + /// <#= t.ToString() == typeof( byte[] ).ToString() ? "[]" : "" #> instance corresponds to this instance. +<# +if (IsNotCLSCompliant(t)) +{ +#> + [CLSCompliant(false)] +<# +} + +var methodName = + t == typeof(byte[]) ? + "Binary" : + t == "ReadOnlyMemory" ? + "ReadOnlyMemoryOfChar" : + t == "ReadOnlyMemory" ? + "ReadOnlyMemoryOfByte" : + GetTypeName(t); +#> + public <#= GetTypeName(t) #><#= GetNullability(t) #> As<#= methodName #>() + { +<# + this.GenerateAsT("this", t, false); +#> + } +<# + if (t == "MessagePackExtendedTypeObject") + { +#> + +#pragma warning restore 0618 // obsolete +<# + } + +} +#> + + #endregion -- Primitive Type Conversion Methods -- + + #region -- Conversion Operator Overloads -- + +<# +foreach (var t in types.Keys) +{ + if (t == "MessagePackExtendedTypeObject") + { +#> + +#pragma warning disable 0618 // obsolete +<# + } +#> + + /// + /// Convert <#= t.ToString() == typeof(byte[]).ToString() ? "[]" : "" #>instance to instance. + /// + /// <#= t.ToString() == typeof(byte[]).ToString() ? "[]" : "" #> instance. + /// instance corresponds to . +<# +if (IsNotCLSCompliant(t)) +{ +#> + [CLSCompliant(false)] +<# +} +#> + public static implicit operator <#= typeName #>(<#= GetTypeName(t) #> value) + => new <#= typeName #>(value); +<# + if (t == "MessagePackExtendedTypeObject") + { +#> + +#pragma warning restore 0618 // obsolete +<# + } +} +#> + +<# +foreach (var t in types.Keys) +{ + if (t == "MessagePackExtendedTypeObject") + { +#> + +#pragma warning disable 0618 // obsolete +<# + } +#> + + /// + /// Convert this instance to <#= t.ToString() == typeof(byte[]).ToString() ? "[]" : "" #> instance. + /// + /// instance. + /// <#= t.ToString() == typeof(byte[]).ToString() ? "[]" : "" #> instance corresponds to . +<# + if (IsNotCLSCompliant(t)) + { +#> + [CLSCompliant(false)] +<# + } +#> + public static explicit operator <#= GetTypeName(t) #><#= GetNullability(t) #>(<#= typeName #> value) + { +<# + this.GenerateAsT("value", t, true); +#> + } +<# + if (t == "MessagePackExtendedTypeObject") + { +#> + +#pragma warning restore 0618 // obsolete +<# + } + +} +#> + + #endregion -- Conversion Operator Overloads -- + } +} +<#+ +private static bool IsExplicitConversionRequired(Type type) +{ + return + type == typeof(sbyte) + || type == typeof(short) + || type == typeof(int) + || type == typeof(long); +} + +private void GenerateConstructor(string typeName, string val, object typeOrTypeName) +{ + var t = typeOrTypeName as Type; + var targetTypeName = GetTypeName(typeOrTypeName); +#> + // trick: Avoid long boilerplate initialization. + <#= val #> = new <#= typeName #>(); +<#+ + if (targetTypeName == "MessagePackExtendedTypeObject") + { +#> + <#= val #>._value = value.TypeCode; + <#= val #>._handleOrTypeCode = value.Body; +<#+ + return; + } + else if (targetTypeName == "ExtensionTypeObject") + { +#> + <#= val #>._value = unchecked((ulong)value.Type.Tag); + <#= val #>._handleOrTypeCode = value.Body; +<#+ + return; + } + else if (t == typeof(byte[])) + { + // Only implicit conversion operator +#> + if (value == null) + { + <#= val #>._handleOrTypeCode = null; + } + else + { + <#= val #>._handleOrTypeCode = new MessagePackString(value, false); + } +<#+ + return; + } + else if (t == typeof(string)) + { +#> + if (value == null) + { + <#= val #>._handleOrTypeCode = null; + } + else + { + <#= val #>._handleOrTypeCode = new MessagePackString(value); + } +<#+ + return; + } + else if (targetTypeName.EndsWith("")) + { + // Only implicit conversion operator +#> + <#= val #>._handleOrTypeCode = new MessagePackString(value, false); +<#+ + return; + } + else if (targetTypeName.EndsWith("")) + { +#> + <#= val #>._handleOrTypeCode = new MessagePackString(value); +<#+ + return; + } + + if (t == typeof(bool)) + { +#> + <#= val #>._value = value ? (ulong)1 : 0; +<#+ + } + else if (IsExplicitConversionRequired(t)) + { +#> + <#= val #>._value = unchecked((ulong)value); +<#+ + } + else if (t == typeof(float)) + { +#> + <#= val #>._value = unchecked((ulong)Binary.ToBits(value)); +<#+ + } + else if (t == typeof(double)) + { +#> + <#= val #>._value = unchecked((ulong)BitConverter.DoubleToInt64Bits(value)); +<#+ + } + else + { +#> + <#= val #>._value = value; +<#+ + } + +#> + <#= val #>._handleOrTypeCode = <#= t.Name #>TypeCode; +<#+ +} // GenerateConstructor + +private void GenerateAsT(string val, Object typeOrTypeName, bool passParameterName) +{ + var t = typeOrTypeName as Type; + var typeName = GetTypeName(typeOrTypeName); + + if(t == null || t.IsValueType) + { +#> + if(<#= val #>.IsNil) + { + ThrowCannotBeNilAs<#= "<" + typeName + ">" #>(); + } +<#+ + } + + if (typeName == "MessagePackExtendedTypeObject") + { +#> + VerifyUnderlyingType(<#= val #>, <#= passParameterName ? "\"" + val + "\"" : "null" #>); + + return MessagePackExtendedTypeObject.Unpack(unchecked((byte)<#= val #>._value), (<#= val #>._handleOrTypeCode as byte[])!); +<#+ + } + else if (typeName == "ExtensionTypeObject") + { +#> + VerifyUnderlyingType(<#= val #>, <#= passParameterName ? "\"" + val + "\"" : "null" #>); + Debug.Assert(<#= val #>._handleOrTypeCode != null); + + return new ExtensionTypeObject(new ExtensionType(unchecked((long)<#= val #>._value)), (ReadOnlySequence)<#= val #>._handleOrTypeCode); +<#+ + } + else if (t == typeof(byte[])) + { +#> + VerifyUnderlyingRawType(<#= val #>, <#= passParameterName ? "\"" + val + "\"" : "null" #>); + + if(<#= val #>._handleOrTypeCode == null) + { + // nil + return null; + } + + var asString = <#= val #>._handleOrTypeCode as MessagePackString; + Debug.Assert(asString != null); + return asString.GetByteArray(); +<#+ + } + else if (t == typeof(string)) + { +#> + VerifyUnderlyingRawType(<#= val #>, <#= passParameterName ? "\"" + val + "\"" : "null" #>); + + if(<#= val #>._handleOrTypeCode == null) + { + // nil + return null; + } + + var asString = <#= val #>._handleOrTypeCode as MessagePackString; + Debug.Assert(asString != null); + return asString.GetString(); +<#+ + } + else if (typeName.EndsWith("")) + { +#> + VerifyUnderlyingRawType>(<#= val #>, <#= passParameterName ? "\"" + val + "\"" : "null" #>); + + var asString = <#= val #>._handleOrTypeCode as MessagePackString; + Debug.Assert(asString != null); + return asString.GetByteMemory(); +<#+ + } + else if (typeName.EndsWith("")) + { +#> + VerifyUnderlyingRawType>(<#= val #>, <#= passParameterName ? "\"" + val + "\"" : "null" #>); + + var asString = <#= val #>._handleOrTypeCode as MessagePackString; + Debug.Assert(asString != null); + return asString.GetCharMemory(); +<#+ + } + else if (t == typeof(bool)) + { +#> + var typeCode = <#= val #>._handleOrTypeCode as ValueTypeCode; + if (typeCode == null || typeCode.TypeCode != MessagePackValueTypeCode.Boolean) + { + ThrowInvalidTypeAs(<#= val #>); + } + + Debug.Assert(typeCode != null); + + return <#= val #>._value != 0; +<#+ + } + else if (t == typeof(float)) + { +#> + var typeCode = <#= val #>._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(<#= val #>); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + return (float)(unchecked((long)<#= val #>._value)); + } + else + { + return (float)<#= val #>._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (<#= t.Name #>)BitConverter.Int64BitsToDouble(unchecked((long)<#= val #>._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (<#= t.Name #>)Binary.FromBits(unchecked((int)<#= val #>._value)); + } + else + { + ThrowInvalidTypeAs(<#= val #>); + return default(float); // Never reaches + } +<#+ + } + else if (t == typeof(double)) + { +#> + var typeCode = <#= val #>._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs(<#= val #>); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { + return (double)(unchecked((long)<#= val #>._value)); + } + else + { + return (double)<#= val #>._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (<#= t.Name #>)BitConverter.Int64BitsToDouble(unchecked((long)<#= val #>._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (<#= t.Name #>)Binary.FromBits(unchecked((int)<#= val #>._value)); + } + else + { + ThrowInvalidTypeAs(<#= val #>); + return default(double); // Never reaches + } +<#+ + } + else if (t.Name[0] == 'U') + { +#> + var typeCode = <#= val #>._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs<#= "<" + t.Name + ">" #>(<#= val #>); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { +<#+ + if (t == typeof(UInt64)) + { +#> + if (typeCode.IsSigned) + { + if (Int64.MaxValue < <#= val #>._value) + { + // Negative. + ThrowInvalidTypeAs<#= "<" + t.Name + ">" #>(<#= val #>); + } + } + + return <#= val #>._value; +<#+ + } + else + { +#> + const ulong maxValue = unchecked((ulong)<#= t.Name #>.MaxValue); + + if (maxValue < <#= val #>._value) + { + // Overflow or negative. + ThrowInvalidTypeAs<#= "<" + t.Name + ">" #>(<#= val #>); + } + + return unchecked((<#= t.Name #> )<#= val #>._value); +<#+ + } +#> + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (<#= t.Name #>)BitConverter.Int64BitsToDouble(unchecked((long)<#= val #>._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (<#= t.Name #>)Binary.FromBits(unchecked((int)<#= val #>._value)); + } + else + { + ThrowInvalidTypeAs<#= "<" + t.Name + ">" #>(<#= val #>); + return default(<#= t.Name #>); // Never reaches + } +<#+ + } + else + { +#> + var typeCode = <#= val #>._handleOrTypeCode as ValueTypeCode; + if (typeCode == null) + { + ThrowInvalidTypeAs<#= "<" + t.Name + ">" #>(<#= val #>); + } + + Debug.Assert(typeCode != null); + + if (typeCode.IsInteger) + { + if (typeCode.IsSigned) + { +<#+ + if (t == typeof(Int64) ) + { +#> + return unchecked((long)<#= val #>._value); +<#+ + } + else + { +#> + const long minValue = (long)<#= t.Name #>.MinValue; + const long maxValue = (long)<#= t.Name #>.MaxValue; + + long asInt64 = unchecked((long)<#= val #>._value); + if(asInt64 < minValue || maxValue < asInt64) + { + ThrowInvalidTypeAs<#= "<" + t.Name + ">" #>(<#= val #>); + } + + return unchecked((<#= t.Name #>)asInt64); +<#+ + } +#> + } + else + { + const ulong maxValue = unchecked((ulong)<#= t.Name #>.MaxValue); + if(maxValue < <#= val #>._value) + { + ThrowInvalidTypeAs<#= "<" + t.Name + ">" #>(<#= val #>); + } + + return (<#= t.Name #>)<#= val #>._value; + } + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Double) + { + return (<#= t.Name #>)BitConverter.Int64BitsToDouble(unchecked((long)<#= val #>._value)); + } + else if (typeCode.TypeCode == MessagePackValueTypeCode.Single) + { + return (<#= t.Name #>)Binary.FromBits(unchecked((int)<#= val #>._value)); + } + else + { + ThrowInvalidTypeAs<#= "<" + t.Name + ">" #>(<#= val #>); + return default(<#= t.Name #>); // Never reaches + } +<#+ + } +} + +private static string GetTypeName(object typeOrTypeName) + => (typeOrTypeName as string) ?? (typeOrTypeName as Type).Name; + +private static string GetNullability(object typeOrTypeName) + => (typeOrTypeName == typeof(byte[]) || typeOrTypeName == typeof(string)) ? "?" : String.Empty; +#> diff --git a/src/MsgPack.Abstraction/MsgPack.Abstraction.csproj b/src/MsgPack.Abstraction/MsgPack.Abstraction.csproj new file mode 100644 index 000000000..b8660a931 --- /dev/null +++ b/src/MsgPack.Abstraction/MsgPack.Abstraction.csproj @@ -0,0 +1,124 @@ + + + + netcoreapp3.1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + TextTemplatingFileGenerator + FormatDecoder.Strings.cs + + + FormatEncoder.Primitives.cs + TextTemplatingFileGenerator + + + TextTemplatingFileGenerator + FormatDecoder.Primitives.cs + + + AsyncSerializationOperationContext.cs + TextTemplatingFileGenerator + + + AsyncDeserializationOperationContext.cs + TextTemplatingFileGenerator + + + DeserializationOperationContext.cs + TextTemplatingFileGenerator + + + SerializationOperationContext.cs + + TextTemplatingFileGenerator + MessagePackObject.cs + + + + + + + + + + True + True + FormatDecoder.Primitives.tt + + + True + True + FormatDecoder.Strings.tt + + + True + True + FormatEncoder.Primitives.tt + + + True + True + AsyncDeserializationOperationContext.tt + + + True + True + AsyncSerializationOperationContext.tt + + + True + True + DeserializationOperationContext.tt + + + True + True + SerializationOperationContext.tt + + + True + True + MessagePackObject.tt + + + + diff --git a/src/MsgPack.Abstraction/Properties/AssemblyInfo.cs b/src/MsgPack.Abstraction/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..430590546 --- /dev/null +++ b/src/MsgPack.Abstraction/Properties/AssemblyInfo.cs @@ -0,0 +1,8 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("MsgPack.Core")] +[assembly: InternalsVisibleTo("MsgPack.Json")] diff --git a/src/MsgPack.Abstraction/Serialization/AvailableSerializationMethods.cs b/src/MsgPack.Abstraction/Serialization/AvailableSerializationMethods.cs new file mode 100644 index 000000000..e0efe48c3 --- /dev/null +++ b/src/MsgPack.Abstraction/Serialization/AvailableSerializationMethods.cs @@ -0,0 +1,16 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; + +namespace MsgPack.Serialization +{ + [Flags] + public enum AvailableSerializationMethods + { + None = 0, + Array = 0x1, + Map = 0x2 + } +} diff --git a/src/MsgPack.Abstraction/StringEscape.cs b/src/MsgPack.Abstraction/StringEscape.cs new file mode 100644 index 000000000..8e1dc7b4b --- /dev/null +++ b/src/MsgPack.Abstraction/StringEscape.cs @@ -0,0 +1,165 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +#warning TODO: Move to Common/ + +using System; +using System.Buffers; +using System.Globalization; +using System.Text; + +namespace MsgPack +{ + internal static class StringEscape + { + public static string ForDisplay(string value) + { + if (value == null) + { + return String.Empty; + } + + var buffer = new StringBuilder(); + foreach (Rune r in value.EnumerateRunes()) + { + if (buffer.Length > 64) + { + buffer.Append("..."); + break; + } + + switch (r.Value) + { + case '\0': + { + buffer.Append("\\0"); + continue; + } + case '\t': + { + buffer.Append("\\t"); + continue; + } + case '\r': + { + buffer.Append("\\r"); + continue; + } + case '\n': + { + buffer.Append("\\n"); + continue; + } + case '\a': + { + buffer.Append("\\a"); + continue; + } + case '\b': + { + buffer.Append("\\b"); + continue; + } + case '\f': + { + buffer.Append("\\f"); + continue; + } + case '\v': + { + buffer.Append("\\v"); + continue; + } + case '\\': + { + buffer.Append("\\\\"); + continue; + } + case ' ': + { + buffer.Append(' '); + continue; + } + } + + switch (Rune.GetUnicodeCategory(r)) + { + case UnicodeCategory.Control: + case UnicodeCategory.Format: + case UnicodeCategory.LineSeparator: + case UnicodeCategory.ModifierLetter: + case UnicodeCategory.ModifierSymbol: + case UnicodeCategory.NonSpacingMark: + case UnicodeCategory.OtherNotAssigned: + case UnicodeCategory.ParagraphSeparator: + case UnicodeCategory.SpacingCombiningMark: + case UnicodeCategory.Surrogate: + case UnicodeCategory.PrivateUse: + { + if (r.Value == ' ') + { + goto default; + } + + buffer.Append("\\u").Append((r.Value).ToString("X4", CultureInfo.InvariantCulture)); + continue; + } + default: + { + Span c = stackalloc char[2]; + r.TryEncodeToUtf16(c, out var charsWritten); + buffer.Append(c.Slice(0, charsWritten)); + continue; + } + } + } + + return buffer.ToString(); + } + + public static string Stringify(in ReadOnlySequence unit) + { + var buffer = new StringBuilder(); + foreach (var rune in Encoding.UTF8.GetString(unit.Slice(0, 260).ToArray()).EnumerateRunes()) + { + if (buffer.Length > 64) + { + buffer.Append("..."); + break; + } + + var category = Rune.GetUnicodeCategory(rune); + switch (category) + { + case UnicodeCategory.Control: + case UnicodeCategory.Format: + case UnicodeCategory.LineSeparator: + case UnicodeCategory.ModifierLetter: + case UnicodeCategory.ModifierSymbol: + case UnicodeCategory.NonSpacingMark: + case UnicodeCategory.OtherNotAssigned: + case UnicodeCategory.ParagraphSeparator: + case UnicodeCategory.SpacingCombiningMark: + case UnicodeCategory.Surrogate: + { + buffer.Append($"(U+{rune.Value:X4}, {category})"); + break; + } + case UnicodeCategory.SpaceSeparator: + { + buffer.Append($"'{rune}'(U+{rune.Value:X4}, {category})"); + break; + } + default: + { + buffer.Append('\'').Append(rune).Append('\''); + break; + } + } + } + + return buffer.ToString(); + } + } +} diff --git a/src/MsgPack.Abstraction/System.Text/StringBuilderCache.cs b/src/MsgPack.Abstraction/System.Text/StringBuilderCache.cs new file mode 100644 index 000000000..bf50a5cee --- /dev/null +++ b/src/MsgPack.Abstraction/System.Text/StringBuilderCache.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#nullable enable +namespace System.Text +{ + /// Provide a cached reusable instance of stringbuilder per thread. + internal static class StringBuilderCache + { + // The value 360 was chosen in discussion with performance experts as a compromise between using + // as litle memory per thread as possible and still covering a large part of short-lived + // StringBuilder creations on the startup path of VS designers. + internal const int MaxBuilderSize = 360; + private const int DefaultCapacity = 16; // == StringBuilder.DefaultCapacity + + // WARNING: We allow diagnostic tools to directly inspect this member (t_cachedInstance). + // See https://github.com/dotnet/corert/blob/master/Documentation/design-docs/diagnostics/diagnostics-tools-contract.md for more details. + // Please do not change the type, the name, or the semantic usage of this member without understanding the implication for tools. + // Get in touch with the diagnostics team if you have questions. + [ThreadStatic] + private static StringBuilder? t_cachedInstance; + + /// Get a StringBuilder for the specified capacity. + /// If a StringBuilder of an appropriate size is cached, it will be returned and the cache emptied. + public static StringBuilder Acquire(int capacity = DefaultCapacity) + { + if (capacity <= MaxBuilderSize) + { + StringBuilder? sb = t_cachedInstance; + if (sb != null) + { + // Avoid stringbuilder block fragmentation by getting a new StringBuilder + // when the requested size is larger than the current capacity + if (capacity <= sb.Capacity) + { + t_cachedInstance = null; + sb.Clear(); + return sb; + } + } + } + + return new StringBuilder(capacity); + } + + /// Place the specified builder in the cache if it is not too big. + public static void Release(StringBuilder sb) + { + if (sb.Capacity <= MaxBuilderSize) + { + t_cachedInstance = sb; + } + } + + /// ToString() the stringbuilder, Release it to the cache, and return the resulting string. + public static string GetStringAndRelease(StringBuilder sb) + { + string result = sb.ToString(); + Release(sb); + return result; + } + } +} diff --git a/src/MsgPack.Abstraction/Throw.cs b/src/MsgPack.Abstraction/Throw.cs new file mode 100644 index 000000000..d450d8583 --- /dev/null +++ b/src/MsgPack.Abstraction/Throw.cs @@ -0,0 +1,203 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Runtime.CompilerServices; +using System.Text; +using MsgPack.Internal; + +namespace MsgPack +{ + internal static class Throw + { + public static void ObjectDisposed(string? name) + => throw new ObjectDisposedException(name); + public static char? InvalidIso8601DecimalSeparator(char? value, [CallerArgumentExpression("value")] string paramName = null!) + => throw new ArgumentOutOfRangeException(paramName, $"Value '{StringEscape.ForDisplay(value.GetValueOrDefault().ToString())}' is not valid ISO 8601 decimal separator."); + + public static void CannotBeNilAs() + => throw new InvalidOperationException($"Nil value cannot be converted to '{typeof(T)}' value."); + + public static void InvalidTypeAs(MessagePackObject obj) + => throw new InvalidOperationException($"'{obj}' cannot be converted to '{typeof(T)}' value."); + + public static void ExtensionsIsNotSupported() + => throw new NotSupportedException($"Extension type is not supported in this encoder."); + + public static void StreamMustBeAbleToRead(string paramName) + => throw new ArgumentException($"The stream must be able to read.", paramName); + + public static void TooSmallBuffer(string paramName, int minimumInclusive) + => throw new ArgumentException($"The size of buffer must be greater than or equal to {minimumInclusive:#,0}.", paramName); + + public static void TooLargeLength(int length, int requestHint) + => throw new ArgumentException($"Requested buffer size {((long)length + requestHint):#,0} is too large. It must be less than or equal to {OptionsDefaults.MaxSingleByteCollectionLength:#,0} bytes."); + + public static void EmptyObject(Type type) + => throw new InvalidOperationException($"Cannot use empty '{type}' object."); + + public static void DepthExeeded(long position, int maxDepth) + => throw new LimitExceededException(position, $"The depth of collection exceeds max depth {maxDepth:#,0}."); + + public static void StringLengthExceeded(long position, long length, int maxLength) + => throw new LimitExceededException(position, $"The byte length of encoded string ({length:#,0}) exceeds max length ({maxLength:#,0})."); + + public static void BinaryLengthExceeded(long position, long length, int maxLength) + => throw new LimitExceededException(position, $"The byte length of encoded binary ({length:#,0}) exceeds max length ({maxLength:#,0})."); + + public static void DepthUnderflow() + => throw new InvalidOperationException("CurrentDepth is 0."); + + public static void TooLargeByteLength(long size, string encodingName) + => throw new InvalidOperationException($"Input ReadOnlySequence is too large. It will be encoded to {size:#,0} bytes with '{encodingName}' encoding, but it must be less than or equal to {UInt32.MaxValue:#,0} bytes."); + + public static void TooLargeByteLength(Exception innerException, string encodingName) + => throw new InvalidOperationException($"Input ReadOnlySequence is too large. It will be encoded to larger than {Int32.MaxValue:#,0} bytes with '{encodingName}' encoding, but it must be less than or equal to {UInt32.MaxValue:#,0} bytes.", innerException); + + public static void TooLargePropertyKey(long position, int length, int maxPropertyKeyLength) + => throw new LimitExceededException(position, $"Property key is too large. The size {length:#,0} is larger than configured limit {maxPropertyKeyLength:#,0}."); + + public static void InsufficientInput(long position, Type targetType, int requestHint) + { + if (requestHint < 0) + { + throw new InsufficientInputException(position, $"It is required more bytes in input ReadOnlySequence to decode {targetType} value."); + } + else + { + throw new InsufficientInputException(position, $"It is required more {requestHint:#,0} bytes in input ReadOnlySequence to decode {targetType} value."); + } + } + + public static void InsufficientInputForAnyItem(long position, int requestHint) + { + if (requestHint < 0) + { + throw new InsufficientInputException(position, $"It is required more bytes in input ReadOnlySequence to decode any value."); + } + else + { + throw new InsufficientInputException(position, $"It is required more {requestHint:#,0} bytes in input ReadOnlySequence to decode any value."); + } + } + + public static void InsufficientInputForNull(long position, int requestHint) + { + if (requestHint < 0) + { + throw new InsufficientInputException(position, $"It is required more bytes in input ReadOnlySequence to decode null."); + } + else + { + throw new InsufficientInputException(position, $"It is required more {requestHint:#,0} bytes in input ReadOnlySequence to decode null."); + } + } + + + public static void InsufficientInputForDecodeArrayOrMapHeader(long position, int requestHint) + { + if (requestHint < 0) + { + throw new InsufficientInputException(position, $"It is required more bytes in input ReadOnlySequence to decode array or map header."); + } + else + { + throw new InsufficientInputException(position, $"It is required more {requestHint:#,0} bytes in input ReadOnlySequence to decode array or map header."); + } + } + + public static void InsufficientInputForDecodeArrayHeader(long position, int requestHint) + { + if (requestHint < 0) + { + throw new InsufficientInputException(position, $"It is required more bytes in input ReadOnlySequence to decode array header."); + } + else + { + throw new InsufficientInputException(position, $"It is required more {requestHint:#,0} bytes in input ReadOnlySequence to decode array header."); + } + } + + public static void InsufficientInputForDecodeMapHeader(long position, int requestHint) + { + if (requestHint < 0) + { + throw new InsufficientInputException(position, $"It is required more bytes in input ReadOnlySequence to decode map header."); + } + else + { + throw new InsufficientInputException(position, $"It is required more {requestHint:#,0} bytes in input ReadOnlySequence to decode map header."); + } + } + + public static void InsufficientInputForString(long position, Type type, Encoding? encoding, int requestHint) + { + if (requestHint < 0) + { + throw new InsufficientInputException(position, $"It is required more bytes in input ReadOnlySequence to decode string as {type} with '{encoding}' encoding."); + } + else + { + throw new InsufficientInputException(position, $"It is required more {requestHint:#,0} bytes in input ReadOnlySequence to decode string as {type} with '{encoding}' encoding."); + } + } + + public static void InsufficientInputForRawString(long position, int requestHint) + { + if (requestHint < 0) + { + throw new InsufficientInputException(position, $"It is required more bytes in input ReadOnlySequence to fetch raw string."); + } + else + { + throw new InsufficientInputException(position, $"It is required more {requestHint:#,0} bytes in input ReadOnlySequence to fetch raw string."); + } + } + + public static void InsufficientInputForSkip(long position, int requestHint) + { + if (requestHint < 0) + { + throw new InsufficientInputException(position, $"It is required more bytes in input ReadOnlySequence to skip current subtree."); + } + else + { + throw new InsufficientInputException(position, $"It is required more {requestHint:#,0} bytes in input ReadOnlySequence to skip current subtree."); + } + } + + public static void InsufficientInputForDetectCollectionEnds(long position, int requestHint) + { + if (requestHint < 0) + { + throw new InsufficientInputException(position, $"It is required more bytes in input ReadOnlySequence to detect whether current collection is end."); + } + else + { + throw new InsufficientInputException(position, $"It is required more {requestHint:#,0} bytes in input ReadOnlySequence to detect whether current collection is end."); + } + } + + public static void InsufficientInputForDrainCollectionItems(long position, int requestHint) + { + if (requestHint < 0) + { + throw new InsufficientInputException(position, $"It is required more bytes in input ReadOnlySequence to drain current collection items."); + } + else + { + throw new InsufficientInputException(position, $"It is required more {requestHint:#,0} bytes in input ReadOnlySequence to current collection items."); + } + } + + public static void InvalidTimestampLength(long length, string paramName) + => throw new ArgumentException($"The value's length {length} bytes is not valid. It should be 4, 8, or 12 bytes.", paramName); + + public static void InvalidTimestampTypeCode(byte tag, string paramName) + => throw new ArgumentException($"Timestamp must have type code -1 (0xFF in byte), but actual value is {unchecked((sbyte)tag)}(0x{tag:x}).", paramName); + + public static void InvalidTimestampTypeCode(long tag, string paramName) + => throw new ArgumentException($"Timestamp must have type code -1 (0xFF in byte), but actual value is {tag}(0x{tag:x}).", paramName); + } +} diff --git a/src/MsgPack.Compatibility/MsgPack.Compatibility.csproj b/src/MsgPack.Compatibility/MsgPack.Compatibility.csproj new file mode 100644 index 000000000..cb6319069 --- /dev/null +++ b/src/MsgPack.Compatibility/MsgPack.Compatibility.csproj @@ -0,0 +1,7 @@ + + + + netcoreapp3.1 + + + diff --git a/src/MsgPack.Core/Codecs/MessagePackCodecProvider.cs b/src/MsgPack.Core/Codecs/MessagePackCodecProvider.cs new file mode 100644 index 000000000..073430a39 --- /dev/null +++ b/src/MsgPack.Core/Codecs/MessagePackCodecProvider.cs @@ -0,0 +1,39 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using MsgPack.Internal; + +namespace MsgPack.Codecs +{ + public sealed class MessagePackCodecProvider : CodecProvider + { + public static MessagePackCodecProvider Default { get; } = new MessagePackCodecProvider(MessagePackEncoderOptions.Default, MessagePackDecoderOptions.Default); + + private readonly MessagePackEncoder _encoder; + private readonly MessagePackDecoder _decoder; + + public MessagePackCodecProvider( + MessagePackEncoderOptions encoderOptions, + MessagePackDecoderOptions decoderOptions, + MessagePackCompatibilityLevel compatibilityLevel = MessagePackCompatibilityLevel.Latest + ) + { + if (compatibilityLevel == MessagePackCompatibilityLevel.Version2008) + { + this._encoder = new LegacyMessagePackEncoder(encoderOptions); + } + else + { + this._encoder = new CurrentMessagePackEncoder(encoderOptions); + } + + this._decoder = new MessagePackDecoder(decoderOptions); + } + + public sealed override FormatEncoder GetEncoder() => this._encoder; + + public sealed override FormatDecoder GetDecoder() => this._decoder; + + } +} diff --git a/src/MsgPack.Core/Internal/CurrentMessagePackEncoder.cs b/src/MsgPack.Core/Internal/CurrentMessagePackEncoder.cs new file mode 100644 index 000000000..d95d12eb1 --- /dev/null +++ b/src/MsgPack.Core/Internal/CurrentMessagePackEncoder.cs @@ -0,0 +1,177 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + /// + /// MessagePack encoder. + /// + internal sealed class CurrentMessagePackEncoder : MessagePackEncoder + { + private static readonly CurrentMessagePackEncoder DefaultInstance = new CurrentMessagePackEncoder(MessagePackEncoderOptions.Default); + + public CurrentMessagePackEncoder(MessagePackEncoderOptions options) + : base(options) { } + + internal static byte[] InternalEncodeString(string value) + { + var arrayBuffer = new ArrayBufferWriter(); + DefaultInstance.EncodeString(value, arrayBuffer); + return arrayBuffer.WrittenMemory.ToArray(); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + protected sealed override int EncodeStringHeader(uint length, Span buffer) + { + if (length < 32) + { + buffer[0] = unchecked((byte)(MessagePackCode.MinimumFixedRaw | length)); + return 1; + } + else if (length <= Byte.MaxValue) + { + buffer[0] = MessagePackCode.Str8; + buffer[1] = unchecked((byte)length); + return 2; + } + else if (length <= UInt16.MaxValue) + { + buffer[0] = MessagePackCode.Str16; + BinaryPrimitives.WriteUInt16BigEndian(buffer.Slice(1), unchecked((ushort)length)); + return sizeof(ushort) + 1; + } + else + { + buffer[0] = MessagePackCode.Str32; + BinaryPrimitives.WriteUInt32BigEndian(buffer.Slice(1), length); + return sizeof(uint) + 1; + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + protected sealed override int EncodeBinaryHeader(uint length, Span buffer) + { + if (length <= Byte.MaxValue) + { + buffer[0] = MessagePackCode.Bin8; + buffer[1] = unchecked((byte)length); + return 2; + } + else if (length <= UInt16.MaxValue) + { + buffer[0] = MessagePackCode.Bin16; + BinaryPrimitives.WriteUInt16BigEndian(buffer.Slice(1), unchecked((ushort)length)); + return sizeof(ushort) + 1; + } + else + { + buffer[0] = MessagePackCode.Bin32; + BinaryPrimitives.WriteUInt32BigEndian(buffer.Slice(1), length); + return sizeof(uint) + 1; + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeExtension(ExtensionType typeCode, ReadOnlySpan serializedValue, IBufferWriter buffer) + { + EncodeExtensionHeader(typeCode, unchecked((uint)serializedValue.Length), buffer); + buffer.Write(serializedValue); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeExtension(ExtensionType typeCode, in ReadOnlySequence serializedValue, IBufferWriter buffer) + { + EncodeExtensionHeader(typeCode, unchecked((uint)serializedValue.Length), buffer); + this.WriteRaw(serializedValue, buffer); + } + + private static void EncodeExtensionHeader(ExtensionType typeCode, uint serializedValueLength, IBufferWriter buffer) + { + if (typeCode.Tag > Byte.MaxValue) + { + MessagePackThrow.InvalidTypeCode(typeCode.Tag); + } + + switch (serializedValueLength) + { + case 1: + { + var span = buffer.GetSpan(1); + span[0] = MessagePackCode.FixExt1; + buffer.Advance(1); + break; + } + case 2: + { + var span = buffer.GetSpan(1); + span[0] = MessagePackCode.FixExt2; + buffer.Advance(1); + break; + } + case 4: + { + var span = buffer.GetSpan(1); + span[0] = MessagePackCode.FixExt4; + buffer.Advance(1); + break; + } + case 8: + { + var span = buffer.GetSpan(1); + span[0] = MessagePackCode.FixExt8; + buffer.Advance(1); + break; + } + case 16: + { + var span = buffer.GetSpan(1); + span[0] = MessagePackCode.FixExt16; + buffer.Advance(1); + break; + } + default: + { + if (serializedValueLength <= Byte.MaxValue) + { + var span = buffer.GetSpan(2); + span[0] = MessagePackCode.Ext8; + span[1] = unchecked((byte)serializedValueLength); + buffer.Advance(2); + } + else if (serializedValueLength <= UInt16.MaxValue) + { + var span = buffer.GetSpan(sizeof(ushort) + 1); + span[0] = MessagePackCode.Ext16; + span = span.Slice(1); + BinaryPrimitives.WriteUInt16BigEndian(span, unchecked((ushort)serializedValueLength)); + buffer.Advance(sizeof(ushort) + 1); + } + else + { + var span = buffer.GetSpan(sizeof(uint) + 1); + span[0] = MessagePackCode.Ext32; + span = span.Slice(1); + BinaryPrimitives.WriteUInt32BigEndian(span, serializedValueLength); + buffer.Advance(sizeof(uint) + 1); + } + + break; + } + } // switch + + // type code + { + var span = buffer.GetSpan(1); + span[0] = unchecked((byte)typeCode.Tag); + buffer.Advance(1); + } + } + + } +} diff --git a/src/MsgPack.Core/Internal/LegacyMessagePackEncoder.cs b/src/MsgPack.Core/Internal/LegacyMessagePackEncoder.cs new file mode 100644 index 000000000..eb5f16bc0 --- /dev/null +++ b/src/MsgPack.Core/Internal/LegacyMessagePackEncoder.cs @@ -0,0 +1,55 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + /// + /// MessagePack encoder for legacy format which uses raw type instead of str type and does not support ext type. + /// + internal sealed class LegacyMessagePackEncoder : MessagePackEncoder + { + private static readonly LegacyMessagePackEncoder DefaultInstance = new LegacyMessagePackEncoder(MessagePackEncoderOptions.Default); + + public LegacyMessagePackEncoder(MessagePackEncoderOptions options) + : base(options) { } + + internal static byte[] InternalEncodeString(string value) + { + var arrayBuffer = new ArrayBufferWriter(); + DefaultInstance.EncodeString(value, arrayBuffer); + return arrayBuffer.WrittenMemory.ToArray(); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + protected sealed override int EncodeStringHeader(uint length, Span buffer) + { + if (length < 32) + { + buffer[0] = unchecked((byte)(MessagePackCode.MinimumFixedRaw | length)); + return 1; + } + else if (length <= UInt16.MaxValue) + { + buffer[0] = MessagePackCode.Raw16; + BinaryPrimitives.WriteUInt16BigEndian(buffer, unchecked((ushort)length)); + return sizeof(ushort) + 1; + } + else + { + buffer[0] = MessagePackCode.Raw32; + BinaryPrimitives.WriteUInt32BigEndian(buffer, length); + return sizeof(uint) + 1; + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + protected sealed override int EncodeBinaryHeader(uint length, Span buffer) + => this.EncodeStringHeader(length, buffer); + } +} diff --git a/src/MsgPack.Core/Internal/MemoryExtensions.cs b/src/MsgPack.Core/Internal/MemoryExtensions.cs new file mode 100644 index 000000000..02c2e72dc --- /dev/null +++ b/src/MsgPack.Core/Internal/MemoryExtensions.cs @@ -0,0 +1,34 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + internal static class MemoryExtensions + { + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static void Convert(this System.Text.Encoder encoder, ReadOnlyMemory source, Memory sink, bool flush, out int charsUsed, out int bytesUsed, out bool completed) + => encoder.Convert(source.Span, sink.Span, flush, out charsUsed, out bytesUsed, out completed); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static unsafe Memory Compact(this Memory memory, T[] array, int used) + where T : unmanaged + { + var span = memory.Span.Slice(used); + if (span.IsEmpty) + { + return array; + } + + fixed (T* pSrc = span) + fixed (T* pDest = array) + { + Buffer.MemoryCopy(pSrc, pDest, array.Length, span.Length); + return array.AsMemory(0, span.Length); + } + } + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.CollectionHeaders.cs b/src/MsgPack.Core/Internal/MessagePackDecoder.CollectionHeaders.cs new file mode 100644 index 000000000..7b2436854 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.CollectionHeaders.cs @@ -0,0 +1,165 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache 2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; + +namespace MsgPack.Internal +{ + public partial class MessagePackDecoder + { + public sealed override CollectionType DecodeArrayOrMapHeader(ref SequenceReader source, out int itemsCount, out int requestHint) + { + var startOffset = source.Consumed; + var result = this.PrivateDecodeArrayOrMapHeader(ref source, out var header, out itemsCount, out requestHint); + + if (itemsCount > Int32.MaxValue) + { + MessagePackThrow.TooLargeArrayOrMapLength(header, startOffset, itemsCount); + } + + return result; + } + + private CollectionType PrivateDecodeArrayOrMapHeader(ref SequenceReader source, out byte header, out int itemsCount, out int requestHint) + { + var startOffset = source.Consumed; + var result = this.DecodeArrayOrMapHeaderCore(ref source, out header, out itemsCount, out requestHint); + if (result == CollectionType.None) + { + MessagePackThrow.TypeIsNotArrayNorMap(header, startOffset); + } + + return result; + } + + private CollectionType DecodeArrayOrMapHeaderCore(ref SequenceReader source, out byte header, out int itemsCount, out int requestHint) + { + if (!source.TryRead(out header)) + { + requestHint = 1; + itemsCount = 0; + return default; + } + + requestHint = 0; + + if (header == MessagePackCode.NilValue) + { + itemsCount = 0; + return CollectionType.Null; + } + + if ((header & MessagePackCode.MinimumFixedArray) == MessagePackCode.MinimumFixedArray) + { + itemsCount = (int)header - MessagePackCode.MinimumFixedArray; + return CollectionType.Array; + } + + if ((header & MessagePackCode.MinimumFixedMap) == MessagePackCode.MinimumFixedMap) + { + itemsCount = (int)header - MessagePackCode.MinimumFixedMap; + return CollectionType.Map; + } + + int lengthOfLength; + CollectionType type; + switch (header) + { + case MessagePackCode.Array16: + { + lengthOfLength = 2; + type = CollectionType.Array; + break; + } + case MessagePackCode.Array32: + { + lengthOfLength = 4; + type = CollectionType.Array; + break; + } + case MessagePackCode.Map16: + { + lengthOfLength = 2; + type = CollectionType.Map; + break; + } + case MessagePackCode.Map32: + { + lengthOfLength = 4; + type = CollectionType.Map; + break; + } + default: + { + requestHint = 0; + itemsCount = 0; + return default; + } + } + + switch (lengthOfLength) + { + case 2: + { + itemsCount = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: // 4 + { + var count = ReadValue(ref source, offset: 1, out requestHint); + if (count > OptionsDefaults.MaxMultiByteCollectionLength) + { + MessagePackThrow.TooLargeArrayOrMapLength(header, source.Consumed - 1, count); + } + + itemsCount = unchecked((int)count); + break; + } + } + + if (requestHint != 0) + { + source.Rewind(1); + return CollectionType.None; + } + + return type; + } + + public sealed override int DecodeArrayHeader(ref SequenceReader source, out int requestHint) + { + var startOffset = source.Consumed; + var type = this.DecodeArrayOrMapHeaderCore(ref source, out var header, out var itemsCount, out requestHint); + if (requestHint != 0) + { + return 0; + } + + if (type.IsMap) + { + MessagePackThrow.TypeIsNotArray(header, startOffset); + } + + return (int)itemsCount; + } + + public sealed override int DecodeMapHeader(ref SequenceReader source, out int requestHint) + { + var startOffset = source.Consumed; + var type = this.DecodeArrayOrMapHeaderCore(ref source, out var header, out var itemsCount, out requestHint); + if (requestHint != 0) + { + return 0; + } + + if (type.IsArray) + { + MessagePackThrow.TypeIsNotMap(header, startOffset); + } + + return (int)itemsCount; + } + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.DecodeItem.cs b/src/MsgPack.Core/Internal/MessagePackDecoder.DecodeItem.cs new file mode 100644 index 000000000..790ba708f --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.DecodeItem.cs @@ -0,0 +1,375 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache 2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Threading; + +namespace MsgPack.Internal +{ + public partial class MessagePackDecoder + { + public sealed override bool DecodeItem(ref SequenceReader source, out DecodeItemResult result, CancellationToken cancellationToken = default) + { + var elementType = this.ReadHeader(ref source, out var consumed, out var valueOrLength, out var requestHint); + if (requestHint != 0) + { + result = DecodeItemResult.InsufficientInput(requestHint); + return false; + } + + requestHint = 0; + + switch (elementType) + { + case ElementType.True: + { + result = DecodeItemResult.True(); + break; + } + case ElementType.False: + { + result = DecodeItemResult.False(); + break; + } + case ElementType.Null: + { + result = DecodeItemResult.Null(); + break; + } + case ElementType.Int32: + case ElementType.Single: + { + var buffer = new byte[sizeof(int)]; + var span = MemoryMarshal.Cast(buffer); + span[0] = (int)valueOrLength; + result = DecodeItemResult.ScalarOrSequence(elementType, buffer); + break; + } + case ElementType.Array: + case ElementType.Map: + { + this.DecodeArrayOrMap(ref source, out var iterator); + result = DecodeItemResult.CollectionHeader(elementType, this.CreateIterator((uint)valueOrLength), valueOrLength); + break; + } + case ElementType.Int64: + case ElementType.UInt64: + case ElementType.Double: + { + var buffer = new byte[sizeof(long)]; + var span = MemoryMarshal.Cast(buffer); + span[0] = valueOrLength; + result = DecodeItemResult.ScalarOrSequence(elementType, buffer); + break; + } + case ElementType.String: + case ElementType.Binary: + { + if (source.Remaining < valueOrLength + consumed) + { + result = + DecodeItemResult.InsufficientInput( + (int)((valueOrLength + consumed - source.Remaining) & Int32.MaxValue) + ); + return false; + } + + result = DecodeItemResult.ScalarOrSequence(elementType, source.Sequence.Slice(source.Position).Slice(consumed, valueOrLength)); + consumed += valueOrLength; + break; + } + default: + { + Debug.Assert(elementType == ElementType.Extension, $"elementType({elementType}, 0x{elementType:X8}) == ElementType.Extension"); + + if(source.Remaining < valueOrLength + consumed) + { + result = + DecodeItemResult.InsufficientInput( + (int)((valueOrLength + consumed - source.Remaining) & Int32.MaxValue) + ); + return false; + } + + var extensionSlice = source.Sequence.Slice(source.Position).Slice(consumed - 1); + var typeCode = extensionSlice.FirstSpan[0]; + var body = extensionSlice.Slice(1, valueOrLength); + + result = DecodeItemResult.ExtensionType(new ExtensionTypeObject(new ExtensionType(typeCode), body)); + consumed += valueOrLength; + break; + } + } + + source.Advance(consumed); + return true; + } + + private ElementType ReadHeader(ref SequenceReader source, out long consumed, out long valueOrLength, out int requestHint) + { + if (!source.TryPeek(out var header)) + { + requestHint = 1; + valueOrLength = default; + consumed = 0; + return default; + } + + consumed = 1; + ElementType result; + requestHint = 0; + + if (header < 128) + { + valueOrLength = header; + result = ElementType.Int32; + } + else if (header >= 0xE0) + { + valueOrLength = unchecked((sbyte)header); + result = ElementType.Int32; + } + else if (header >= MessagePackCode.MinimumFixedRaw && header <= MessagePackCode.MaximumFixedRaw) + { + valueOrLength = header - MessagePackCode.MinimumFixedRaw; + result = ElementType.String; + } + else if (header >= MessagePackCode.MinimumFixedArray && header <= MessagePackCode.MaximumFixedArray) + { + valueOrLength = header - MessagePackCode.MinimumFixedArray; + result = ElementType.Array; + } + else if (header >= MessagePackCode.MinimumFixedMap && header <= MessagePackCode.MaximumFixedMap) + { + valueOrLength = header - MessagePackCode.MinimumFixedMap; + result = ElementType.Map; + } + else + { + switch (header) + { + case MessagePackCode.SignedInt8: + { + valueOrLength = ReadSByte(ref source, offset: 1, out requestHint); + consumed += sizeof(sbyte); + result = ElementType.Int32; + break; + } + case MessagePackCode.SignedInt16: + { + valueOrLength = ReadValue(ref source, offset: 1, out requestHint); + consumed += sizeof(short); + result = ElementType.Int32; + break; + } + case MessagePackCode.SignedInt32: + { + valueOrLength = ReadValue(ref source, offset: 1, out requestHint); + consumed += sizeof(int); + result = ElementType.Int32; + break; + } + case MessagePackCode.SignedInt64: + { + valueOrLength = ReadValue(ref source, offset: 1, out requestHint); + consumed += sizeof(long); + result = ElementType.Int64; + break; + } + case MessagePackCode.UnsignedInt8: + { + valueOrLength = ReadByte(ref source, offset: 1, out requestHint); + consumed += sizeof(byte); + result = ElementType.Int32; + break; + } + case MessagePackCode.UnsignedInt16: + { + valueOrLength = ReadValue(ref source, offset: 1, out requestHint); + consumed += sizeof(ushort); + result = ElementType.Int32; + break; + } + case MessagePackCode.UnsignedInt32: + { + valueOrLength = ReadValue(ref source, offset: 1, out requestHint); + consumed += sizeof(uint); + result = ElementType.Int64; + break; + } + case MessagePackCode.UnsignedInt64: + { + valueOrLength = unchecked((long)ReadValue(ref source, offset: 1, out requestHint)); + consumed += sizeof(ulong); + result = ElementType.UInt64; + break; + } + case MessagePackCode.Real32: + { + valueOrLength = ReadValue(ref source, offset: 1, out requestHint); + consumed += sizeof(int); + result = ElementType.Single; + break; + } + case MessagePackCode.Real64: + { + valueOrLength = ReadValue(ref source, offset: 1, out requestHint); + consumed += sizeof(long); + result = ElementType.Double; + break; + } + case MessagePackCode.NilValue: + { + valueOrLength = 0; + result = ElementType.Null; + break; + } + case MessagePackCode.TrueValue: + { + valueOrLength = 1; + result = ElementType.True; + break; + } + case MessagePackCode.FalseValue: + { + valueOrLength = 0; + result = ElementType.False; + break; + } + case MessagePackCode.Array16: + { + valueOrLength = ReadValue(ref source, offset: 1, out requestHint); + consumed += sizeof(ushort); + result = ElementType.Array; + break; + } + case MessagePackCode.Array32: + { + valueOrLength = ReadValue(ref source, offset: 1, out requestHint); + consumed += sizeof(uint); + result = ElementType.Array; + break; + } + case MessagePackCode.Map16: + { + valueOrLength = ReadValue(ref source, offset: 1, out requestHint); + consumed += sizeof(ushort); + result = ElementType.Map; + break; + } + case MessagePackCode.Map32: + { + valueOrLength = ReadValue(ref source, offset: 1, out requestHint); + consumed += sizeof(uint); + result = ElementType.Map; + break; + } + case MessagePackCode.FixExt1: + case MessagePackCode.FixExt2: + case MessagePackCode.FixExt4: + case MessagePackCode.FixExt8: + case MessagePackCode.FixExt16: + case MessagePackCode.Ext8: + case MessagePackCode.Ext16: + case MessagePackCode.Ext32: + { + result = ReadExtentionItem(header, ref source, offset: 1, ref consumed, out valueOrLength, out requestHint); + break; + } + default: + { + Debug.Assert(header == 0xC1, $"header(0x{header:X2}) == 0xC1"); + valueOrLength = 0; + result = ElementType.None; + break; + } + } // switch + } // else + + return result; + } + + private static ElementType ReadExtentionItem(byte header, ref SequenceReader source, int offset, ref long consumed, out long valueOrLength, out int requestHint) + { + consumed++; + requestHint = 0; + + switch (header) + { + case MessagePackCode.FixExt1: + { + valueOrLength = 1; + break; + } + case MessagePackCode.FixExt2: + { + valueOrLength = 2; + break; + } + case MessagePackCode.FixExt4: + { + valueOrLength = 4; + break; + } + case MessagePackCode.FixExt8: + { + valueOrLength = 8; + break; + } + case MessagePackCode.FixExt16: + { + valueOrLength = 16; + break; + } + default: + { + _ = ReadByte(ref source, offset, out requestHint); + if (requestHint != 0) + { + valueOrLength = default; + return default; + } + + consumed++; + offset++; + + switch (header) + { + case MessagePackCode.Ext8: + { + valueOrLength = ReadByte(ref source, offset, out requestHint); + consumed++; + break; + } + case MessagePackCode.Ext16: + { + valueOrLength = ReadValue(ref source, offset, out requestHint); + consumed += sizeof(ushort); + break; + } + default: + { + Debug.Assert(header == MessagePackCode.Ext32, $"header(0x{header:X2}) == MessagePackCode.Ext32(0x{MessagePackCode.Ext32:X2})"); + valueOrLength = ReadValue(ref source, offset + 1, out requestHint); + consumed += sizeof(uint); + break; + } + } + + break; + } + } + + if (requestHint != 0) + { + return default; + } + + return ElementType.Extension; + } + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.Extension.cs b/src/MsgPack.Core/Internal/MessagePackDecoder.Extension.cs new file mode 100644 index 000000000..902e51454 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.Extension.cs @@ -0,0 +1,137 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache 2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Threading; + +namespace MsgPack.Internal +{ + public partial class MessagePackDecoder + { + private uint ReadExtensionHeader(ref SequenceReader source, out int consumed, out int requestHint) + { + if(!source.TryPeek(out var header)) + { + consumed = 0; + requestHint = 1; + return default; + } + + consumed = 1; + + int lengthOflength; + switch (header) + { + case MessagePackCode.FixExt1: + { + requestHint = 0; + return 1; + } + case MessagePackCode.FixExt2: + { + requestHint = 0; + return 2; + } + case MessagePackCode.FixExt4: + { + requestHint = 0; + return 4; + } + case MessagePackCode.FixExt8: + { + requestHint = 0; + return 8; + } + case MessagePackCode.FixExt16: + { + requestHint = 0; + return 16; + } + case MessagePackCode.Ext8: + { + lengthOflength = 1; + break; + } + case MessagePackCode.Ext16: + { + lengthOflength = 2; + break; + } + case MessagePackCode.Ext32: + { + lengthOflength = 4; + break; + } + default: + { + MessagePackThrow.IsNotExtension(header, source.Consumed); + // never + requestHint = default; + return default; + } + } // switch + + uint length; + switch (lengthOflength) + { + case 1: + { + length = ReadByte(ref source, offset: 1, out requestHint); + break; + } + case 2: + { + length = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: + { + Debug.Assert(lengthOflength == 4, $"length({lengthOflength}) == 4"); + length = ReadValue(ref source, offset: 1, out requestHint); + break; + } + } + + if (requestHint != 0) + { + return default; + } + + consumed += lengthOflength; + return length; + } + + public override void DecodeExtension(ref SequenceReader source, out ExtensionTypeObject result, out int requestHint, CancellationToken cancellationToken = default) + { + var bodyLength = this.ReadExtensionHeader(ref source, out var consumed, out requestHint); + if (requestHint != 0) + { + result = default; + return; + } + + if (!source.TryPeek(out byte typeCode)) + { + requestHint = 1; + result = default; + return; + } + + consumed++; + + if (source.Remaining < consumed + bodyLength) + { + requestHint = (int)((consumed + bodyLength - (int)source.Remaining) & Int32.MaxValue); + result = default; + return; + } + + requestHint = 0; + result = new ExtensionTypeObject(new ExtensionType(typeCode), source.Sequence.Slice(consumed, bodyLength)); + source.Advance(consumed + bodyLength); + } + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.Integers.cs b/src/MsgPack.Core/Internal/MessagePackDecoder.Integers.cs new file mode 100644 index 000000000..ff2a60e40 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.Integers.cs @@ -0,0 +1,464 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + partial class MessagePackDecoder + { + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override SByte DecodeSByte(ref SequenceReader source, out int requestHint) + { + if (!this.TryDecodeSignedInteger(ref source, typeof(SByte), out var header, out var result, out requestHint)) + { + return default; + } + if (result < SByte.MinValue || result > SByte.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, typeof(SByte)); + } + + return unchecked((SByte)result); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Int16 DecodeInt16(ref SequenceReader source, out int requestHint) + { + if (!this.TryDecodeSignedInteger(ref source, typeof(Int16), out var header, out var result, out requestHint)) + { + return default; + } + if (result < Int16.MinValue || result > Int16.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, typeof(Int16)); + } + + return unchecked((Int16)result); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Int32 DecodeInt32(ref SequenceReader source, out int requestHint) + { + if (!this.TryDecodeSignedInteger(ref source, typeof(Int32), out var header, out var result, out requestHint)) + { + return default; + } + if (result < Int32.MinValue || result > Int32.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, typeof(Int32)); + } + + return unchecked((Int32)result); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Int64 DecodeInt64(ref SequenceReader source, out int requestHint) + { + if (!this.TryDecodeSignedInteger(ref source, typeof(Int64), out var header, out var result, out requestHint)) + { + return default; + } + return unchecked((Int64)result); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Byte DecodeByte(ref SequenceReader source, out int requestHint) + { + if (!this.TryDecodeUnsignedInteger(ref source, typeof(Byte), out var header, out var result, out requestHint)) + { + return default; + } + if (result < Byte.MinValue || result > Byte.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, typeof(Byte)); + } + + return unchecked((Byte)result); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override UInt16 DecodeUInt16(ref SequenceReader source, out int requestHint) + { + if (!this.TryDecodeUnsignedInteger(ref source, typeof(UInt16), out var header, out var result, out requestHint)) + { + return default; + } + if (result < UInt16.MinValue || result > UInt16.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, typeof(UInt16)); + } + + return unchecked((UInt16)result); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override UInt32 DecodeUInt32(ref SequenceReader source, out int requestHint) + { + if (!this.TryDecodeUnsignedInteger(ref source, typeof(UInt32), out var header, out var result, out requestHint)) + { + return default; + } + if (result < UInt32.MinValue || result > UInt32.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, typeof(UInt32)); + } + + return unchecked((UInt32)result); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override UInt64 DecodeUInt64(ref SequenceReader source, out int requestHint) + { + if (!this.TryDecodeUnsignedInteger(ref source, typeof(UInt64), out var header, out var result, out requestHint)) + { + return default; + } + return unchecked((UInt64)result); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private bool TryDecodeSignedInteger(ref SequenceReader source, Type type, out byte header, out Int64 value, out int requestHint) + { + if (!source.TryPeek(out header)) + { + requestHint = 1; + value = default; + return false; + } + + requestHint = 0; + + if (header < 128) + { + value = header; + source.Advance(1); + return true; + } + + if (header >= 0xE0) + { + value = unchecked((sbyte)header); + source.Advance(1); + return true; + } + + return this.TryDecodeSignedIntegerSlow(ref source, type, header, out value, out requestHint); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private bool TryDecodeSignedIntegerSlow(ref SequenceReader source, Type type, byte header, out Int64 value, out int requestHint) + { + ParseNumberHeader(header, ref source, type, out var length, out var kind); + + if ((kind & NumberKind.RealBitMask) != 0 && !this.Options.CanTreatRealAsInteger) + { + MessagePackThrow.RealCannotBeInteger(header, source.Consumed, type); + } + + switch (kind) + { + case NumberKind.Signed: + { + switch (length) + { + case 1: + { + value = ReadSByte(ref source, offset: 1, out requestHint); + break; + } + case 2: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + case 4: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: + { + Debug.Assert(length == 8, $"length({length}) != 8"); + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + } + + if (requestHint != 0) + { + value = default; + return false; + } + break; + } + case NumberKind.Unsigned: + { + ulong unsigned; + switch (length) + { + case 1: + { + unsigned = ReadByte(ref source, offset: 1, out requestHint); + break; + } + case 2: + { + unsigned = ReadValue(ref source, offset: 1, out requestHint); + break; + } + case 4: + { + unsigned = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: + { + Debug.Assert(length == 8, $"length({length}) != 8"); + unsigned = ReadValue(ref source, offset: 1, out requestHint); + break; + } + } + + if (requestHint != 0) + { + value = default; + return false; + } + + if (unsigned > Int64.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + value = unchecked((Int64)unsigned); + break; + } + case NumberKind.Single: + { + var real = ReadValue(ref source, offset: 1, out requestHint); + if (requestHint != 0) + { + value = default; + return false; + } + + if (real > Int64.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + if (real < Int64.MinValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + value = unchecked((Int64)real); + break; + } + default: + { + Debug.Assert(kind == NumberKind.Double, $"kind({kind}) == NumberType.Double"); + var real = ReadValue(ref source, offset: 1, out requestHint); + if (requestHint != 0) + { + value = default; + return false; + } + + if (real > Int64.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + if (real < Int64.MinValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + value = unchecked((Int64)real); + break; + } + } + + return true; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private bool TryDecodeUnsignedInteger(ref SequenceReader source, Type type, out byte header, out UInt64 value, out int requestHint) + { + if (!source.TryPeek(out header)) + { + requestHint = 1; + value = default; + return false; + } + + requestHint = 0; + + if (header < 128) + { + value = header; + source.Advance(1); + return true; + } + + return this.TryDecodeUnsignedIntegerSlow(ref source, type, header, out value, out requestHint); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private bool TryDecodeUnsignedIntegerSlow(ref SequenceReader source, Type type, byte header, out UInt64 value, out int requestHint) + { + ParseNumberHeader(header, ref source, type, out var length, out var kind); + + if ((kind & NumberKind.RealBitMask) != 0 && !this.Options.CanTreatRealAsInteger) + { + MessagePackThrow.RealCannotBeInteger(header, source.Consumed, type); + } + + switch (kind) + { + case NumberKind.Signed: + { + long signed; + switch (length) + { + case 1: + { + signed = ReadSByte(ref source, offset: 1, out requestHint); + break; + } + case 2: + { + signed = ReadValue(ref source, offset: 1, out requestHint); + break; + } + case 4: + { + signed = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: + { + Debug.Assert(length == 8, $"length({length}) != 8"); + signed = ReadValue(ref source, offset: 1, out requestHint); + break; + } + } + + if (requestHint != 0) + { + value = default; + return false; + } + + if (signed < 0) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + value = unchecked((ulong)signed); + break; + } + case NumberKind.Unsigned: + { + switch (length) + { + case 1: + { + value = ReadByte(ref source, offset: 1, out requestHint); + break; + } + case 2: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + case 4: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: + { + Debug.Assert(length == 8, $"length({length}) != 8"); + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + } + + if (requestHint != 0) + { + value = default; + return false; + } + break; + } + case NumberKind.Single: + { + var real = ReadValue(ref source, offset: 1, out requestHint); + if (requestHint != 0) + { + value = default; + return false; + } + + if (real > UInt64.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + if (real < UInt64.MinValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + value = unchecked((UInt64)real); + break; + } + default: + { + Debug.Assert(kind == NumberKind.Double, $"kind({kind}) == NumberType.Double"); + var real = ReadValue(ref source, offset: 1, out requestHint); + if (requestHint != 0) + { + value = default; + return false; + } + + if (real > UInt64.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + if (real < UInt64.MinValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + value = unchecked((UInt64)real); + break; + } + } + + return true; + } + + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.Integers.tt b/src/MsgPack.Core/Internal/MessagePackDecoder.Integers.tt new file mode 100644 index 000000000..329a60b0c --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.Integers.tt @@ -0,0 +1,270 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ include file="./MessagePackDecoder.Number.ttinclude" #> +<#@ output extension=".cs" #> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + partial class MessagePackDecoder + { +<# +// -------------- Decode() ------------------- + +// Signed +foreach (var type in + new [] + { + "SByte", + "Int16", + "Int32" + } +) +{ + this.WriteMethod(type, isSigned: true, is64Bit: false); +} + +this.WriteMethod("Int64", isSigned: true, is64Bit: true); + +// Unsigned +foreach (var type in + new [] + { + "Byte", + "UInt16", + "UInt32" + } +) +{ + this.WriteMethod(type, isSigned: false, is64Bit: false); +} + +this.WriteMethod("UInt64", isSigned: false, is64Bit: true); + +// ------------- Decode[Uns|S]ignedInteger ------------ +foreach (var isSigned in new [] { true, false }) +{ + var methodName = $"TryDecode{(isSigned ? "Signed" : "Unsigned")}Integer"; + var outType = isSigned ? "Int64": "UInt64"; +#> + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private bool <#= methodName #>(ref SequenceReader source, Type type, out byte header, out <#= outType #> value, out int requestHint) + { + if (!source.TryPeek(out header)) + { + requestHint = 1; + value = default; + return false; + } + + requestHint = 0; + + if (header < 128) + { + value = header; + source.Advance(1); + return true; + } + +<# + if (isSigned) + { +#> + if (header >= 0xE0) + { + value = unchecked((sbyte)header); + source.Advance(1); + return true; + } + +<# + } // if (isSigned) +#> + return this.<#= methodName #>Slow(ref source, type, header, out value, out requestHint); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private bool <#= methodName #>Slow(ref SequenceReader source, Type type, byte header, out <#= outType #> value, out int requestHint) + { + ParseNumberHeader(header, ref source, type, out var length, out var kind); + + if ((kind & NumberKind.RealBitMask) != 0 && !this.Options.CanTreatRealAsInteger) + { + MessagePackThrow.RealCannotBeInteger(header, source.Consumed, type); + } + + switch (kind) + { + case NumberKind.Signed: + { +<# + if (isSigned) + { + WriteDecodeInteger("value", isSigned: true); +#> + if (requestHint != 0) + { + value = default; + return false; + } +<# + } + else + { +#> + long signed; +<# + WriteDecodeInteger("signed", isSigned: true); +#> + if (requestHint != 0) + { + value = default; + return false; + } + + if (signed < 0) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + value = unchecked((ulong)signed); +<# + } // if (isSigned) +#> + break; + } + case NumberKind.Unsigned: + { +<# + if (isSigned) + { +#> + ulong unsigned; +<# + WriteDecodeInteger("unsigned", isSigned: false); +#> + if (requestHint != 0) + { + value = default; + return false; + } + + if (unsigned > Int64.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + value = unchecked((<#= outType #>)unsigned); +<# + } + else + { + WriteDecodeInteger("value", isSigned: false); +#> + if (requestHint != 0) + { + value = default; + return false; + } +<# + } // if (isSigned) +#> + break; + } + case NumberKind.Single: + { + var real = ReadValue(ref source, offset: 1, out requestHint); + if (requestHint != 0) + { + value = default; + return false; + } + + if (real > <#= outType #>.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + if (real < <#= outType #>.MinValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + value = unchecked((<#= outType #>)real); + break; + } + default: + { + Debug.Assert(kind == NumberKind.Double, $"kind({kind}) == NumberType.Double"); + var real = ReadValue(ref source, offset: 1, out requestHint); + if (requestHint != 0) + { + value = default; + return false; + } + + if (real > <#= outType #>.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + if (real < <#= outType #>.MinValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, type); + } + + value = unchecked((<#= outType #>)real); + break; + } + } + + return true; + } + +<# +} // foreach (var isSigned) +#> + } +} +<#+ +void WriteMethod(string type, bool isSigned, bool is64Bit) +{ + var coreMethod = $"TryDecode{(isSigned ? "Signed" : "Unsigned")}Integer"; + var resultType = isSigned ? "long" : "ulong"; +#> + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override <#= type #> Decode<#= type #>(ref SequenceReader source, out int requestHint) + { + if (!this.<#= coreMethod #>(ref source, typeof(<#= type #>), out var header, out var result, out requestHint)) + { + return default; + } +<#+ + if (!is64Bit) + { +#> + if (result < <#= type #>.MinValue || result > <#= type #>.MaxValue) + { + MessagePackThrow.IsNotType(header, source.Consumed, typeof(<#= type #>)); + } + +<#+ + } // if (!is64Bit) +#> + return unchecked((<#= type #>)result); + } + +<#+ +} +#> diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.Iteration.cs b/src/MsgPack.Core/Internal/MessagePackDecoder.Iteration.cs new file mode 100644 index 000000000..cb5fc938d --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.Iteration.cs @@ -0,0 +1,66 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache 2 license. +// See the LICENSE in the project root for more information. + +using System.Buffers; + +namespace MsgPack.Internal +{ + public partial class MessagePackDecoder + { + private readonly CollectionItemIterator.CollectionEndDetection _detectCollectionEnds; + + // This is instance method now because CLR/CoreCLR delegates are optmized for instance method invocation. + // This method might be changed static if C# supports function pointer. + private bool DetectCollectionEnds(ref SequenceReader source, ref long nextItemIndex, long itemsCount, out int requestHint) + { + requestHint = 0; + + if (nextItemIndex >= itemsCount) + { + return false; + } + + nextItemIndex++; + return true; + } + + public override CollectionType DecodeArrayOrMap(ref SequenceReader source, out CollectionItemIterator iterator, out int requestHint) + { + var type = this.PrivateDecodeArrayOrMapHeader(ref source, out _, out var itemsCount, out requestHint); + if (requestHint != 0) + { + iterator = default; + return default; + } + + iterator = this.CreateIterator(itemsCount); + return type; + } + + private CollectionItemIterator CreateIterator(long itemsCount) + => new CollectionItemIterator(this._detectCollectionEnds, itemsCount); + + public override CollectionItemIterator DecodeArray(ref SequenceReader source, out int requestHint) + { + var itemsCount = this.DecodeArrayHeader(ref source, out requestHint); + if (requestHint != 0) + { + return default; + } + + return new CollectionItemIterator(this._detectCollectionEnds, itemsCount); + } + + public override CollectionItemIterator DecodeMap(ref SequenceReader source, out int requestHint) + { + var itemsCount = this.DecodeMapHeader(ref source, out requestHint); + if (requestHint != 0) + { + return default; + } + + return new CollectionItemIterator(this._detectCollectionEnds, itemsCount); + } + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.Nullables.cs b/src/MsgPack.Core/Internal/MessagePackDecoder.Nullables.cs new file mode 100644 index 000000000..272a67fc9 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.Nullables.cs @@ -0,0 +1,161 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + partial class MessagePackDecoder + { + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Byte? DecodeNullableByte(ref SequenceReader source, out int requestHint) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeByte(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override SByte? DecodeNullableSByte(ref SequenceReader source, out int requestHint) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeSByte(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Int16? DecodeNullableInt16(ref SequenceReader source, out int requestHint) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeInt16(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override UInt16? DecodeNullableUInt16(ref SequenceReader source, out int requestHint) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeUInt16(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Int32? DecodeNullableInt32(ref SequenceReader source, out int requestHint) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeInt32(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override UInt32? DecodeNullableUInt32(ref SequenceReader source, out int requestHint) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeUInt32(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Int64? DecodeNullableInt64(ref SequenceReader source, out int requestHint) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeInt64(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override UInt64? DecodeNullableUInt64(ref SequenceReader source, out int requestHint) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeUInt64(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Single? DecodeNullableSingle(ref SequenceReader source, out int requestHint) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeSingle(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Double? DecodeNullableDouble(ref SequenceReader source, out int requestHint) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeDouble(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Boolean? DecodeNullableBoolean(ref SequenceReader source, out int requestHint) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeBoolean(ref source, out requestHint); + } + + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.Nullables.tt b/src/MsgPack.Core/Internal/MessagePackDecoder.Nullables.tt new file mode 100644 index 000000000..5af782a13 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.Nullables.tt @@ -0,0 +1,55 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ output extension=".cs" #> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + partial class MessagePackDecoder + { +<# +foreach (var type in + new [] + { + "Byte", + "SByte", + "Int16", + "UInt16", + "Int32", + "UInt32", + "Int64", + "UInt64", + "Single", + "Double", + "Boolean" + } +) +{ +#> + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override <#= type #>? DecodeNullable<#= type #>(ref SequenceReader source, out int requestHint) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.Decode<#= type #>(ref source, out requestHint); + } + +<# +} // foreach (var type) +#> + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.Number.ttinclude b/src/MsgPack.Core/Internal/MessagePackDecoder.Number.ttinclude new file mode 100644 index 000000000..f694f1406 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.Number.ttinclude @@ -0,0 +1,32 @@ +<#+ +void WriteDecodeInteger(string variable, bool isSigned) +{ +#> + switch (length) + { + case 1: + { + <#= variable #> = Read<#= isSigned ? "SByte" : "Byte" #>(ref source, offset: 1, out requestHint); + break; + } + case 2: + { + <#= variable #> = ReadValue<<#= isSigned ? "short" : "ushort" #>>(ref source, offset: 1, out requestHint); + break; + } + case 4: + { + <#= variable #> = ReadValue<<#= isSigned ? "int" : "uint" #>>(ref source, offset: 1, out requestHint); + break; + } + default: + { + Debug.Assert(length == 8, $"length({length}) != 8"); + <#= variable #> = ReadValue<<#= isSigned ? "long" : "ulong" #>>(ref source, offset: 1, out requestHint); + break; + } + } + +<#+ +} +#> diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.Reals.cs b/src/MsgPack.Core/Internal/MessagePackDecoder.Reals.cs new file mode 100644 index 000000000..3023f9ff2 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.Reals.cs @@ -0,0 +1,223 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + partial class MessagePackDecoder + { + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Single DecodeSingle(ref SequenceReader source, out int requestHint) + { + if (!source.TryPeek(out var header)) + { + requestHint = 1; + return default; + } + + requestHint = 0; + + if (header < 128 || header >= 0xE0) + { + source.Advance(1); + return header; + } + + ParseNumberHeader(header, ref source, typeof(Single), out var length, out var kind); + + Single value; + switch (kind) + { + case NumberKind.Signed: + { + switch (length) + { + case 1: + { + value = ReadSByte(ref source, offset: 1, out requestHint); + break; + } + case 2: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + case 4: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: + { + Debug.Assert(length == 8, $"length({length}) != 8"); + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + } + + break; + } + case NumberKind.Unsigned: + { + switch (length) + { + case 1: + { + value = ReadByte(ref source, offset: 1, out requestHint); + break; + } + case 2: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + case 4: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: + { + Debug.Assert(length == 8, $"length({length}) != 8"); + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + } + + break; + } + case NumberKind.Single: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: + { + // Double + value = (Single)ReadValue(ref source, offset: 1, out requestHint); + break; + } + } + + if (requestHint != 0) + { + return default; + } + + return value; + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Double DecodeDouble(ref SequenceReader source, out int requestHint) + { + if (!source.TryPeek(out var header)) + { + requestHint = 1; + return default; + } + + requestHint = 0; + + if (header < 128 || header >= 0xE0) + { + source.Advance(1); + return header; + } + + ParseNumberHeader(header, ref source, typeof(Double), out var length, out var kind); + + Double value; + switch (kind) + { + case NumberKind.Signed: + { + switch (length) + { + case 1: + { + value = ReadSByte(ref source, offset: 1, out requestHint); + break; + } + case 2: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + case 4: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: + { + Debug.Assert(length == 8, $"length({length}) != 8"); + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + } + + break; + } + case NumberKind.Unsigned: + { + switch (length) + { + case 1: + { + value = ReadByte(ref source, offset: 1, out requestHint); + break; + } + case 2: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + case 4: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: + { + Debug.Assert(length == 8, $"length({length}) != 8"); + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + } + + break; + } + case NumberKind.Single: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: + { + // Double + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + } + + if (requestHint != 0) + { + return default; + } + + return value; + } + + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.Reals.tt b/src/MsgPack.Core/Internal/MessagePackDecoder.Reals.tt new file mode 100644 index 000000000..72d577988 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.Reals.tt @@ -0,0 +1,100 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ include file="./MessagePackDecoder.Number.ttinclude" #> +<#@ output extension=".cs" #> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + partial class MessagePackDecoder + { +<# +foreach (var type in new [] { "Single", "Double" }) +{ +#> + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override <#= type #> Decode<#= type #>(ref SequenceReader source, out int requestHint) + { + if (!source.TryPeek(out var header)) + { + requestHint = 1; + return default; + } + + requestHint = 0; + + if (header < 128 || header >= 0xE0) + { + source.Advance(1); + return header; + } + + ParseNumberHeader(header, ref source, typeof(<#= type #>), out var length, out var kind); + + <#= type #> value; + switch (kind) + { + case NumberKind.Signed: + { +<# + WriteDecodeInteger("value", isSigned: true); +#> + break; + } + case NumberKind.Unsigned: + { +<# + WriteDecodeInteger("value", isSigned: false); +#> + break; + } + case NumberKind.Single: + { + value = ReadValue(ref source, offset: 1, out requestHint); + break; + } + default: + { + // Double +<# + if (type != "Double") + { +#> + value = (<#= type #>)ReadValue(ref source, offset: 1, out requestHint); +<# + } + else + { +#> + value = ReadValue(ref source, offset: 1, out requestHint); +<# + } +#> + break; + } + } + + if (requestHint != 0) + { + return default; + } + + return value; + } + +<# +} // foreach (var isSigned) +#> + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.SkipDrain.cs b/src/MsgPack.Core/Internal/MessagePackDecoder.SkipDrain.cs new file mode 100644 index 000000000..6337cec11 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.SkipDrain.cs @@ -0,0 +1,288 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache 2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace MsgPack.Internal +{ + public partial class MessagePackDecoder + { + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void Skip(ref SequenceReader source, in CollectionContext collectionContext, out int requestHint, CancellationToken cancellationToken = default) + => this.Drain(ref source, collectionContext, itemsCount: 1, out requestHint, cancellationToken); + +#warning TODO: Remove ref consumed + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void Drain(ref SequenceReader source, in CollectionContext collectionContext, long itemsCount, out int requestHint, CancellationToken cancellationToken = default) + { + var consumed = 0L; + if (!this.SkipItems(ref source, collectionContext, itemsCount, ref consumed, out requestHint, cancellationToken)) + { + source.Rewind(consumed); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private static bool SkipLength(ref SequenceReader source, long length, ref long consumed, out int requestHint) + { + if (source.Remaining < length) + { + requestHint = (int)((length - source.Remaining) & Int32.MaxValue); + return false; + } + + requestHint = 0; + source.Advance(length); + consumed += length; + return true; + } + + private bool SkipArray(ref SequenceReader source, in CollectionContext collectionContext, ref long consumed, out int requestHint, CancellationToken cancellationToken = default) + { + collectionContext.IncrementDepth(); + + var initialConsumed = consumed; + var type = this.DecodeArrayOrMapHeaderCore(ref source, out var header, out var arrayLength, out requestHint); + if (requestHint != 0) + { + return false; + } + + if (!type.IsArray) + { + MessagePackThrow.TypeIsNotArray(header, initialConsumed); + } + + consumed = source.Consumed - initialConsumed; + + if (!this.SkipItems(ref source, collectionContext, arrayLength, ref consumed, out requestHint, cancellationToken)) + { + return false; + } + + collectionContext.DecrementDepth(); + return true; + } + + private bool SkipMap(ref SequenceReader source, in CollectionContext collectionContext, ref long consumed, out int requestHint, CancellationToken cancellationToken = default) + { + collectionContext.IncrementDepth(); + + var initialConsumed = consumed; + var type = this.DecodeArrayOrMapHeaderCore(ref source, out var header, out var mapCount, out requestHint); + if (requestHint != 0) + { + return false; + } + + if (!type.IsMap) + { + MessagePackThrow.TypeIsNotMap(header, initialConsumed); + } + + consumed = source.Consumed - initialConsumed; + + if (!this.SkipItems(ref source, collectionContext, mapCount * 2, ref consumed, out requestHint, cancellationToken)) + { + return false; + } + + collectionContext.DecrementDepth(); + return true; + } + + private bool SkipItems(ref SequenceReader source, in CollectionContext collectionContext, long itemsCount, ref long consumed, out int requestHint, CancellationToken cancellationToken = default) + { + while (itemsCount > 0) + { + if (!source.TryPeek(out var header)) + { + requestHint = 1; + return false; + } + + consumed++; + + var length = 0L; + + if (header >= MessagePackCode.MinimumFixedArray && header <= MessagePackCode.MaximumFixedArray) + { + if (!this.SkipArray(ref source, collectionContext, ref consumed, out requestHint, cancellationToken)) + { + return false; + } + } + else if (header >= MessagePackCode.MinimumFixedMap && header <= MessagePackCode.MaximumFixedMap) + { + if (!this.SkipMap(ref source, collectionContext, ref consumed, out requestHint, cancellationToken)) + { + return false; + } + } + else if (header >= MessagePackCode.MinimumFixedRaw && header <= MessagePackCode.MaximumFixedRaw) + { + length = header - MessagePackCode.MinimumFixedRaw; + } + else + { + switch (header) + { + case MessagePackCode.SignedInt8: + case MessagePackCode.UnsignedInt8: + { + length = 1; + break; + } + case MessagePackCode.SignedInt16: + case MessagePackCode.UnsignedInt16: + case MessagePackCode.Real32: + case MessagePackCode.SignedInt32: + case MessagePackCode.UnsignedInt32: + { + length = 4; + break; + } + case MessagePackCode.Real64: + case MessagePackCode.SignedInt64: + case MessagePackCode.UnsignedInt64: + { + length = 8; + break; + } + case MessagePackCode.Bin8: + { + if (!source.TryRead(out var b)) + { + requestHint = 1; + return false; + } + + length = b; + consumed++; + break; + } + case MessagePackCode.Bin16: + case MessagePackCode.Raw16: + { + length = ReadValue(ref source, offset: 0, out requestHint); + if (requestHint != 0) + { + source.Rewind(consumed); + return false; + } + + consumed += sizeof(ushort); + break; + } + case MessagePackCode.Bin32: + case MessagePackCode.Raw32: + { + length = ReadValue(ref source, offset: 0, out requestHint); + if (requestHint != 0) + { + source.Rewind(consumed); + return false; + } + + consumed += sizeof(int); + break; + } + case MessagePackCode.FixExt1: + { + length = 2; + break; + } + case MessagePackCode.FixExt2: + { + length = 3; + break; + } + case MessagePackCode.FixExt4: + { + length = 5; + break; + } + case MessagePackCode.FixExt8: + { + length = 9; + break; + } + case MessagePackCode.FixExt16: + { + length = 17; + break; + } + case MessagePackCode.Ext8: + { + if (!source.TryRead(out var b)) + { + requestHint = 1; + return false; + } + + length = b + 1; + consumed++; + break; + } + case MessagePackCode.Ext16: + { + length = ReadValue(ref source, offset: 0, out requestHint); + if (requestHint != 0) + { + source.Rewind(consumed); + return false; + } + + consumed += sizeof(ushort) + 1; + break; + } + case MessagePackCode.Ext32: + { + length = ReadValue(ref source, offset: 0, out requestHint); + if (requestHint != 0) + { + source.Rewind(consumed); + return false; + } + + consumed += sizeof(int) + 1; + break; + } + case MessagePackCode.Array16: + case MessagePackCode.Array32: + { + if (!this.SkipArray(ref source, collectionContext, ref consumed, out requestHint, cancellationToken)) + { + return false; + } + break; + } + case MessagePackCode.Map16: + case MessagePackCode.Map32: + { + if (!this.SkipMap(ref source, collectionContext, ref consumed, out requestHint, cancellationToken)) + { + return false; + } + break; + } + } + } + + if (length > 0 && !SkipLength(ref source, length, ref consumed, out requestHint)) + { + return false; + } + + itemsCount--; + } // while (itemsCount > 0) + + requestHint = 0; + return true; + } + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.Strings.cs b/src/MsgPack.Core/Internal/MessagePackDecoder.Strings.cs new file mode 100644 index 000000000..af13ee46a --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.Strings.cs @@ -0,0 +1,289 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache 2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; + +namespace MsgPack.Internal +{ + public partial class MessagePackDecoder + { + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override bool GetRawString(ref SequenceReader source, out ReadOnlySpan rawString, out int requestHint, CancellationToken cancellationToken = default) + { + var subReader = source; + + var length = this.GetRawStringLength(ref subReader, out var headerLength, out requestHint, cancellationToken); + if (length < 0) + { + rawString = default; + return false; + } + + if (source.UnreadSpan.Length < length) + { + rawString = source.UnreadSpan.Slice(0, length + headerLength); + source.Advance(length); + requestHint = 0; + return true; + } + + return this.GetRawStringMultiSegment(ref source, out rawString, out requestHint, length + headerLength, cancellationToken); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private bool GetRawStringMultiSegment(ref SequenceReader source, out ReadOnlySpan rawString, out int requestHint, int length, CancellationToken cancellationToken) + { + if (source.Remaining < length) + { + requestHint = length - (int)source.Remaining; + rawString = default; + return false; + } + + var result = new byte[length]; + + var bufferSpan = result.AsSpan(); + + var copyLength = Math.Min(this.Options.CancellationSupportThreshold, length); + while (bufferSpan.Length > 0) + { + var destination = bufferSpan.Slice(0, copyLength); + var shouldTrue = source.TryCopyTo(destination); + Debug.Assert(shouldTrue, "SequenceReader.Remaining lied."); + + bufferSpan = bufferSpan.Slice(copyLength); + source.Advance(copyLength); + + cancellationToken.ThrowIfCancellationRequested(); + } + + rawString = result; + requestHint = 0; + return true; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private int GetRawStringLength(ref SequenceReader source, out int consumed, out int requestHint, CancellationToken cancellationToken) + { + var length = this.DecodeStringHeader(ref source, out var header, out requestHint, out consumed); + if (requestHint != 0) + { + return default; + } + + if (length > Int32.MaxValue) + { + MessagePackThrow.TooLargeByteLength(header, source.Consumed - consumed, length); + } + + return (int)length; + } + + private long DecodeStringHeader(ref SequenceReader source, out byte header, out int requestHint, out int consumed) + { + if (!source.TryPeek(out header)) + { + requestHint = 1; + consumed = 0; + return 0; + } + + if (header >= MessagePackCode.MinimumFixedRaw && header <= MessagePackCode.MaximumFixedRaw) + { + requestHint = 0; + consumed = 1; + source.Advance(1); + return header - MessagePackCode.MinimumFixedRaw; + } + + long length; + switch (header) + { + case MessagePackCode.Str8: + { + length = ReadByte(ref source, offset: 1, out requestHint); + consumed = 2; + break; + } + case MessagePackCode.Str16: + { + length = ReadValue(ref source, offset: 1, out requestHint); + consumed = 3; + break; + } + case MessagePackCode.Str32: + { + length = ReadValue(ref source, offset: 1, out requestHint); + consumed = 5; + break; + } + default: + { + MessagePackThrow.IsNotUtf8String(header, source.Consumed); + // never + requestHint = default; + consumed = default; + return 0; + } + } + + source.Advance(consumed); + return requestHint == 0 ? length : 0; + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override string? DecodeNullableString(ref SequenceReader source, out int requestHint, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeString(ref source, out requestHint, encoding, cancellationToken); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override string? DecodeString(ref SequenceReader source, out int requestHint, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + var length = this.DecodeStringHeader(ref source, out _, out requestHint, out var consumed); + if (requestHint != 0) + { + return default; + } + + if (source.Remaining < length) + { + requestHint = (int)((length - source.Remaining) & Int32.MaxValue); + return default; + } + + if (length > this.Options.MaxBinaryLengthInBytes) + { + Throw.StringLengthExceeded(source.Consumed - consumed, length, this.Options.MaxBinaryLengthInBytes); + } + + if (encoding == null && length <= this.Options.CancellationSupportThreshold) + { + // fast-path + var value = Utf8EncodingNonBom.Instance.GetString(source.UnreadSpan.Slice(0, (int)length)); + source.Advance(length); + requestHint = 0; + return value; + } + + var result = (encoding ?? Utf8EncodingNonBom.Instance).GetStringMultiSegment(source.Sequence.Slice(source.Position), this.Options.CharBufferPool, cancellationToken); + source.Advance(length); + requestHint = 0; + return result; + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override byte[]? DecodeNullableBinary(ref SequenceReader source, out int requestHint, CancellationToken cancellationToken = default) + { + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeBinary(ref source, out requestHint, cancellationToken); + } + + private int DecodeBinaryHeader(ref SequenceReader source, out int requestHint, out int consumed) + { + if (!source.TryPeek(out var header)) + { + requestHint = 1; + consumed = 0; + return 0; + } + + if (header >= MessagePackCode.MinimumFixedRaw && header <= MessagePackCode.MaximumFixedRaw) + { + requestHint = 0; + consumed = 1; + return header - MessagePackCode.MinimumFixedRaw; + } + + int length; + switch (header) + { + case MessagePackCode.Str8: + case MessagePackCode.Bin8: + { + length = ReadByte(ref source, offset: 1, out requestHint); + consumed = 2; + break; + } + case MessagePackCode.Str16: + case MessagePackCode.Bin16: + { + length = ReadValue(ref source, offset: 1, out requestHint); + consumed = 3; + break; + } + case MessagePackCode.Str32: + case MessagePackCode.Bin32: + { + length = ReadValue(ref source, offset: 1, out requestHint); + if (length < 0) + { + MessagePackThrow.TooLargeByteLength(header, source.Consumed - 5, unchecked((uint)length)); + } + + consumed = 5; + break; + } + default: + { + MessagePackThrow.IsNotUtf8String(header, source.Consumed); + // never + requestHint = default; + consumed = default; + return 0; + } + } + + return requestHint == 0 ? length : 0; + } + + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override byte[]? DecodeBinary(ref SequenceReader source, out int requestHint, CancellationToken cancellationToken = default) + { + var length = this.DecodeBinaryHeader(ref source, out requestHint, out var consumed); + if(requestHint != 0) + { + return default; + } + + if (length > this.Options.MaxBinaryLengthInBytes) + { + Throw.BinaryLengthExceeded(source.Consumed - consumed, length, this.Options.MaxBinaryLengthInBytes); + } + + if(source.Remaining < length) + { + requestHint = length - (int)source.Remaining; + return default; + } + + // This line may throw OutOfMemoryException, but we cannot determine the OOM is caused by heap exhausion or excess of the implementation specific max length of arrays. + // So, we just throws OOM for such conditions. + var result = new byte[length]; + var shouldBeTrue = source.TryCopyTo(result); + Debug.Assert(shouldBeTrue, "SequenceReader.Remaining lied."); + source.Advance(length); + return result; + } + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoder.cs b/src/MsgPack.Core/Internal/MessagePackDecoder.cs new file mode 100644 index 000000000..f9dbd80d6 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoder.cs @@ -0,0 +1,185 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache 2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace MsgPack.Internal +{ + public sealed partial class MessagePackDecoder : FormatDecoder + { + public MessagePackDecoder(MessagePackDecoderOptions options) + : base(options) + { + this._detectCollectionEnds = this.DetectCollectionEnds; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public override bool DecodeBoolean(ref SequenceReader source, out int requestHint) + { + if (!source.TryPeek(out var header)) + { + requestHint = 1; + return false; + } + + requestHint = 0; + switch (header) + { + case MessagePackCode.TrueValue: + { + source.Advance(1); + return true; + } + case MessagePackCode.FalseValue: + { + source.Advance(1); + return false; + } + default: + { + MessagePackThrow.IsNotType(header, source.Consumed, typeof(bool)); + // Never reached. + return false; + } + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override bool TryDecodeNull(ref SequenceReader source, out int requestHint) + { + if(!source.TryPeek(out var b)) + { + requestHint = 1; + return default; + } + + requestHint = 0; + + if (b == MessagePackCode.NilValue) + { + source.Advance(1); + return true; + } + + return false; + } + + private static void ParseNumberHeader(byte header, ref SequenceReader source, Type type, out int length, out NumberKind kind) + { + // 0xD0-D3 -- SignedIntN + // 1101-0000 -> 1101-0011 + // 0xCC-CF -- UnsignedIntN + // 1100-1100 -> 1100-1111 + // 0XCA-CB -- RealN + // 1100-1010 -> 1100-1011 + + if ((header & 0xD3) == header) + { + // SignedIntN + length = (int)Math.Pow(2, (header & 0x3)); + kind = NumberKind.Signed; + } + else if ((header & 0xCC) == header) + { + // UnsignedIntN + length = (int)Math.Pow(2, (header & 0x3)); + kind = NumberKind.Unsigned; + } + else if (header == MessagePackCode.Real64) + { + length = 9; + kind = NumberKind.Double; + } + else if (header == MessagePackCode.Real32) + { + length = 5; + kind = NumberKind.Single; + } + else + { + MessagePackThrow.IsNotNumber(header, source.Consumed, type); + length = default; + kind = NumberKind.Unknown; + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private static byte ReadByte(ref SequenceReader source, int offset, out int requestHint) + { + if (source.UnreadSpan.Length > offset + 1) + { + var result = source.UnreadSpan[offset]; + source.Advance(offset + 1); + requestHint = 0; + return result; + } + + return ReadByteMultiSegment(ref source, offset, out requestHint); + } + + private static byte ReadByteMultiSegment(ref SequenceReader source, int offset, out int requestHint) + { + if (source.Remaining < offset + 1) + { + requestHint = 1; + return default; + } + + Span buffer = stackalloc byte[offset + 1]; + source.TryCopyTo(buffer); + source.Advance(offset + 1); + requestHint = 0; + return buffer[offset]; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private static sbyte ReadSByte(ref SequenceReader source, int offset, out int requestHint) + => unchecked((sbyte)ReadByte(ref source, offset, out requestHint)); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private static unsafe T ReadValue(ref SequenceReader source, int offset, out int requestHint) + where T : unmanaged + { + if (source.UnreadSpan.Length > offset + sizeof(T)) + { + var result = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(source.UnreadSpan.Slice(offset))); + source.Advance(offset + sizeof(T)); + requestHint = 0; + return result; + } + + return ReadMultiSegment(ref source, offset, out requestHint); + } + + private static unsafe T ReadMultiSegment(ref SequenceReader source, int offset, out int requestHint) + where T : unmanaged + { + if (source.Remaining < offset + sizeof(T)) + { + requestHint = offset + sizeof(T) - (int)source.Remaining; + return default; + } + + Span buffer = stackalloc byte[offset + sizeof(T)]; + source.TryCopyTo(buffer); + source.Advance(offset + sizeof(T)); + requestHint = 0; + return Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(buffer.Slice(offset))); + } + + private enum NumberKind + { + Unknown = 0, + Unsigned = 0x1, + Signed = 0x2, + Half = 0x11, + Single = 0x12, + Double = 0x13, + RealBitMask = 0x10 + } + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoderOptions.cs b/src/MsgPack.Core/Internal/MessagePackDecoderOptions.cs new file mode 100644 index 000000000..c608c8cc4 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoderOptions.cs @@ -0,0 +1,14 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache 2 license. +// See the LICENSE in the project root for more information. + +namespace MsgPack.Internal +{ + public sealed class MessagePackDecoderOptions : FormatDecoderOptions + { + public static MessagePackDecoderOptions Default { get; } = new MessagePackDecoderOptionsBuilder().Build(); + + internal MessagePackDecoderOptions(MessagePackDecoderOptionsBuilder builder) + : base(builder) { } + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackDecoderOptionsBuilder.cs b/src/MsgPack.Core/Internal/MessagePackDecoderOptionsBuilder.cs new file mode 100644 index 000000000..aab974591 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackDecoderOptionsBuilder.cs @@ -0,0 +1,13 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache 2 license. +// See the LICENSE in the project root for more information. + +namespace MsgPack.Internal +{ + public sealed class MessagePackDecoderOptionsBuilder : FormatDecoderOptionsBuilder + { + public MessagePackDecoderOptionsBuilder() { } + + public MessagePackDecoderOptions Build() => new MessagePackDecoderOptions(this); + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackEncoder.Integers.cs b/src/MsgPack.Core/Internal/MessagePackEncoder.Integers.cs new file mode 100644 index 000000000..aa94028cc --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackEncoder.Integers.cs @@ -0,0 +1,190 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + partial class MessagePackEncoder + { + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeInt32(Int32 value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var leastByte = value & 0x000000FF; + if (leastByte == value || (value ^ 0xFFFFFF00) == leastByte) + { + var span = buffer.GetSpan(1); + span[0] = unchecked((byte)(leastByte)); + buffer.Advance(1); + } + else + { + this.EncodeSlow(value, buffer); + } + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeUInt32(UInt32 value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var leastByte = value & 0x000000FF; + if (leastByte == value || (value ^ 0xFFFFFF00) == leastByte) + { + var span = buffer.GetSpan(1); + span[0] = unchecked((byte)(leastByte)); + buffer.Advance(1); + } + else + { + this.EncodeSlow(value, buffer); + } + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeInt64(Int64 value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var leastByte = value & 0x000000FF; + if (leastByte == value || (value ^ 0xFFFFFF00) == leastByte) + { + var span = buffer.GetSpan(1); + span[0] = unchecked((byte)(leastByte)); + buffer.Advance(1); + } + else + { + this.EncodeSlow(value, buffer); + } + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeUInt64(UInt64 value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var leastByte = value & 0x000000FF; + if (leastByte == value || (value ^ 0xFFFFFF00) == leastByte) + { + var span = buffer.GetSpan(1); + span[0] = unchecked((byte)(leastByte)); + buffer.Advance(1); + } + else + { + this.EncodeSlow(value, buffer); + } + } + + private void EncodeSlow(int value, IBufferWriter buffer) + { + if (value >= SByte.MinValue && value <= SByte.MaxValue) + { + // SingedInt8 (SByte) + var span = buffer.GetSpan(2); + span[1] = unchecked((byte)(sbyte)value); + span[0] = MessagePackCode.SignedInt8; + buffer.Advance(2); + } + else if (value >= Int16.MinValue && value <= Int16.MaxValue) + { + // SignedInt16 (Int16) + var span = buffer.GetSpan(3); + span[0] = MessagePackCode.SignedInt16; + span = span.Slice(1); + BinaryPrimitives.WriteInt16BigEndian(span, unchecked((Int16)value)); + buffer.Advance(3); + } + else + { + // SignedInt32 (Int32) + var span = buffer.GetSpan(5); + span[0] = MessagePackCode.SignedInt32; + span = span.Slice(1); + BinaryPrimitives.WriteInt32BigEndian(span, unchecked((Int32)value)); + buffer.Advance(5); + } + } // EncodeSlow(int) + + private void EncodeSlow(uint value, IBufferWriter buffer) + { + if (value <= Byte.MaxValue) + { + // UnsingedInt8 (Byte) + var span = buffer.GetSpan(2); + span[1] = unchecked((byte)value); + span[0] = MessagePackCode.UnsignedInt8; + buffer.Advance(2); + } + else if (value <= UInt16.MaxValue) + { + // UnsignedInt16 (UInt16) + var span = buffer.GetSpan(3); + span[0] = MessagePackCode.UnsignedInt16; + span = span.Slice(1); + BinaryPrimitives.WriteUInt16BigEndian(span, unchecked((UInt16)value)); + buffer.Advance(3); + } + else + { + // UnsignedInt32 (UInt32) + var span = buffer.GetSpan(5); + span[0] = MessagePackCode.UnsignedInt32; + span = span.Slice(1); + BinaryPrimitives.WriteUInt32BigEndian(span, unchecked((UInt32)value)); + buffer.Advance(5); + } + } // EncodeSlow(uint) + + private void EncodeSlow(long value, IBufferWriter buffer) + { + if (value <= Int32.MaxValue && value >= Int32.MinValue) + { + this.EncodeSlow(unchecked((int)value), buffer); + } + else + { + // SignedInt64 (Int64) + var span = buffer.GetSpan(9); + span[0] = MessagePackCode.SignedInt64; + span = span.Slice(1); + BinaryPrimitives.WriteInt64BigEndian(span, unchecked((Int64)value)); + buffer.Advance(9); + } + } // EncodeSlow(long) + + private void EncodeSlow(ulong value, IBufferWriter buffer) + { + if (value <= UInt32.MaxValue) + { + this.EncodeSlow(unchecked((uint)value), buffer); + } + else + { + // UnsignedInt64 (UInt64) + var span = buffer.GetSpan(9); + span[0] = MessagePackCode.UnsignedInt64; + span = span.Slice(1); + BinaryPrimitives.WriteUInt64BigEndian(span, unchecked((UInt64)value)); + buffer.Advance(9); + } + } // EncodeSlow(ulong) + + } +} + diff --git a/src/MsgPack.Core/Internal/MessagePackEncoder.Integers.tt b/src/MsgPack.Core/Internal/MessagePackEncoder.Integers.tt new file mode 100644 index 000000000..4796d7ef4 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackEncoder.Integers.tt @@ -0,0 +1,137 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ output extension=".cs" #> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + partial class MessagePackEncoder + { +<# +foreach (var inputType in new []{ "Int32", "UInt32", "Int64", "UInt64" }) +{ +#> + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void Encode<#= inputType #>(<#= inputType #> value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var leastByte = value & 0x000000FF; + if (leastByte == value || (value ^ 0xFFFFFF00) == leastByte) + { + var span = buffer.GetSpan(1); + span[0] = unchecked((byte)(leastByte)); + buffer.Advance(1); + } + else + { + this.EncodeSlow(value, buffer); + } + } + +<# +} // public int/long +#> + private void EncodeSlow(int value, IBufferWriter buffer) + { + if (value >= SByte.MinValue && value <= SByte.MaxValue) + { + // SingedInt8 (SByte) + var span = buffer.GetSpan(2); + span[1] = unchecked((byte)(sbyte)value); + span[0] = MessagePackCode.SignedInt8; + buffer.Advance(2); + } + else if (value >= Int16.MinValue && value <= Int16.MaxValue) + { +<# + WriteEncodeSlowBody("Int16", sizeof(short) + 1, "SignedInt16"); +#> + } + else + { +<# + WriteEncodeSlowBody("Int32", sizeof(int) + 1, "SignedInt32"); +#> + } + } // EncodeSlow(int) + + private void EncodeSlow(uint value, IBufferWriter buffer) + { + if (value <= Byte.MaxValue) + { + // UnsingedInt8 (Byte) + var span = buffer.GetSpan(2); + span[1] = unchecked((byte)value); + span[0] = MessagePackCode.UnsignedInt8; + buffer.Advance(2); + } + else if (value <= UInt16.MaxValue) + { +<# + WriteEncodeSlowBody("UInt16", sizeof(ushort) + 1, "UnsignedInt16"); +#> + } + else + { +<# + WriteEncodeSlowBody("UInt32", sizeof(uint) + 1, "UnsignedInt32"); +#> + } + } // EncodeSlow(uint) + + private void EncodeSlow(long value, IBufferWriter buffer) + { + if (value <= Int32.MaxValue && value >= Int32.MinValue) + { + this.EncodeSlow(unchecked((int)value), buffer); + } + else + { +<# + WriteEncodeSlowBody("Int64", sizeof(long) + 1, "SignedInt64"); +#> + } + } // EncodeSlow(long) + + private void EncodeSlow(ulong value, IBufferWriter buffer) + { + if (value <= UInt32.MaxValue) + { + this.EncodeSlow(unchecked((uint)value), buffer); + } + else + { +<# + WriteEncodeSlowBody("UInt64", sizeof(ulong) + 1, "UnsignedInt64"); +#> + } + } // EncodeSlow(ulong) + + } +} + +<#+ +private void WriteEncodeSlowBody(string typeName, int size, string code) +{ +#> + // <#= code #> (<#= typeName #>) + var span = buffer.GetSpan(<#= size #>); + span[0] = MessagePackCode.<#= code #>; + span = span.Slice(1); + BinaryPrimitives.Write<#= typeName #>BigEndian(span, unchecked((<#= typeName #>)value)); + buffer.Advance(<#= size #>); +<#+ +} +#> diff --git a/src/MsgPack.Core/Internal/MessagePackEncoder.Strings.cs b/src/MsgPack.Core/Internal/MessagePackEncoder.Strings.cs new file mode 100644 index 000000000..5af1509be --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackEncoder.Strings.cs @@ -0,0 +1,269 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +#nullable enable + +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; + +namespace MsgPack.Internal +{ + partial class MessagePackEncoder + { + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeString(ReadOnlySpan value, IBufferWriter buffer, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + buffer = Ensure.NotNull(buffer); + if (value.Length == 0) + { + buffer.GetSpan(1)[0] = MessagePackCode.MinimumFixedRaw; + buffer.Advance(1); + return; + } + + encoding = encoding ?? Utf8EncodingNonBom.Instance; + + if (value.Length < 256) + { + var valueLength = unchecked((int)value.Length); + var maxByteLength = encoding.GetMaxByteCount(valueLength); + + if (maxByteLength < 32) + { + // stack alloc is fastest + var sink = buffer.GetSpan(maxByteLength + 1); + var actualLength = encoding.GetBytes(value, sink.Slice(1)); + sink[0] = unchecked((byte)(MessagePackCode.MinimumFixedRaw | actualLength)); + buffer.Advance(actualLength + 1); + return; + } + else if (maxByteLength < 256) + { + // stack alloc is fastest + Span encodingBuffer = stackalloc byte[maxByteLength + 3]; + var actualLength = encoding.GetBytes(value, encodingBuffer); + var sink = buffer.GetSpan(3); + var headerLength = this.EncodeStringHeader(unchecked((uint)actualLength), sink); + buffer.Advance(headerLength); + encodingBuffer.Slice(0, actualLength).CopyTo(buffer.GetSpan(actualLength)); + buffer.Advance(actualLength); + return; + } + } + + this.EncodeStringSlow(value, buffer, encoding); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void EncodeStringSlow(ReadOnlySpan value, IBufferWriter buffer, Encoding encoding) + { + // Use 1-path logic borrowed from MessagePack C# + + // Get max char bytes. Use 2 for 0 (maybe bug) or 1 (ASCII) to simplify following logic. + var maxBytesPerChar = Math.Max(encoding.GetMaxByteCount(1), 2); + + // In most case, msgpack uses UTF-8 and clob contains 1 to 3 byte chars, + // and may charactors are ASCII. + // So, estimate with 1char is 2bytes is good enough here. + var estimatedLength = unchecked((uint)value.Length * 2); + var bufferLength = encoding.GetMaxByteCount(unchecked((int)value.Length)) + 5; + + if (bufferLength <= 0x20_0000) // 2MB + { + Memory sinkMemory = buffer.GetMemory(bufferLength); + + if (MemoryMarshal.TryGetArray(sinkMemory, out var sinkBufferSegment)) + { + // We can get single array + buffer.Advance(this.EncodeLargeString1Path(value, buffer, encoding, estimatedLength, sinkBufferSegment)); + } + else + { + // Memory is not array backend, so use local buffer. + var encodingBuffer = base.Options.ByteBufferPool.Rent(bufferLength); + try + { + var totalLength = this.EncodeLargeString1Path(value, buffer, encoding, estimatedLength, encodingBuffer); + buffer.Write(encodingBuffer.AsMemory(0, totalLength).Span); + } + finally + { + base.Options.ByteBufferPool.Return(encodingBuffer, base.Options.ClearsBuffer); + } + } + } + + this.EncodeHugeString2Path(value, buffer, encoding); + } + + + private unsafe int EncodeLargeString1Path(ReadOnlySpan value, IBufferWriter buffer, Encoding encoding, uint estimatedLength, ArraySegment sinkBufferSegment) + { + ref var head = ref sinkBufferSegment.Array![sinkBufferSegment.Offset]; + + Span sinkBuffer = sinkBufferSegment; + var estimatedHeaderLength = this.EncodeStringHeader(estimatedLength, sinkBuffer); + + int actualLength; + + try + { + actualLength = encoding.GetBytes(value, sinkBuffer.Slice(estimatedHeaderLength)); + } + catch (OverflowException ex) + { + Throw.TooLargeByteLength(ex, encoding.EncodingName); + return 0; + } + + var realHeaderLength = this.EncodeStringHeader(unchecked((uint)actualLength), sinkBuffer); + + if (estimatedHeaderLength != realHeaderLength) + { + fixed (byte* pSinkBuffer = &head) + { + Buffer.MemoryCopy((void*)(pSinkBuffer + estimatedHeaderLength), (void*)(pSinkBuffer + realHeaderLength), actualLength, actualLength); + } + } + + return actualLength + realHeaderLength; + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeString(in ReadOnlySequence value, IBufferWriter buffer, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + buffer = Ensure.NotNull(buffer); + if (value.Length > UInt32.MaxValue) + { + Throw.TooLargeCharLength(value.Length); + return; + } + + if (value.Length == 0) + { + buffer.GetSpan(1)[0] = MessagePackCode.MinimumFixedRaw; + buffer.Advance(1); + return; + } + + encoding = encoding ?? Utf8EncodingNonBom.Instance; + + if (value.Length < 256) + { + var valueLength = unchecked((int)value.Length); + var maxByteLength = encoding.GetMaxByteCount(valueLength); + + if (maxByteLength < 32) + { + // stack alloc is fastest + var sink = buffer.GetSpan(maxByteLength + 1); + var actualLength = encoding.GetBytes(value, sink.Slice(1)); + sink[0] = unchecked((byte)(MessagePackCode.MinimumFixedRaw | actualLength)); + buffer.Advance(actualLength + 1); + return; + } + else if (maxByteLength < 256) + { + // stack alloc is fastest + Span encodingBuffer = stackalloc byte[maxByteLength + 3]; + var actualLength = encoding.GetBytes(value, encodingBuffer); + var sink = buffer.GetSpan(3); + var headerLength = this.EncodeStringHeader(unchecked((uint)actualLength), sink); + buffer.Advance(headerLength); + encodingBuffer.Slice(0, actualLength).CopyTo(buffer.GetSpan(actualLength)); + buffer.Advance(actualLength); + return; + } + } + + this.EncodeStringSlow(value, buffer, encoding); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void EncodeStringSlow(in ReadOnlySequence value, IBufferWriter buffer, Encoding encoding) + { + // Use 1-path logic borrowed from MessagePack C# + + // Get max char bytes. Use 2 for 0 (maybe bug) or 1 (ASCII) to simplify following logic. + var maxBytesPerChar = Math.Max(encoding.GetMaxByteCount(1), 2); + + // In most case, msgpack uses UTF-8 and clob contains 1 to 3 byte chars, + // and may charactors are ASCII. + // So, estimate with 1char is 2bytes is good enough here. + var estimatedLength = unchecked((uint)value.Length * 2); + var bufferLength = encoding.GetMaxByteCount(unchecked((int)value.Length)) + 5; + + if (bufferLength <= 0x20_0000) // 2MB + { + Memory sinkMemory = buffer.GetMemory(bufferLength); + + if (MemoryMarshal.TryGetArray(sinkMemory, out var sinkBufferSegment)) + { + // We can get single array + buffer.Advance(this.EncodeLargeString1Path(value, buffer, encoding, estimatedLength, sinkBufferSegment)); + } + else + { + // Memory is not array backend, so use local buffer. + var encodingBuffer = base.Options.ByteBufferPool.Rent(bufferLength); + try + { + var totalLength = this.EncodeLargeString1Path(value, buffer, encoding, estimatedLength, encodingBuffer); + buffer.Write(encodingBuffer.AsMemory(0, totalLength).Span); + } + finally + { + base.Options.ByteBufferPool.Return(encodingBuffer, base.Options.ClearsBuffer); + } + } + } + + this.EncodeHugeString2Path(value, buffer, encoding); + } + + private unsafe int EncodeLargeString1Path(in ReadOnlySequence value, IBufferWriter buffer, Encoding encoding, uint estimatedLength, ArraySegment sinkBufferSegment) + { + ref var head = ref sinkBufferSegment.Array![sinkBufferSegment.Offset]; + + Span sinkBuffer = sinkBufferSegment; + var estimatedHeaderLength = this.EncodeStringHeader(estimatedLength, sinkBuffer); + + int actualLength; + + try + { + actualLength = encoding.GetBytes(value, sinkBuffer.Slice(estimatedHeaderLength)); + } + catch (OverflowException ex) + { + Throw.TooLargeByteLength(ex, encoding.EncodingName); + return 0; + } + + var realHeaderLength = this.EncodeStringHeader(unchecked((uint)actualLength), sinkBuffer); + + if (estimatedHeaderLength != realHeaderLength) + { + fixed (byte* pSinkBuffer = &head) + { + Buffer.MemoryCopy((void*)(pSinkBuffer + estimatedHeaderLength), (void*)(pSinkBuffer + realHeaderLength), actualLength, actualLength); + } + } + + return actualLength + realHeaderLength; + } + + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackEncoder.Strings.tt b/src/MsgPack.Core/Internal/MessagePackEncoder.Strings.tt new file mode 100644 index 000000000..eb7638a19 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackEncoder.Strings.tt @@ -0,0 +1,172 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ output extension=".cs" #> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +#nullable enable + +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; + +namespace MsgPack.Internal +{ + partial class MessagePackEncoder + { +<# +foreach (var inputType in + new [] + { + new { Signature = "ReadOnlySpan", MayBe64BitLength = false }, + new { Signature = "in ReadOnlySequence" , MayBe64BitLength = true }, + } +) +{ +#> + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeString(<#= inputType.Signature #> value, IBufferWriter buffer, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + buffer = Ensure.NotNull(buffer); +<# + if (inputType.MayBe64BitLength) + { +#> + if (value.Length > UInt32.MaxValue) + { + Throw.TooLargeCharLength(value.Length); + return; + } + +<# + } +#> + if (value.Length == 0) + { + buffer.GetSpan(1)[0] = MessagePackCode.MinimumFixedRaw; + buffer.Advance(1); + return; + } + + encoding = encoding ?? Utf8EncodingNonBom.Instance; + + if (value.Length < 256) + { + var valueLength = unchecked((int)value.Length); + var maxByteLength = encoding.GetMaxByteCount(valueLength); + + if (maxByteLength < 32) + { + // stack alloc is fastest + var sink = buffer.GetSpan(maxByteLength + 1); + var actualLength = encoding.GetBytes(value, sink.Slice(1)); + sink[0] = unchecked((byte)(MessagePackCode.MinimumFixedRaw | actualLength)); + buffer.Advance(actualLength + 1); + return; + } + else if (maxByteLength < 256) + { + // stack alloc is fastest + Span encodingBuffer = stackalloc byte[maxByteLength + 3]; + var actualLength = encoding.GetBytes(value, encodingBuffer); + var sink = buffer.GetSpan(3); + var headerLength = this.EncodeStringHeader(unchecked((uint)actualLength), sink); + buffer.Advance(headerLength); + encodingBuffer.Slice(0, actualLength).CopyTo(buffer.GetSpan(actualLength)); + buffer.Advance(actualLength); + return; + } + } + + this.EncodeStringSlow(value, buffer, encoding); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void EncodeStringSlow(<#= inputType.Signature #> value, IBufferWriter buffer, Encoding encoding) + { + // Use 1-path logic borrowed from MessagePack C# + + // Get max char bytes. Use 2 for 0 (maybe bug) or 1 (ASCII) to simplify following logic. + var maxBytesPerChar = Math.Max(encoding.GetMaxByteCount(1), 2); + + // In most case, msgpack uses UTF-8 and clob contains 1 to 3 byte chars, + // and may charactors are ASCII. + // So, estimate with 1char is 2bytes is good enough here. + var estimatedLength = unchecked((uint)value.Length * 2); + var bufferLength = encoding.GetMaxByteCount(unchecked((int)value.Length)) + 5; + + if (bufferLength <= 0x20_0000) // 2MB + { + Memory sinkMemory = buffer.GetMemory(bufferLength); + + if (MemoryMarshal.TryGetArray(sinkMemory, out var sinkBufferSegment)) + { + // We can get single array + buffer.Advance(this.EncodeLargeString1Path(value, buffer, encoding, estimatedLength, sinkBufferSegment)); + } + else + { + // Memory is not array backend, so use local buffer. + var encodingBuffer = base.Options.ByteBufferPool.Rent(bufferLength); + try + { + var totalLength = this.EncodeLargeString1Path(value, buffer, encoding, estimatedLength, encodingBuffer); + buffer.Write(encodingBuffer.AsMemory(0, totalLength).Span); + } + finally + { + base.Options.ByteBufferPool.Return(encodingBuffer, base.Options.ClearsBuffer); + } + } + } + + this.EncodeHugeString2Path(value, buffer, encoding); + } + + + private unsafe int EncodeLargeString1Path(<#= inputType.Signature #> value, IBufferWriter buffer, Encoding encoding, uint estimatedLength, ArraySegment sinkBufferSegment) + { + ref var head = ref sinkBufferSegment.Array![sinkBufferSegment.Offset]; + + Span sinkBuffer = sinkBufferSegment; + var estimatedHeaderLength = this.EncodeStringHeader(estimatedLength, sinkBuffer); + + int actualLength; + + try + { + actualLength = encoding.GetBytes(value, sinkBuffer.Slice(estimatedHeaderLength)); + } + catch (OverflowException ex) + { + Throw.TooLargeByteLength(ex, encoding.EncodingName); + return 0; + } + + var realHeaderLength = this.EncodeStringHeader(unchecked((uint)actualLength), sinkBuffer); + + if (estimatedHeaderLength != realHeaderLength) + { + fixed (byte* pSinkBuffer = &head) + { + Buffer.MemoryCopy((void*)(pSinkBuffer + estimatedHeaderLength), (void*)(pSinkBuffer + realHeaderLength), actualLength, actualLength); + } + } + + return actualLength + realHeaderLength; + } + +<# +} +#> + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackEncoder.cs b/src/MsgPack.Core/Internal/MessagePackEncoder.cs new file mode 100644 index 000000000..b3647aac9 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackEncoder.cs @@ -0,0 +1,253 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; + +namespace MsgPack.Internal +{ + /// + /// Common encoder implementation between legacy and current MessagePack format enconders. + /// + public abstract partial class MessagePackEncoder : FormatEncoder + { +#warning TODO: Can devirt? If not, change to Create(MessagePackEncoderOptions) + public static MessagePackEncoder CreateLegacy(MessagePackEncoderOptions options) => new LegacyMessagePackEncoder(options); + public static MessagePackEncoder CreateCurrent(MessagePackEncoderOptions options) => new CurrentMessagePackEncoder(options); + + public new MessagePackEncoderOptions Options => (base.Options as MessagePackEncoderOptions)!; + + protected MessagePackEncoder(MessagePackEncoderOptions options) + : base(options) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeRawString(ReadOnlySpan rawString, int charLength, IBufferWriter buffer, CancellationToken cancellationToken = default) + => Ensure.NotNull(buffer).Write(rawString); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeSingle(float value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var span = buffer.GetSpan(sizeof(float) + 1); + + span[0] = MessagePackCode.Real32; + span = span.Slice(1); + BinaryPrimitives.WriteInt32BigEndian(span, BitConverter.SingleToInt32Bits(value)); + buffer.Advance(sizeof(float) + 1); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeDouble(double value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var span = buffer.GetSpan(sizeof(double) + 1); + + span[0] = MessagePackCode.Real64; + span = span.Slice(1); + BinaryPrimitives.WriteInt64BigEndian(span, BitConverter.DoubleToInt64Bits(value)); + buffer.Advance(sizeof(double) + 1); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeBoolean(bool value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var span = buffer.GetSpan(1); + span[0] = unchecked((byte)(value ? MessagePackCode.TrueValue : MessagePackCode.FalseValue)); + buffer.Advance(1); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeArrayStart(int length, IBufferWriter buffer, in CollectionContext collectionContext) + { + buffer = Ensure.NotNull(buffer); + collectionContext.IncrementDepth(); + + if (length < 16) + { + var span = buffer.GetSpan(1); + span[0] = unchecked((byte)(MessagePackCode.MinimumFixedArray | length)); + buffer.Advance(1); + } + else if (length < 16) + { + var span = buffer.GetSpan(sizeof(ushort) + 1); + span[0] = MessagePackCode.Array16; + span = span.Slice(1); + BinaryPrimitives.WriteUInt16BigEndian(span, unchecked((ushort)length)); + buffer.Advance(sizeof(ushort) + 1); + } + else + { + var span = buffer.GetSpan(sizeof(uint) + 1); + span[0] = MessagePackCode.Array32; + span = span.Slice(1); + BinaryPrimitives.WriteInt32BigEndian(span, length); + buffer.Advance(sizeof(uint) + 1); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeArrayEnd(int length, IBufferWriter buffer, in CollectionContext collectionContext) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeArrayItemStart(int index, IBufferWriter buffer, in CollectionContext collectionContext) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeArrayItemEnd(int index, IBufferWriter buffer, in CollectionContext collectionContext) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeMapStart(int length, IBufferWriter buffer, in CollectionContext collectionContext) + { + buffer = Ensure.NotNull(buffer); + + if (length < 16) + { + var span = buffer.GetSpan(1); + span[0] = unchecked((byte)(MessagePackCode.MinimumFixedMap | length)); + buffer.Advance(1); + } + else if (length < 16) + { + var span = buffer.GetSpan(sizeof(ushort) + 1); + span[0] = MessagePackCode.Map16; + span = span.Slice(1); + BinaryPrimitives.WriteUInt16BigEndian(span, unchecked((ushort)length)); + buffer.Advance(sizeof(ushort) + 1); + } + else + { + var span = buffer.GetSpan(sizeof(uint) + 1); + span[0] = MessagePackCode.Map32; + span = span.Slice(1); + BinaryPrimitives.WriteInt32BigEndian(span, length); + buffer.Advance(sizeof(uint) + 1); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeMapEnd(int length, IBufferWriter buffer, in CollectionContext collectionContext) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeMapKeyStart(int index, IBufferWriter buffer, in CollectionContext collectionContext) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeMapKeyEnd(int index, IBufferWriter buffer, in CollectionContext collectionContext) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeMapValueStart(int index, IBufferWriter buffer, in CollectionContext collectionContext) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeMapValueEnd(int index, IBufferWriter buffer, in CollectionContext collectionContext) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeNull(IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var span = buffer.GetSpan(1); + span[0] = MessagePackCode.NilValue; + buffer.Advance(1); + } + + private void EncodeHugeString2Path(ReadOnlySpan value, IBufferWriter buffer, Encoding encoding) + { + // Huge path, we cannot use rollback because it requires multiple span from buffer. + int actualLength; + try + { + actualLength = encoding.GetByteCount(value); + } + catch(OverflowException ex) + { + Throw.TooLargeByteLength(ex, encoding.EncodingName); + return; + } + + var span = buffer.GetSpan(5); + var headerLength = this.EncodeStringHeader(unchecked((uint)actualLength), span); + buffer.Advance(headerLength); + encoding.GetBytes(value, buffer); + } + + private void EncodeHugeString2Path(in ReadOnlySequence value, IBufferWriter buffer, Encoding encoding) + { + // Huge path, we cannot use rollback because it requires multiple span from buffer. + var actualLength = 0L; + var reader = new SequenceReader(value); + while (!reader.End) + { + actualLength += encoding.GetByteCount(reader.UnreadSpan); + reader.Advance(reader.UnreadSpan.Length); + } + + if (actualLength > UInt32.MaxValue) + { + Throw.TooLargeByteLength(actualLength, encoding.EncodingName); + } + + var span = buffer.GetSpan(5); + var headerLength = this.EncodeStringHeader(unchecked((uint)actualLength), span); + buffer.Advance(headerLength); + encoding.GetBytes(value, buffer); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeString(ReadOnlySpan encodedValue, int charLength, IBufferWriter buffer, CancellationToken cancellationToken = default) + { + buffer = Ensure.NotNull(buffer); + + var span = buffer.GetSpan(5); + var used = this.EncodeStringHeader(unchecked((uint)encodedValue.Length), span); + buffer.Advance(used); + this.WriteRaw(encodedValue, buffer, cancellationToken); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private void EncodeStringHeader(uint length, Memory memory) + => this.EncodeStringHeader(length, memory.Span); + + protected abstract int EncodeStringHeader(uint length, Span buffer); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeString(in ReadOnlySequence encodedValue, int charLength, IBufferWriter buffer, CancellationToken cancellationToken = default) + { + var span = buffer.GetSpan(5); + var used = this.EncodeStringHeader(unchecked((uint)encodedValue.Length), span); + buffer.Advance(used); + this.WriteRaw(encodedValue, buffer, cancellationToken); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeBinary(ReadOnlySpan value, IBufferWriter buffer, CancellationToken cancellationToken = default) + { + var span = buffer.GetSpan(5); + var used = this.EncodeBinaryHeader(unchecked((uint)value.Length), span); + buffer.Advance(used); + this.WriteRaw(value, buffer, cancellationToken); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeBinary(in ReadOnlySequence value, IBufferWriter buffer, CancellationToken cancellationToken) + { + var span = buffer.GetSpan(5); + var used = this.EncodeBinaryHeader(unchecked((uint)value.Length), span); + buffer.Advance(used); + this.WriteRaw(value, buffer, cancellationToken); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private int EncodeBinaryHeader(uint length, Memory memory) + => this.EncodeBinaryHeader(length, memory.Span); + + protected abstract int EncodeBinaryHeader(uint length, Span buffer); + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackEncoderOptions.cs b/src/MsgPack.Core/Internal/MessagePackEncoderOptions.cs new file mode 100644 index 000000000..a36d7e9ed --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackEncoderOptions.cs @@ -0,0 +1,13 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +namespace MsgPack.Internal +{ + public sealed class MessagePackEncoderOptions : FormatEncoderOptions + { + public static MessagePackEncoderOptions Default { get; } = new MessagePackEncoderOptionsBuilder().Build(); + + public MessagePackEncoderOptions(MessagePackEncoderOptionsBuilder builder) : base(builder) { } + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackEncoderOptionsBuilder.cs b/src/MsgPack.Core/Internal/MessagePackEncoderOptionsBuilder.cs new file mode 100644 index 000000000..7ceeba6a1 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackEncoderOptionsBuilder.cs @@ -0,0 +1,13 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +namespace MsgPack.Internal +{ + public sealed class MessagePackEncoderOptionsBuilder : FormatEncoderOptionsBuilder + { + public MessagePackEncoderOptionsBuilder() { } + + public MessagePackEncoderOptions Build() => new MessagePackEncoderOptions(this); + } +} diff --git a/src/MsgPack.Core/Internal/MessagePackThrow.cs b/src/MsgPack.Core/Internal/MessagePackThrow.cs new file mode 100644 index 000000000..7e1665bd5 --- /dev/null +++ b/src/MsgPack.Core/Internal/MessagePackThrow.cs @@ -0,0 +1,48 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache 2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Runtime.CompilerServices; + +namespace MsgPack.Internal +{ + internal static class MessagePackThrow + { + public static void IsNotType(byte header, long position, Type requestType) + => throw new MessageTypeException($"The type is {MessagePackCode.ToString(header)}(0x{header:X2}) but {requestType} is required at {position:#,0}."); + + public static void RealCannotBeInteger(byte header, long position, Type requestType) + => throw new MessageTypeException($"The type is {MessagePackCode.ToString(header)}(0x{header:X2}) is not compatible for {requestType} in current configuration at {position:#,0}."); + + public static void TypeIsNotArray(byte header, long position) + => throw new MessageTypeException($"The type is {MessagePackCode.ToString(header)}(0x{header:X2}) but array is required at {position:#,0}."); + + public static void TypeIsNotMap(byte header, long position) + => throw new MessageTypeException($"The type is {MessagePackCode.ToString(header)}(0x{header:X2}) but map is required at {position:#,0}."); + + public static void TypeIsNotArrayNorMap(byte header, long position) + => throw new MessageTypeException($"The type is {MessagePackCode.ToString(header)}(0x{header:X2}) but array or map is required at {position:#,0}."); + + public static void IsNotNumber(byte header, long position, Type requestType) + => throw new MessageTypeException($"The type is {MessagePackCode.ToString(header)}(0x{header:X2}) but it is not compatible for {requestType} at {position:#,0}."); + + public static void TooLargeByteLength(byte header, long position, long byteLength) + => throw new MessageTypeException($"The length of string {MessagePackCode.ToString(header)}(0x{header:X2}) {byteLength:#,0}(0x{byteLength:X8}) exceeds Int32.MaxValue (0x7FFFFFFF) at {position:#,0}."); + + public static void TooLargeArrayOrMapLength(byte header, long position, long byteLength) + => throw new MessageTypeException($"The length of array or map {MessagePackCode.ToString(header)}(0x{header:X2}) {byteLength:#,0}(0x{byteLength:X8}) exceeds Int32.MaxValue (0x7FFFFFFF) at {position:#,0}."); + + public static void OutOfRangeExtensionTypeCode(int typeCode, [CallerArgumentExpression("typeCode")] string? paramName = default) + => throw new ArgumentOutOfRangeException(paramName, $"A type code of MessagePack must be non negative 1byte integer (between 0 to 127 inclusive). '{typeCode}' is too large."); + + public static void IsNotUtf8String(byte header, long position) + => throw new MessageTypeException($"The type is {MessagePackCode.ToString(header)}(0x{header:X2}) but it is not compatible for UTF8String at {position:#,0}."); + + public static void IsNotExtension(byte header, long position) + => throw new MessageTypeException($"The type is {MessagePackCode.ToString(header)}(0x{header:X2}) but it is not extension at {position:#,0}."); + + public static void InvalidTypeCode(ulong typeCode) + => throw new ArgumentOutOfRangeException("typeCode", $"Extension type code for MessagePack must be 1 byte, but 0x{typeCode:X} is specified."); + } +} diff --git a/src/MsgPack.Core/Internal/MsgPackStringTrie`1.cs b/src/MsgPack.Core/Internal/MsgPackStringTrie`1.cs new file mode 100644 index 000000000..18bb5f3e1 --- /dev/null +++ b/src/MsgPack.Core/Internal/MsgPackStringTrie`1.cs @@ -0,0 +1,621 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace MsgPack.Internal +{ + // Basic idea is borrwed from AutomataDictionary of Message Pack C# + // https://github.com/neuecc/MessagePack-CSharp/blob/51649e0d7b8641ad5d3cdd6dfdc130c7671066fc/src/MessagePack.UnityClient/Assets/Scripts/MessagePack/Internal/AutomataDictionary.cs#L1 + +#warning TODO: Pubternal + public sealed class MsgPackStringTrie + { + private readonly T _default; + private readonly Node _root; + + public MsgPackStringTrie(T defaultValue) + { + this._default = defaultValue; + this._root = new Node(defaultValue, Array.Empty(), Array.Empty()); + } + + public bool TryAdd(ReadOnlySpan utf8Key, T value) + => this.TryAdd(this._root, MsgPackStringTrieKey.GetAsMsgPackString(ref utf8Key), utf8Key, value); + + public bool TryAddRaw(ReadOnlySpan utf8Key, T value) + => this.TryAdd(this._root, MsgPackStringTrieKey.GetRaw64(ref utf8Key), utf8Key, value); + + private static int BinarySearch(ulong[] nodes, ulong key) + { + // Span.BinarySearch is slower maybe because of ComapreTo method overhead, so we implement binary search manually. + var low = 0; + var high = nodes.Length - 1; + while (low <= high) + { + // Peformance trick borrowed from https://github.com/dotnet/runtime/blob/f2786223508c0c70040fbf48ec3a39a607dd7f75/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.BinarySearch.cs#L42 + var index = unchecked((int)(((uint)high + (uint)low) >> 1)); + var found = nodes[index]; + if (found == key) + { + return index; + } + else if (found < key) + { + low = index + 1; + } + else + { + high = index - 1; + } + } + + return ~low; + } + + public T GetOrDefault(ReadOnlySpan msgPackStringKey) + { + var result = this.Find(this._root, MsgPackStringTrieKey.GetRaw64(ref msgPackStringKey), msgPackStringKey); + return result != null ? result.Value : this._default; + } + + private bool TryAdd(Node parent, ulong key, ReadOnlySpan trailingKey, T value) + { + var nodes = parent.ChildNodes; + var keys = parent.ChildKeys; + + while (true) + { + var index = BinarySearch(keys, key); + + if (index < 0) + { + // No matching leaf. + this.AddNode(parent, key, trailingKey, ~index, value, out _); + return true; + } + + var found = nodes[index]; + if (trailingKey.IsEmpty) + { + // The leaf matches. + return false; + } + + if (found.ChildNodes.Length == 0) + { + // Search key is longer than trie path. + this.AddNode(found, key, trailingKey, 0, value, out _); + return true; + } + + nodes = found.ChildNodes; + keys = found.ChildKeys; + + key = MsgPackStringTrieKey.GetRaw64(ref trailingKey); + } + } + + private Node? Find(Node parent, ulong key, ReadOnlySpan trailingKey) + { + var nodes = parent.ChildNodes; + var keys = parent.ChildKeys; + + while (true) + { + var index = BinarySearch(keys, key); + + if (index < 0) + { + // No matching leaf. + return null; + } + + var found = nodes[index]; + if (trailingKey.IsEmpty) + { + // The leaf matches. + return found; + } + + if (found.ChildNodes.Length == 0) + { + // Search key is longer than trie path. + return null; + } + + nodes = found.ChildNodes; + keys = found.ChildKeys; + key = MsgPackStringTrieKey.GetRaw64(ref trailingKey); + } + } + + private void AddNode(Node node, ulong key, ReadOnlySpan trailingKey, int targetIndex, T value, out Node result) + { + Array.Resize(ref node.ChildKeys, node.ChildKeys.Length + 1); + Array.Resize(ref node.ChildNodes, node.ChildNodes.Length + 1); + var nodes = node.ChildNodes; + var keys = node.ChildKeys; + + while (true) + { + var child = + trailingKey.IsEmpty ? + new Node(value, Array.Empty(), Array.Empty()) : + new Node(this._default, new Node[1], new ulong[1]); + + if (nodes.Length > 1 && targetIndex < nodes.Length - 1) + { + // Shift existing. + Array.Copy(nodes, targetIndex, nodes, targetIndex + 1, nodes.Length - targetIndex - 1); + Array.Copy(keys, targetIndex, keys, targetIndex + 1, keys.Length - targetIndex - 1); + } + + nodes[targetIndex] = child; + keys[targetIndex] = key; + + if (trailingKey.IsEmpty) + { + // This is leaf. + result = child; + return; + } + + nodes = child.ChildNodes; + keys = child.ChildKeys; + + Debug.Assert(nodes.Length == 1); + Debug.Assert(keys.Length == 1); + targetIndex = 0; + + key = MsgPackStringTrieKey.GetRaw64(ref trailingKey); + } + } + + private sealed class Node + { + // There 2 separate array to improve search performance due to CPU cache line and prediction. + public ulong[] ChildKeys; + public Node[] ChildNodes; + public readonly T Value; + + public Node(T value, Node[] childNodes, ulong[] childKeys) + { + this.Value = value; + this.ChildNodes = childNodes; + this.ChildKeys = childKeys; + } + } + +#warning TODO: REMOVE + public IEnumerable<(ulong[] Keys, T Value)> GetDebugView() + => GetDebugView(this._root, new List()); + + private static IEnumerable<(ulong[] Keys, T Value)> GetDebugView(Node node, List keyChain) + { + if (node.ChildKeys.Length == 0) + { + yield return (keyChain.ToArray(), node.Value); + } + else + { + for (var i = 0; i < node.ChildKeys.Length; i++) + { + var childKey = node.ChildKeys[i]; + keyChain.Add(childKey); + var childNode = node.ChildNodes[i]; + foreach (var item in GetDebugView(childNode, keyChain)) + { + yield return item; + } + keyChain.RemoveAt(keyChain.Count - 1); + } + } + } + } + +#if FALSE + public sealed class MsgPackStringTrie32 + { + private readonly T _default; + private readonly Node _root; + + public MsgPackStringTrie32(T defaultValue) + { + this._default = defaultValue; + this._root = new Node(defaultValue, Array.Empty(), Array.Empty()); + } + + public bool TryAdd(ReadOnlySpan utf8Key, T value) + => this.TryAdd(this._root, MsgPackStringTrieKey.GetAsMsgPackString32(ref utf8Key), utf8Key, value); + + private static int BinarySearch(uint[] nodes, ulong key) + { + // Span.BinarySearch is slower maybe because of ComapreTo method overhead, so we implement binary search manually. + var low = 0; + var high = nodes.Length - 1; + while (low <= high) + { + // Peformance trick borrowed from https://github.com/dotnet/runtime/blob/f2786223508c0c70040fbf48ec3a39a607dd7f75/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.BinarySearch.cs#L42 + var index = unchecked((int)(((uint)high + (uint)low) >> 1)); + var found = nodes[index]; + if (found == key) + { + return index; + } + else if (found < key) + { + low = index + 1; + } + else + { + high = index - 1; + } + } + + return ~low; + } + + public T GetOrDefault(ReadOnlySpan msgPackStringKey) + { + var result = this.Find(this._root, MsgPackStringTrieKey.GetRaw32(ref msgPackStringKey), msgPackStringKey); + return result != null ? result.Value : this._default; + } + + private bool TryAdd(Node parent, uint key, ReadOnlySpan trailingKey, T value) + { + var nodes = parent.ChildNodes; + var keys = parent.ChildKeys; + + while (true) + { + var index = BinarySearch(keys, key); + + if (index < 0) + { + // No matching leaf. + this.AddNode(parent, key, trailingKey, ~index, value, out _); + return true; + } + + var found = nodes[index]; + if (trailingKey.IsEmpty) + { + // The leaf matches. + return false; + } + + if (found.ChildNodes.Length == 0) + { + // Search key is longer than trie path. + this.AddNode(found, key, trailingKey, 0, value, out _); + return true; + } + + nodes = found.ChildNodes; + keys = found.ChildKeys; + + key = MsgPackStringTrieKey.GetRaw32(ref trailingKey); + } + } + + private Node? Find(Node parent, uint key, ReadOnlySpan trailingKey) + { + var nodes = parent.ChildNodes; + var keys = parent.ChildKeys; + + while (true) + { + var index = BinarySearch(keys, key); + + if (index < 0) + { + // No matching leaf. + return null; + } + + var found = nodes[index]; + if (trailingKey.IsEmpty) + { + // The leaf matches. + return found; + } + + if (found.ChildNodes.Length == 0) + { + // Search key is longer than trie path. + return null; + } + + nodes = found.ChildNodes; + keys = found.ChildKeys; + key = MsgPackStringTrieKey.GetRaw32(ref trailingKey); + } + } + + private void AddNode(Node node, uint key, ReadOnlySpan trailingKey, int targetIndex, T value, out Node result) + { + Array.Resize(ref node.ChildKeys, node.ChildKeys.Length + 1); + Array.Resize(ref node.ChildNodes, node.ChildNodes.Length + 1); + var nodes = node.ChildNodes; + var keys = node.ChildKeys; + + while (true) + { + var child = + trailingKey.IsEmpty ? + new Node(value, Array.Empty(), Array.Empty()) : + new Node(this._default, new Node[1], new uint[1]); + + if (nodes.Length > 1 && targetIndex < nodes.Length - 1) + { + // Shift existing. + Array.Copy(nodes, targetIndex, nodes, targetIndex + 1, nodes.Length - targetIndex - 1); + Array.Copy(keys, targetIndex, keys, targetIndex + 1, keys.Length - targetIndex - 1); + } + + nodes[targetIndex] = child; + keys[targetIndex] = key; + + if (trailingKey.IsEmpty) + { + // This is leaf. + result = child; + return; + } + + nodes = child.ChildNodes; + keys = child.ChildKeys; + + Debug.Assert(nodes.Length == 1); + Debug.Assert(keys.Length == 1); + targetIndex = 0; + + key = MsgPackStringTrieKey.GetRaw32(ref trailingKey); + } + } + + private sealed class Node + { + // There 2 separate array to improve search performance due to CPU cache line and prediction. + public uint[] ChildKeys; + public Node[] ChildNodes; + public readonly T Value; + + public Node(T value, Node[] childNodes, uint[] childKeys) + { + this.Value = value; + this.ChildNodes = childNodes; + this.ChildKeys = childKeys; + } + } + +#warning TODO: REMOVE + public IEnumerable<(uint[] Keys, T Value)> GetDebugView() + => GetDebugView(this._root, new List()); + + private static IEnumerable<(uint[] Keys, T Value)> GetDebugView(Node node, List keyChain) + { + if (node.ChildKeys.Length == 0) + { + yield return (keyChain.ToArray(), node.Value); + } + else + { + for (var i = 0; i < node.ChildKeys.Length; i++) + { + var childKey = node.ChildKeys[i]; + keyChain.Add(childKey); + var childNode = node.ChildNodes[i]; + foreach (var item in GetDebugView(childNode, keyChain)) + { + yield return item; + } + keyChain.RemoveAt(keyChain.Count - 1); + } + } + } + } +#endif + +#warning TODO: Pubternal + public static class MsgPackStringTrieKey + { + public static ulong GetAsMsgPackString(ref ReadOnlySpan source) + { + Span bytes = stackalloc byte[sizeof(ulong)]; + var buffer = bytes; + int consumes; + if (source.Length < 16) + { + buffer[0] = (byte)(0xA0 | source.Length); + buffer = buffer.Slice(1); + consumes = Math.Min(source.Length, sizeof(long) - 1); + } + else if (source.Length <= Byte.MaxValue) + { + buffer[0] = 0xD9; + buffer[1] = (byte)source.Length; + buffer = buffer.Slice(2); + consumes = Math.Min(source.Length, sizeof(long) - 1 - 1); + } + else if (source.Length <= UInt16.MaxValue) + { + buffer[0] = 0xDA; + buffer = buffer.Slice(1); + BinaryPrimitives.WriteUInt16BigEndian(buffer, (ushort)source.Length); + buffer = buffer.Slice(sizeof(ushort)); + consumes = Math.Min(source.Length, sizeof(long) - 1 - sizeof(ushort)); + } + else + { + buffer[0] = 0xDB; + buffer = buffer.Slice(1); + BinaryPrimitives.WriteInt32BigEndian(buffer, source.Length); + buffer = buffer.Slice(sizeof(int)); + consumes = Math.Min(source.Length, sizeof(long) - 1 - sizeof(int)); + } + + source.Slice(0, consumes).CopyTo(buffer); + source = source.Slice(consumes); + return MemoryMarshal.Cast(bytes)[0]; + } + + public static ulong GetRaw64(ref ReadOnlySpan source) + { + Debug.Assert(!source.IsEmpty); + if (source.Length >= sizeof(ulong)) + { + var result = MemoryMarshal.Cast(source)[0]; + source = source.Slice(sizeof(ulong)); + return result; + } + else + { + ulong result; + unchecked + { + switch (source.Length) + { + case 1: + { + result = source[0]; + break; + } + case 2: + { + result = BinaryPrimitives.ReadUInt16LittleEndian(source); + break; + } + case 3: + { + result = BinaryPrimitives.ReadUInt16LittleEndian(source); + source = source.Slice(2); + result |= (uint)(source[0] << 16); + break; + } + case 4: + { + result = BinaryPrimitives.ReadUInt32LittleEndian(source); + break; + } + case 5: + { + result = BinaryPrimitives.ReadUInt32LittleEndian(source); + source = source.Slice(4); + result |= ((ulong)source[0] << 32); + break; + } + case 6: + { + result = BinaryPrimitives.ReadUInt32LittleEndian(source); + source = source.Slice(4); + result |= ((ulong)BinaryPrimitives.ReadUInt16LittleEndian(source) << 32); + break; + } + default: // 7 + { + result = BinaryPrimitives.ReadUInt32LittleEndian(source); + source = source.Slice(4); + result |= ((ulong)BinaryPrimitives.ReadUInt16LittleEndian(source) << 32); + source = source.Slice(2); + result |= ((ulong)source[0] << 48); + break; + } + } + } + + source = ReadOnlySpan.Empty; + return result; + } + } + +#if FALSE + public static uint GetAsMsgPackString32(ref ReadOnlySpan source) + { + Span bytes = stackalloc byte[sizeof(uint)]; + var buffer = bytes; + int consumes; + if (source.Length < 16) + { + buffer[0] = (byte)(0xA0 | source.Length); + buffer = buffer.Slice(1); + consumes = Math.Min(source.Length, sizeof(uint) - 1); + } + else if (source.Length <= Byte.MaxValue) + { + buffer[0] = 0xD9; + buffer[1] = (byte)source.Length; + buffer = buffer.Slice(2); + consumes = Math.Min(source.Length, sizeof(uint) - 1 - 1); + } + else if (source.Length <= UInt16.MaxValue) + { + buffer[0] = 0xDA; + buffer = buffer.Slice(1); + BinaryPrimitives.WriteUInt16BigEndian(buffer, (ushort)source.Length); + buffer = buffer.Slice(sizeof(ushort)); + consumes = Math.Min(source.Length, sizeof(uint) - 1 - sizeof(ushort)); + } + else + { + buffer[0] = 0xDB; + buffer = buffer.Slice(1); + BinaryPrimitives.WriteInt32BigEndian(buffer, source.Length); + buffer = buffer.Slice(sizeof(int)); + consumes = Math.Min(source.Length, sizeof(uint) - 1 - sizeof(int)); + } + + source.Slice(0, consumes).CopyTo(buffer); + source = source.Slice(consumes); + return MemoryMarshal.Cast(bytes)[0]; + } + + public static uint GetRaw32(ref ReadOnlySpan source) + { + Debug.Assert(!source.IsEmpty); + if (source.Length >= sizeof(uint)) + { + var result = MemoryMarshal.Cast(source)[0]; + source = source.Slice(sizeof(uint)); + return result; + } + else + { + uint result; + unchecked + { + switch (source.Length) + { + case 1: + { + result = source[0]; + break; + } + case 2: + { + result = BinaryPrimitives.ReadUInt16LittleEndian(source); + break; + } + default: // 3 + { + result = BinaryPrimitives.ReadUInt16LittleEndian(source); + source = source.Slice(2); + result |= (uint)(source[0] << 16); + break; + } + } + } + + source = ReadOnlySpan.Empty; + return result; + } + } +#endif + } +} diff --git a/src/MsgPack.Core/MsgPack.Core.csproj b/src/MsgPack.Core/MsgPack.Core.csproj new file mode 100644 index 000000000..e04106cd6 --- /dev/null +++ b/src/MsgPack.Core/MsgPack.Core.csproj @@ -0,0 +1,71 @@ + + + + netcoreapp3.1 + true + + + + + + + + + + + + + True + True + MessagePackDecoder.Integers.tt + + + True + True + MessagePackDecoder.Nullables.tt + + + True + True + MessagePackDecoder.Reals.tt + + + True + True + MessagePackEncoder.Integers.tt + + + True + True + MessagePackEncoder.Strings.tt + + + + + + TextTemplatingFileGenerator + MessagePackDecoder.Nullables.cs + + + TextTemplatingFileGenerator + MessagePackDecoder.Reals.cs + + + MessagePackEncoder.Integers.cs + TextTemplatingFileGenerator + + + TextTemplatingFileGenerator + MessagePackDecoder.Integers.cs + + + MessagePackEncoder.Strings.cs + TextTemplatingFileGenerator + + + + + + + + diff --git a/src/MsgPack.Core/System.Text/EncodingExtensions.cs b/src/MsgPack.Core/System.Text/EncodingExtensions.cs new file mode 100644 index 000000000..09f42c63d --- /dev/null +++ b/src/MsgPack.Core/System.Text/EncodingExtensions.cs @@ -0,0 +1,679 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under MIT license. +// See the LICENSE.DotNetFoundation in the project root for more information. + +// This file is ported from .NET 5 https://github.com/dotnet/runtime/blob/a4c050ebf45a9745e2941863a8d7ae9c70ee88b2/src/libraries/System.Memory/src/System/Text/EncodingExtensions.cs#L1 + +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; +using MsgPack.Internal; + +namespace System.Text +{ + internal static class EncodingExtensions + { + /// + /// The maximum number of input elements after which we'll begin to chunk the input. + /// + /// + /// The reason for this chunking is that the existing Encoding / Encoder / Decoder APIs + /// like GetByteCount / GetCharCount will throw if an integer overflow occurs. Since + /// we may be working with large inputs in these extension methods, we don't want to + /// risk running into this issue. While it's technically possible even for 1 million + /// input elements to result in an overflow condition, such a scenario is unrealistic, + /// so we won't worry about it. + /// + private const int MaxInputElementsPerIteration = 1 * 1024 * 1024; + + /// + /// Encodes the specified to s using the specified + /// and writes the result to . + /// + /// The which represents how the data in should be encoded. + /// The to encode to s. + /// The buffer to which the encoded bytes will be written. + /// Thrown if contains data that cannot be encoded and is configured + /// to throw an exception when such data is seen. + public static long GetBytes(this Encoding encoding, ReadOnlySpan chars, IBufferWriter writer) + { + if (encoding is null) + { + throw new ArgumentNullException(nameof(encoding)); + } + + if (writer is null) + { + throw new ArgumentNullException(nameof(writer)); + } + + if (chars.Length <= MaxInputElementsPerIteration) + { + // The input span is small enough where we can one-shot this. + + int byteCount = encoding.GetByteCount(chars); + Span scratchBuffer = writer.GetSpan(byteCount); + + int actualBytesWritten = encoding.GetBytes(chars, scratchBuffer); + + writer.Advance(actualBytesWritten); + return actualBytesWritten; + } + else + { + // Allocate a stateful Encoder instance and chunk this. + + Convert(encoding.GetEncoder(), chars, writer, flush: true, out long totalBytesWritten, out bool completed); + return totalBytesWritten; + } + } + + /// + /// Decodes the specified to s using the specified + /// and writes the result to . + /// + /// The which represents how the data in should be encoded. + /// The whose contents should be encoded. + /// The buffer to which the encoded bytes will be written. + /// The number of bytes written to . + /// Thrown if contains data that cannot be encoded and is configured + /// to throw an exception when such data is seen. + public static long GetBytes(this Encoding encoding, in ReadOnlySequence chars, IBufferWriter writer) + { + if (encoding is null) + { + throw new ArgumentNullException(nameof(encoding)); + } + + if (writer is null) + { + throw new ArgumentNullException(nameof(writer)); + } + + // Delegate to the Span-based method if possible. + // If that doesn't work, allocate the Encoder instance and run a loop. + + if (chars.IsSingleSegment) + { + return GetBytes(encoding, chars.FirstSpan, writer); + } + else + { + Convert(encoding.GetEncoder(), chars, writer, flush: true, out long bytesWritten, out bool completed); + return bytesWritten; + } + } + + /// + /// Encodes the specified to s using the specified + /// and outputs the result to . + /// + /// The which represents how the data in should be encoded. + /// The to encode to s. + /// The destination buffer to which the encoded bytes will be written. + /// The number of bytes written to . + /// Thrown if is not large enough to contain the encoded form of . + /// Thrown if contains data that cannot be encoded and is configured + /// to throw an exception when such data is seen. + public static int GetBytes(this Encoding encoding, in ReadOnlySequence chars, Span bytes) + { + if (encoding is null) + { + throw new ArgumentNullException(nameof(encoding)); + } + + if (chars.IsSingleSegment) + { + // If the incoming sequence is single-segment, one-shot this. + + return encoding.GetBytes(chars.FirstSpan, bytes); + } + else + { + // If the incoming sequence is multi-segment, create a stateful Encoder + // and use it as the workhorse. On the final iteration we'll pass flush=true. + + ReadOnlySequence remainingChars = chars; + int originalBytesLength = bytes.Length; + var encoder = encoding.GetEncoder(); + bool isFinalSegment; + + do + { + GetFirstSpan(remainingChars, out ReadOnlySpan firstSpan, out SequencePosition next); + isFinalSegment = remainingChars.IsSingleSegment; + + int bytesWrittenJustNow = encoder.GetBytes(firstSpan, bytes, flush: isFinalSegment); + bytes = bytes.Slice(bytesWrittenJustNow); + remainingChars = remainingChars.Slice(next); + } while (!isFinalSegment); + + return originalBytesLength - bytes.Length; // total number of bytes we wrote + } + } + + /// + /// Encodes the specified into a array using the specified . + /// + /// The which represents how the data in should be encoded. + /// The to encode to s. + /// A array which represents the encoded contents of . + /// Thrown if contains data that cannot be encoded and is configured + /// to throw an exception when such data is seen. + public static byte[] GetBytes(this Encoding encoding, in ReadOnlySequence chars) + { + if (encoding is null) + { + throw new ArgumentNullException(nameof(encoding)); + } + + if (chars.IsSingleSegment) + { + // If the incoming sequence is single-segment, one-shot this. + + ReadOnlySpan span = chars.FirstSpan; + + byte[] retVal = new byte[encoding.GetByteCount(span)]; + encoding.GetBytes(span, retVal); + return retVal; + } + else + { + // If the incoming sequence is multi-segment, create a stateful Encoder + // and use it as the workhorse. On the final iteration we'll pass flush=true. + + var encoder = encoding.GetEncoder(); + + // Maintain a list of all the segments we'll need to concat together. + // These will be released back to the pool at the end of the method. + + List<(byte[], int)> listOfSegments = new List<(byte[], int)>(); + int totalByteCount = 0; + + ReadOnlySequence remainingChars = chars; + bool isFinalSegment; + + do + { + GetFirstSpan(remainingChars, out ReadOnlySpan firstSpan, out SequencePosition next); + isFinalSegment = remainingChars.IsSingleSegment; + + int byteCountThisIteration = encoder.GetByteCount(firstSpan, flush: isFinalSegment); + byte[] rentedArray = ArrayPool.Shared.Rent(byteCountThisIteration); + int actualBytesWrittenThisIteration = encoder.GetBytes(firstSpan, rentedArray, flush: isFinalSegment); // could throw ArgumentException if overflow would occur + listOfSegments.Add((rentedArray, actualBytesWrittenThisIteration)); + + totalByteCount += actualBytesWrittenThisIteration; + if (totalByteCount < 0) + { + // If we overflowed, call the array ctor, passing int.MaxValue. + // This will end up throwing the expected OutOfMemoryException + // since arrays are limited to under int.MaxValue elements in length. + + totalByteCount = Int32.MaxValue; + break; + } + + remainingChars = remainingChars.Slice(next); + } while (!isFinalSegment); + + // Now build up the byte[] to return, then release all of our scratch buffers + // back to the shared pool. + + byte[] retVal = new byte[totalByteCount]; + Span remainingBytes = retVal; + + foreach ((byte[] array, int length) in listOfSegments) + { + array.AsSpan(0, length).CopyTo(remainingBytes); + ArrayPool.Shared.Return(array); + remainingBytes = remainingBytes.Slice(length); + } + + Debug.Assert(remainingBytes.IsEmpty, "Over-allocated the byte[] instance?"); + + return retVal; + } + } + + /// + /// Decodes the specified to s using the specified + /// and writes the result to . + /// + /// The which represents how the data in should be decoded. + /// The whose bytes should be decoded. + /// The buffer to which the decoded chars will be written. + /// The number of chars written to . + /// Thrown if contains data that cannot be decoded and is configured + /// to throw an exception when such data is seen. + public static long GetChars(this Encoding encoding, ReadOnlySpan bytes, IBufferWriter writer) + { + if (encoding is null) + { + throw new ArgumentNullException(nameof(encoding)); + } + + if (writer is null) + { + throw new ArgumentNullException(nameof(writer)); + } + + if (bytes.Length <= MaxInputElementsPerIteration) + { + // The input span is small enough where we can one-shot this. + + int charCount = encoding.GetCharCount(bytes); + Span scratchBuffer = writer.GetSpan(charCount); + + int actualCharsWritten = encoding.GetChars(bytes, scratchBuffer); + + writer.Advance(actualCharsWritten); + return actualCharsWritten; + } + else + { + // Allocate a stateful Decoder instance and chunk this. + + Convert(encoding.GetDecoder(), bytes, writer, flush: true, out long totalCharsWritten, out bool completed); + return totalCharsWritten; + } + } + + /// + /// Decodes the specified to s using the specified + /// and writes the result to . + /// + /// The which represents how the data in should be decoded. + /// The whose bytes should be decoded. + /// The buffer to which the decoded chars will be written. + /// The number of chars written to . + /// Thrown if contains data that cannot be decoded and is configured + /// to throw an exception when such data is seen. + public static long GetChars(this Encoding encoding, in ReadOnlySequence bytes, IBufferWriter writer) + { + if (encoding is null) + { + throw new ArgumentNullException(nameof(encoding)); + } + + if (writer is null) + { + throw new ArgumentNullException(nameof(writer)); + } + + // Delegate to the Span-based method if possible. + // If that doesn't work, allocate the Encoder instance and run a loop. + + if (bytes.IsSingleSegment) + { + return GetChars(encoding, bytes.FirstSpan, writer); + } + else + { + Convert(encoding.GetDecoder(), bytes, writer, flush: true, out long charsWritten, out bool completed); + return charsWritten; + } + } + + /// + /// Decodes the specified to s using the specified + /// and outputs the result to . + /// + /// The which represents how the data in is encoded. + /// The to decode to characters. + /// The destination buffer to which the decoded characters will be written. + /// The number of chars written to . + /// Thrown if is not large enough to contain the encoded form of . + /// Thrown if contains data that cannot be decoded and is configured + /// to throw an exception when such data is seen. + public static int GetChars(this Encoding encoding, in ReadOnlySequence bytes, Span chars) + { + if (encoding is null) + { + throw new ArgumentNullException(nameof(encoding)); + } + + if (bytes.IsSingleSegment) + { + // If the incoming sequence is single-segment, one-shot this. + + return encoding.GetChars(bytes.FirstSpan, chars); + } + else + { + // If the incoming sequence is multi-segment, create a stateful Decoder + // and use it as the workhorse. On the final iteration we'll pass flush=true. + + ReadOnlySequence remainingBytes = bytes; + int originalCharsLength = chars.Length; + var decoder = encoding.GetDecoder(); + bool isFinalSegment; + + do + { + GetFirstSpan(remainingBytes, out ReadOnlySpan firstSpan, out SequencePosition next); + isFinalSegment = remainingBytes.IsSingleSegment; + + int charsWrittenJustNow = decoder.GetChars(firstSpan, chars, flush: isFinalSegment); + chars = chars.Slice(charsWrittenJustNow); + remainingBytes = remainingBytes.Slice(next); + } while (!isFinalSegment); + + return originalCharsLength - chars.Length; // total number of chars we wrote + } + } + + /// + /// Decodes the specified into a using the specified . + /// + /// The which represents how the data in is encoded. + /// The to decode into characters. + /// A which represents the decoded contents of . + /// Thrown if contains data that cannot be decoded and is configured + /// to throw an exception when such data is seen. + public static string GetString(this Encoding encoding, in ReadOnlySequence bytes) + { + if (encoding is null) + { + throw new ArgumentNullException(nameof(encoding)); + } + + if (bytes.IsSingleSegment) + { + // If the incoming sequence is single-segment, one-shot this. + + return encoding.GetString(bytes.FirstSpan); + } + else + { + return encoding.GetStringMultiSegment(bytes, ArrayPool.Shared); + } + } + + internal static string GetStringMultiSegment(this Encoding encoding, in ReadOnlySequence bytes, ArrayPool arrayPool, CancellationToken cancellationToken = default) + { + // If the incoming sequence is multi-segment, create a stateful Decoder + // and use it as the workhorse. On the final iteration we'll pass flush=true. + + var decoder = encoding.GetDecoder(); + + // Maintain a list of all the segments we'll need to concat together. + // These will be released back to the pool at the end of the method. + + List<(char[], int)> listOfSegments = new List<(char[], int)>(); + int totalCharCount = 0; + + ReadOnlySequence remainingBytes = bytes; + bool isFinalSegment; + + do + { + GetFirstSpan(remainingBytes, out ReadOnlySpan firstSpan, out SequencePosition next); + isFinalSegment = remainingBytes.IsSingleSegment; + + int charCountThisIteration = decoder.GetCharCount(firstSpan, flush: isFinalSegment); // could throw ArgumentException if overflow would occur + char[] rentedArray = arrayPool.Rent(charCountThisIteration); + int actualCharsWrittenThisIteration = decoder.GetChars(firstSpan, rentedArray, flush: isFinalSegment); + listOfSegments.Add((rentedArray, actualCharsWrittenThisIteration)); + + totalCharCount += actualCharsWrittenThisIteration; + if (totalCharCount < 0) + { + foreach(var segment in listOfSegments) + { + arrayPool.Return(segment.Item1); + } + + // overflow + Throw.TooLargeByteLengthForString(encoding.EncodingName); + // never reaches. + return null!; + } + + remainingBytes = remainingBytes.Slice(next); + cancellationToken.ThrowIfCancellationRequested(); + } while (!isFinalSegment); + + // Now build up the string to return, then release all of our scratch buffers + // back to the shared pool. + + return String.Create(totalCharCount, listOfSegments, (span, listOfSegments) => + { + foreach ((char[] array, int length) in listOfSegments) + { + array.AsSpan(0, length).CopyTo(span); + arrayPool.Return(array); + span = span.Slice(length); + } + + Debug.Assert(span.IsEmpty, "Over-allocated the string instance?"); + }); + } + + /// + /// Converts a to bytes using and writes the result to . + /// + /// The instance which can convert s to s. + /// A sequence of characters to encode. + /// The buffer to which the encoded bytes will be written. + /// to indicate no further data is to be converted; otherwise . + /// When this method returns, contains the count of s which were written to . + /// + /// When this method returns, contains if contains no partial internal state; otherwise, . + /// If is , this will always be set to when the method returns. + /// + /// Thrown if contains data that cannot be encoded and is configured + /// to throw an exception when such data is seen. + public static void Convert(this Encoder encoder, ReadOnlySpan chars, IBufferWriter writer, bool flush, out long bytesUsed, out bool completed) + { + if (encoder is null) + { + throw new ArgumentNullException(nameof(encoder)); + } + + if (writer is null) + { + throw new ArgumentNullException(nameof(writer)); + } + + // We need to perform at least one iteration of the loop since the encoder could have internal state. + + long totalBytesWritten = 0; + + do + { + // If our remaining input is very large, instead truncate it and tell the encoder + // that there'll be more data after this call. This truncation is only for the + // purposes of getting the required byte count. Since the writer may give us a span + // larger than what we asked for, we'll pass the entirety of the remaining data + // to the transcoding routine, since it may be able to make progress beyond what + // was initially computed for the truncated input data. + + int byteCountForThisSlice = (chars.Length <= MaxInputElementsPerIteration) + ? encoder.GetByteCount(chars, flush) + : encoder.GetByteCount(chars.Slice(0, MaxInputElementsPerIteration), flush: false /* this isn't the end of the data */); + + Span scratchBuffer = writer.GetSpan(byteCountForThisSlice); + + encoder.Convert(chars, scratchBuffer, flush, out int charsUsedJustNow, out int bytesWrittenJustNow, out completed); + + chars = chars.Slice(charsUsedJustNow); + writer.Advance(bytesWrittenJustNow); + totalBytesWritten += bytesWrittenJustNow; + } while (!chars.IsEmpty); + + bytesUsed = totalBytesWritten; + } + + /// + /// Converts a to encoded bytes and writes the result to . + /// + /// The instance which can convert s to s. + /// A sequence of characters to encode. + /// The buffer to which the encoded bytes will be written. + /// to indicate no further data is to be converted; otherwise . + /// When this method returns, contains the count of s which were written to . + /// When this method returns, contains if all input up until was + /// converted; otherwise, . If is , this will always be set to + /// when the method returns. + /// Thrown if contains data that cannot be encoded and is configured + /// to throw an exception when such data is seen. + public static void Convert(this Encoder encoder, in ReadOnlySequence chars, IBufferWriter writer, bool flush, out long bytesUsed, out bool completed) + { + // Parameter null checks will be performed by the workhorse routine. + + ReadOnlySequence remainingChars = chars; + long totalBytesWritten = 0; + bool isFinalSegment; + + do + { + // Process each segment individually. We need to run at least one iteration of the loop in case + // the Encoder has internal state. + + GetFirstSpan(remainingChars, out ReadOnlySpan firstSpan, out SequencePosition next); + isFinalSegment = remainingChars.IsSingleSegment; + + Convert(encoder, firstSpan, writer, flush && isFinalSegment, out long bytesWrittenThisIteration, out completed); + + totalBytesWritten += bytesWrittenThisIteration; + remainingChars = remainingChars.Slice(next); + } while (!isFinalSegment); + + bytesUsed = totalBytesWritten; + } + + /// + /// Converts a to chars using and writes the result to . + /// + /// The instance which can convert s to s. + /// A sequence of bytes to decode. + /// The buffer to which the decoded chars will be written. + /// to indicate no further data is to be converted; otherwise . + /// When this method returns, contains the count of s which were written to . + /// + /// When this method returns, contains if contains no partial internal state; otherwise, . + /// If is , this will always be set to when the method returns. + /// + /// Thrown if contains data that cannot be encoded and is configured + /// to throw an exception when such data is seen. + public static void Convert(this Decoder decoder, ReadOnlySpan bytes, IBufferWriter writer, bool flush, out long charsUsed, out bool completed) + { + if (decoder is null) + { + throw new ArgumentNullException(nameof(decoder)); + } + + if (writer is null) + { + throw new ArgumentNullException(nameof(writer)); + } + + // We need to perform at least one iteration of the loop since the decoder could have internal state. + + long totalCharsWritten = 0; + + do + { + // If our remaining input is very large, instead truncate it and tell the decoder + // that there'll be more data after this call. This truncation is only for the + // purposes of getting the required char count. Since the writer may give us a span + // larger than what we asked for, we'll pass the entirety of the remaining data + // to the transcoding routine, since it may be able to make progress beyond what + // was initially computed for the truncated input data. + + int charCountForThisSlice = (bytes.Length <= MaxInputElementsPerIteration) + ? decoder.GetCharCount(bytes, flush) + : decoder.GetCharCount(bytes.Slice(0, MaxInputElementsPerIteration), flush: false /* this isn't the end of the data */); + + Span scratchBuffer = writer.GetSpan(charCountForThisSlice); + + decoder.Convert(bytes, scratchBuffer, flush, out int bytesUsedJustNow, out int charsWrittenJustNow, out completed); + + bytes = bytes.Slice(bytesUsedJustNow); + writer.Advance(charsWrittenJustNow); + totalCharsWritten += charsWrittenJustNow; + } while (!bytes.IsEmpty); + + charsUsed = totalCharsWritten; + } + + /// + /// Converts a to UTF-16 encoded characters and writes the result to . + /// + /// The instance which can convert s to s. + /// A sequence of bytes to decode. + /// The buffer to which the decoded characters will be written. + /// to indicate no further data is to be converted; otherwise . + /// When this method returns, contains the count of s which were written to . + /// + /// When this method returns, contains if contains no partial internal state; otherwise, . + /// If is , this will always be set to when the method returns. + /// + /// Thrown if contains data that cannot be decoded and is configured + /// to throw an exception when such data is seen. + public static void Convert(this Decoder decoder, in ReadOnlySequence bytes, IBufferWriter writer, bool flush, out long charsUsed, out bool completed) + { + // Parameter null checks will be performed by the workhorse routine. + + ReadOnlySequence remainingBytes = bytes; + long totalCharsWritten = 0; + bool isFinalSegment; + + do + { + // Process each segment individually. We need to run at least one iteration of the loop in case + // the Decoder has internal state. + + GetFirstSpan(remainingBytes, out ReadOnlySpan firstSpan, out SequencePosition next); + isFinalSegment = remainingBytes.IsSingleSegment; + + Convert(decoder, firstSpan, writer, flush && isFinalSegment, out long charsWrittenThisIteration, out completed); + + totalCharsWritten += charsWrittenThisIteration; + remainingBytes = remainingBytes.Slice(next); + } while (!isFinalSegment); + + charsUsed = totalCharsWritten; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private static void GetFirstSpan(ReadOnlySequence sequence, out ReadOnlySpan first, out SequencePosition next) + { + // Port from https://github.com/dotnet/runtime/blob/a4c050ebf45a9745e2941863a8d7ae9c70ee88b2/src/libraries/System.Memory/src/System/Buffers/ReadOnlySequence.Helpers.cs#L635 + next = default; + + if (SequenceMarshal.TryGetArray(sequence, out var arraySegment)) + { + first = arraySegment; + } + else if(SequenceMarshal.TryGetReadOnlySequenceSegment(sequence, out var segment, out var startIndex, out _, out var endIndex)) + { + // In this pass, startIndex and endIndex should be same as internal _startInteger and _endInteger. + first = segment.Memory.Span; + if (!sequence.IsSingleSegment) + { + first = first.Slice(startIndex); + next = new SequencePosition(segment.Next, 0); + } + else + { + first = first.Slice(startIndex, endIndex - startIndex); + } + } + else + { + // This should fallback to "Slow-Pass" and default value handling. + // And they must be single segment, so next will be default. + first = sequence.FirstSpan; + } + } + } +} diff --git a/src/MsgPack.Core/System.Text/Throw.cs b/src/MsgPack.Core/System.Text/Throw.cs new file mode 100644 index 000000000..07a723adf --- /dev/null +++ b/src/MsgPack.Core/System.Text/Throw.cs @@ -0,0 +1,14 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using MsgPack; + +namespace System.Text +{ + internal static class Throw + { + public static void TooLargeByteLengthForString(string encodingName) + => throw new MessageTypeException($"Input is too large for encoding '{encodingName}' to decode."); + } +} diff --git a/src/MsgPack.Extensions/MsgPack.Extensions.csproj b/src/MsgPack.Extensions/MsgPack.Extensions.csproj new file mode 100644 index 000000000..cb6319069 --- /dev/null +++ b/src/MsgPack.Extensions/MsgPack.Extensions.csproj @@ -0,0 +1,7 @@ + + + + netcoreapp3.1 + + + diff --git a/src/MsgPack.Json/Codecs.Json/JsonCodecProvider.cs b/src/MsgPack.Json/Codecs.Json/JsonCodecProvider.cs new file mode 100644 index 000000000..90dc6274c --- /dev/null +++ b/src/MsgPack.Json/Codecs.Json/JsonCodecProvider.cs @@ -0,0 +1,36 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using MsgPack.Internal; +using MsgPack.Json; + +namespace MsgPack.Codecs +{ + public sealed class JsonCodecProvider : CodecProvider + { + private readonly JsonEncoder _encoder; + private readonly JsonDecoder _decoder; + + public JsonCodecProvider( + JsonEncoderOptions encoderOptions, + JsonDecoderOptions decoderOptions + ) + { + this._encoder = new JsonEncoder(Ensure.NotNull(encoderOptions)); + Ensure.NotNull(decoderOptions); + if (decoderOptions.ParseOptions == JsonParseOptions.None) + { + this._decoder = new SimpleJsonDecoder(decoderOptions); + } + else + { + this._decoder = new FlexibleJsonDecoder(decoderOptions); + } + } + + public sealed override FormatEncoder GetEncoder() => this._encoder; + + public sealed override FormatDecoder GetDecoder() => this._decoder; + } +} diff --git a/src/MsgPack.Json/Json/FlexibleJsonDecoder.ReadTrivia.cs b/src/MsgPack.Json/Json/FlexibleJsonDecoder.ReadTrivia.cs new file mode 100644 index 000000000..db6aa493e --- /dev/null +++ b/src/MsgPack.Json/Json/FlexibleJsonDecoder.ReadTrivia.cs @@ -0,0 +1,149 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + internal partial class FlexibleJsonDecoder + { + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + protected sealed override long ReadTrivia(ref SequenceReader source) + { + ReadOnlySpan singleByteWhitespaces; + unsafe + { + if ((this.Options.ParseOptions & JsonParseOptions.AllowUnicodeWhitespace) == 0) + { + byte* pStandardWhitespaces = stackalloc byte[] { (byte)' ', (byte)'\t', (byte)'\r', (byte)'\n' }; + singleByteWhitespaces = new ReadOnlySpan(pStandardWhitespaces, 4); + } + else + { + byte* pSingleByteUnicodeWhitespaces = + stackalloc byte[] + { + (byte)' ', (byte)'\t', (byte)'\r', (byte)'\n', + (byte)'\u000B', (byte)'\u000C' + }; + singleByteWhitespaces = new ReadOnlySpan(pSingleByteUnicodeWhitespaces, 6); + } + } + + var consumed = 0L; + while (!source.End) + { + consumed += source.AdvancePastAny(singleByteWhitespaces); + + if (this.TryReadMultiByteUnicodeWhitespaces(source, ref consumed)) + { + continue; + } + + // Single line comment + if (this.TryReadSingleLineComment(source, ref consumed)) + { + continue; + } + + // Multi line comment + if (this.TryReadMultiLineComment(source, ref consumed)) + { + continue; + } + } + + return consumed; + } + + private bool TryReadMultiByteUnicodeWhitespaces(in SequenceReader source, ref long consumed) + { + if ((this.Options.ParseOptions & JsonParseOptions.AllowUnicodeWhitespace) == 0) + { + return false; + } + + foreach (var whitespace in JsonTriviaTokens.MultiByteUnicodeWhitespaces) + { + var length = source.AdvancePastAny(whitespace); + if (length > 0) + { + consumed += length; + return true; + } + } + + return false; + } + + private bool TryReadSingleLineComment(in SequenceReader source, ref long consumed) + { + ReadOnlySpan singleLineCommentStart; + ReadOnlySpan newLine; + unsafe + { + byte* pSingleLineCommentStart = stackalloc[] { (byte)'/', (byte)'/' }; + singleLineCommentStart = new ReadOnlySpan(pSingleLineCommentStart, 2); + byte* pNewLine = stackalloc[] { (byte)'\r', (byte)'\n' }; + newLine = new ReadOnlySpan(pNewLine, 2); + } + + if (((this.Options.ParseOptions & JsonParseOptions.AllowHashSingleLineComment) != 0 && source.IsNext((byte)'#', advancePast: true)) + || ((this.Options.ParseOptions & JsonParseOptions.AllowDoubleSolidousSingleLineComment) != 0 && source.IsNext(singleLineCommentStart, advancePast: true))) + { + if (!source.TryReadToAny(out ReadOnlySequence line, newLine, advancePastDelimiter: true)) + { + // to EoF + consumed += source.Remaining; + source.Advance(source.Remaining); + return true; + } + + consumed += line.Length; + consumed += source.AdvancePastAny((byte)'\r', (byte)'\n'); + return true; + } + + return false; + } + + private bool TryReadMultiLineComment(in SequenceReader source, ref long consumed) + { + if((this.Options.ParseOptions & JsonParseOptions.AllowMultilineComment) == 0) + { + return false; + } + + ReadOnlySpan multiLineCommentStart; + ReadOnlySpan multiLineCommentEnd; + unsafe + { + byte* pMultiLineCommentStart = stackalloc[] { (byte)'/', (byte)'*' }; + multiLineCommentStart = new ReadOnlySpan(pMultiLineCommentStart, 2); + byte* pMultiLineCommentEnd = stackalloc[] { (byte)'*', (byte)'/' }; + multiLineCommentEnd = new ReadOnlySpan(pMultiLineCommentEnd, 2); + } + + var offset = source.Consumed; + + if (!source.IsNext(multiLineCommentStart, advancePast: true)) + { + return false; + } + + if (!source.TryReadToAny(out ReadOnlySequence comment, multiLineCommentEnd, advancePastDelimiter: true)) + { + // */ is not found. + source.Rewind(2); + return false; + } + + consumed += comment.Length; + return true; + } + } +} diff --git a/src/MsgPack.Json/Json/FlexibleJsonDecoder.cs b/src/MsgPack.Json/Json/FlexibleJsonDecoder.cs new file mode 100644 index 000000000..b7f4c4b3b --- /dev/null +++ b/src/MsgPack.Json/Json/FlexibleJsonDecoder.cs @@ -0,0 +1,53 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + /// + /// Simple implementation which can handle various resource consuming relaxations. + /// + internal sealed partial class FlexibleJsonDecoder : JsonDecoder + { + public FlexibleJsonDecoder(JsonDecoderOptions options) + : base(options) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override unsafe bool TryDecodeNull(ref SequenceReader source, out int requestHint) + { + byte* pNull= stackalloc byte[] { (byte)'n', (byte)'u', (byte)'l', (byte)'l' }; + ReadOnlySpan @null = new ReadOnlySpan(pNull, 4); + + if (source.IsNext(@null, advancePast: true)) + { + requestHint = 0; + source.Advance(4); + return true; + } + + return this.TryDecodeNullSlow(ref source, out requestHint); + } + + private unsafe bool TryDecodeNullSlow(ref SequenceReader source, out int requestHint) + { + byte* pUndefined = stackalloc byte[] { (byte)'u', (byte)'n', (byte)'d', (byte)'e', (byte)'f', (byte)'i', (byte)'n', (byte)'e', (byte)'d', }; + ReadOnlySpan undefined = new ReadOnlySpan(pUndefined, 9); + + if ((this.Options.ParseOptions & JsonParseOptions.AllowUndefined) != 0 + && source.IsNext(undefined, advancePast: true)) + { + requestHint = 0; + source.Advance(9); + return true; + } + + requestHint = Math.Max(0, 4 - (int)source.Remaining); + return false; + } + } +} diff --git a/src/MsgPack.Json/Json/InfinityHandling.cs b/src/MsgPack.Json/Json/InfinityHandling.cs new file mode 100644 index 000000000..abe9026f4 --- /dev/null +++ b/src/MsgPack.Json/Json/InfinityHandling.cs @@ -0,0 +1,32 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +namespace MsgPack.Json +{ + /// + /// Defines positive and/or negative infinity handling in JSON serialization to be compliant with RFC 8259. + /// + public enum InfinityHandling + { + /// + /// Use system default setting. See for current default. + /// + Default = 0, + + /// + /// Use MinValue for negative infinity and MaxValue for positive infinity. + /// + MinMax = 1, + + /// + /// Throws exception conservatively. + /// + Error = 2, + + /// + /// Use own custom formatting logic. + /// + Custom = 3 + } +} diff --git a/src/MsgPack.Json/Json/JsonCharactor.cs b/src/MsgPack.Json/Json/JsonCharactor.cs new file mode 100644 index 000000000..777c447f7 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonCharactor.cs @@ -0,0 +1,23 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Linq; +using System.Text; + +namespace MsgPack.Json +{ + /// + /// Defines JSON constant chars in UTF-8 or related structs. + /// + internal static class JsonCharactor + { + public static readonly byte[] MustBeEscaped1Byte = new[] { (byte)'\\', (byte)'"' }; + public static readonly ReadOnlyMemory ShouldBeEscaped = new[] { (byte)'/', (byte)'\'', (byte)'<', (byte)'>', (byte)'&' }.Select(b => new Rune(b)).ToArray(); // For HTML embedding + public static ReadOnlySpan CarriageReturn => new[] { (byte)'\r' }; + public static ReadOnlySpan LineFeed => new[] { (byte)'\n' }; + public static ReadOnlySpan Tab => new[] { (byte)'\t' }; + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoder.CollectionHeaders.cs b/src/MsgPack.Json/Json/JsonDecoder.CollectionHeaders.cs new file mode 100644 index 000000000..afe416a8d --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoder.CollectionHeaders.cs @@ -0,0 +1,21 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Buffers; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + public partial class JsonDecoder + { + public sealed override CollectionType DecodeArrayOrMapHeader(ref SequenceReader source, out int itemsCount, out int requestHint) + => JsonThrow.CollectionHeaderDecodingIsNotSupported(out itemsCount, out requestHint); + + public sealed override int DecodeArrayHeader(ref SequenceReader source, out int requestHint) + => JsonThrow.CollectionHeaderDecodingIsNotSupported(out requestHint); + + public sealed override int DecodeMapHeader(ref SequenceReader source, out int requestHint) + => JsonThrow.CollectionHeaderDecodingIsNotSupported(out requestHint); + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoder.DecodeItem.cs b/src/MsgPack.Json/Json/JsonDecoder.DecodeItem.cs new file mode 100644 index 000000000..a72edb68d --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoder.DecodeItem.cs @@ -0,0 +1,149 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + public partial class JsonDecoder + { + public sealed override bool DecodeItem(ref SequenceReader source, out DecodeItemResult result, CancellationToken cancellationToken = default) + { + var startPosition = source.Position; + var triviaLength = this.ReadTrivia(ref source); + if (triviaLength != 0) + { + result = DecodeItemResult.ScalarOrSequence(ElementType.OtherTrivia, source.Sequence.Slice(startPosition, source.Position)); + return true; + } + + if (!source.TryPeek(out var token)) + { + result = DecodeItemResult.InsufficientInput(1); + return false; + } + +#warning HANDLE :/= and , + switch (token) + { + case (byte)'t': + { + ReadOnlySpan @true; + unsafe + { + byte* pTrue = stackalloc byte[] { (byte)'t', (byte)'r', (byte)'u', (byte)'e' }; + @true = new ReadOnlySpan(pTrue, 4); + } + + if (source.IsNext(@true, advancePast: true)) + { + result = DecodeItemResult.True(); + return true; + } + + break; + } + case (byte)'f': + { + ReadOnlySpan @false; + unsafe + { + byte* pFalse = stackalloc byte[] { (byte)'f', (byte)'a', (byte)'l', (byte)'s', (byte)'e' }; + @false = new ReadOnlySpan(pFalse, 5); + } + + if (source.IsNext(@false, advancePast: true)) + { + result = DecodeItemResult.False(); + return true; + } + + break; + } + case (byte)'n': + { + ReadOnlySpan @null; + unsafe + { + byte* pNull = stackalloc byte[] { (byte)'n', (byte)'u', (byte)'l', (byte)'l' }; + @null = new ReadOnlySpan(pNull, 4); + } + + if (source.IsNext(@null, advancePast: true)) + { + result = DecodeItemResult.Null(); + return true; + } + + break; + } + case (byte)'0': + case (byte)'1': + case (byte)'2': + case (byte)'3': + case (byte)'4': + case (byte)'5': + case (byte)'6': + case (byte)'7': + case (byte)'8': + case (byte)'9': + case (byte)'-': + case (byte)'+': + { + var number = this.DecodeNumber(ref source, out var requestHint); + if(requestHint != 0) + { + result = DecodeItemResult.InsufficientInput(requestHint); + return false; + } + + var value = new byte[sizeof(double)]; + Unsafe.WriteUnaligned(ref MemoryMarshal.GetReference(value.AsSpan()), number); + result = DecodeItemResult.ScalarOrSequence(ElementType.Double, value); + return true; + } + case (byte)'[': + { + result = DecodeItemResult.CollectionHeader(ElementType.Array, this.CreateArrayIterator()); + return true; + } + case (byte)'{': + { + result = DecodeItemResult.CollectionHeader(ElementType.OtherTrivia, this.CreateObjectPropertyIterator()); + return true; + } + case (byte)'\'': + { + if ((this.Options.ParseOptions & JsonParseOptions.AllowSingleQuotationString) != 0) + { + goto case (byte)'"'; + } + + break; + } + case (byte)'"': + { + if (this.GetRawStringCore(ref source, out var rawString, out var requestHint)) + { + result = DecodeItemResult.InsufficientInput(requestHint); + return false; + } + + result = DecodeItemResult.ScalarOrSequence(ElementType.String, rawString); + return true; + } + } + + JsonThrow.UnexpectedToken(source.Consumed, token); + // never + result = default; + return default; + } + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoder.Iteration.cs b/src/MsgPack.Json/Json/JsonDecoder.Iteration.cs new file mode 100644 index 000000000..82e84a498 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoder.Iteration.cs @@ -0,0 +1,382 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + public partial class JsonDecoder + { + private sealed class ArrayIterator + { + private const byte ArrayStartToken = (byte)'['; + private const byte ArrayEndToken = (byte)']'; + private const byte ItemDelimiterToken = (byte)','; + + private readonly JsonDecoder _decoder; + private State _state; + + public ArrayIterator(JsonDecoder decoder) + { + this._decoder = decoder; + this._state = State.Head; + } + + public bool DetectArrayEnds(ref SequenceReader source, ref long nextItemIndex, long itemsCount, out int requestHint) + { + if (this._state == State.Tail) + { + requestHint = 0; + return true; // End + } + + this._decoder.ReadTrivia(ref source); + + switch (this._state) + { + case State.Head: + { + if (!source.TryPeek(out var shouldBeArrayStart)) + { + requestHint = 2; + return false; + } + + if (shouldBeArrayStart != ArrayStartToken) + { + // Invalid + JsonThrow.UnexpectedToken(source.Consumed, shouldBeArrayStart); + } + + source.Advance(1); + break; + } // case State.Head + default: + { + Debug.Assert(this._state == State.Item, $"this._state {(this._state)} == State.Item"); + + if (!source.TryPeek(out var mayBeDelimiterOrArrayEnd)) + { + requestHint = 1; + return false; + } + + switch (mayBeDelimiterOrArrayEnd) + { + case ArrayEndToken: + { + requestHint = 0; + source.Advance(1); + this._state = State.Tail; + return true; + } // case ArrayEndToken: + case ItemDelimiterToken: + { + source.Advance(1); + // goto maybe-item + break; + } // ItemDelimiterToken + default: + { + JsonThrow.UnexpectedToken(source.Consumed, mayBeDelimiterOrArrayEnd); + // never + break; + } + } // switch (mayBeDelimiterOrArrayEnd) + + break; + } // default: + } // swtich (this._state) + + // Handle 'maybe item' state + this._decoder.ReadTrivia(ref source); + if (!source.TryPeek(out var mayBeArrayEnd)) + { + requestHint = 1; + return false; + } + + requestHint = 0; + + if (mayBeArrayEnd == ArrayEndToken) + { + source.Advance(1); + this._state = State.Tail; + return true; + } + else + { + this._state = State.Item; + return false; + } + } + + private enum State + { + Head = 0, + Item, + Tail + } + } + + private sealed class ObjectPropertyIterator + { + private const byte ObjectStartToken = (byte)'{'; + private const byte ObjectEndToken = (byte)'}'; + private const byte KeyValueSepratorToken = (byte)':'; + private const byte AltKeyValueSepratorToken = (byte)'='; + private const byte ItemDelimiterToken = (byte)','; + + private readonly JsonDecoder _decoder; + private readonly bool _isEqualSignAllowed; + private State _state; + + public ObjectPropertyIterator(JsonDecoder decoder) + { + this._decoder = decoder; + this._state = State.Head; + this._isEqualSignAllowed = (decoder.Options.ParseOptions & JsonParseOptions.AllowEqualSignSeparator) != 0; + } + + public bool DetectObjectEnds(ref SequenceReader source, ref long nextItemIndex, long itemsCount, out int requestHint) + { + if (this._state == State.Tail) + { + requestHint = 0; + return true; // End + } + + this._decoder.ReadTrivia(ref source); + + switch (this._state) + { + case State.Head: + { + if (!source.TryPeek(out var shouldBeObjectStart)) + { + requestHint = 2; + return false; + } + + if (shouldBeObjectStart != ObjectStartToken) + { + // Invalid + JsonThrow.UnexpectedToken(source.Consumed, shouldBeObjectStart); + } + + source.Advance(1); + break; + } // case State.Head + case State.Key: + { + if (!source.TryPeek(out var mayBeSeparator)) + { + requestHint = 3; + return false; + } + + switch (mayBeSeparator) + { + case AltKeyValueSepratorToken: + { + if (this._isEqualSignAllowed) + { + goto case KeyValueSepratorToken; + } + else + { + goto default; + } + } // case AltKeyValueSepratorToken + case KeyValueSepratorToken: + { + source.Advance(1); + this._state = State.Value; + requestHint = 0; + return false; + } // case KeyValueSepratorToken + default: + { + JsonThrow.UnexpectedToken(source.Consumed, mayBeSeparator); + // never + requestHint = default; + return false; + } + } // switch (mayBeSeparator) + + } // case State.Key + default: + { + Debug.Assert(this._state == State.Value, $"this._state {(this._state)} == State.Value"); + + if (!source.TryPeek(out var mayBeDelimiterOrObjectEnd)) + { + requestHint = 1; + return false; + } + + switch (mayBeDelimiterOrObjectEnd) + { + case ObjectEndToken: + { + source.Advance(1); + this._state = State.Tail; + requestHint = 0; + return true; + } // case ObjectEndToken + case ItemDelimiterToken: + { + source.Advance(1); + // goto maybe-key + break; + } // case ItemDelimiterToken + default: + { + JsonThrow.UnexpectedToken(source.Consumed, mayBeDelimiterOrObjectEnd); + // never + break; + } + } // switch (mayBeDelimiterOrArrayEnd) + + break; + } // default: + } // swtich (this._state) + + // Handle 'maybe key' state + this._decoder.ReadTrivia(ref source); + if (!source.TryPeek(out var mayBeArrayEnd)) + { + requestHint = 1; + return false; + } + + requestHint = 0; + + if (mayBeArrayEnd == ObjectEndToken) + { + source.Advance(1); + this._state = State.Tail; + return true; + } + else + { + this._state = State.Key; + return false; + } + } + + private enum State + { + Head = 0, + Key, + Value, + Tail + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override CollectionType DecodeArrayOrMap(ref SequenceReader source, out CollectionItemIterator iterator, out int requestHint) + { + this.ReadTrivia(ref source); + + if (!source.TryPeek(out var token)) + { + requestHint = 1; + iterator = default; + return CollectionType.None; + } + + requestHint = 0; + switch (token) + { + case (byte)'[': + { + iterator = this.CreateArrayIterator(); + return CollectionType.Array; + } + case (byte)'{': + { + iterator = this.CreateObjectPropertyIterator(); + return CollectionType.Map; + } + case (byte)'n': + { + iterator = default; + + if (this.TryDecodeNull(ref source, out requestHint)) + { + return CollectionType.Null; + } + + if (requestHint != 0) + { + return CollectionType.None; + } + + goto default; + } + default: + { + var offset = source.Consumed; + var kind = TryGetUtf8Unit(ref source, out var sequence); + if (kind == Utf8UnitStatus.Valid) + { + JsonThrow.IsNotArrayNorObject(sequence, offset); + } + else + { + JsonThrow.MalformedUtf8(sequence, offset); + } + + // never + iterator = default; + return default; + } + } + } + + private CollectionItemIterator CreateArrayIterator() + => new CollectionItemIterator(new ArrayIterator(this).DetectArrayEnds, -1); + + private CollectionItemIterator CreateObjectPropertyIterator() + => new CollectionItemIterator(new ObjectPropertyIterator(this).DetectObjectEnds, -1); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override CollectionItemIterator DecodeArray(ref SequenceReader source, out int requestHint) + { + var type = this.DecodeArrayOrMap(ref source, out var iterator, out requestHint); + if (requestHint != 0) + { + return default; + } + + if (!type.IsArray) + { + JsonThrow.IsNotArray(source.Consumed); + } + + return iterator; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override CollectionItemIterator DecodeMap(ref SequenceReader source, out int requestHint) + { + var type = this.DecodeArrayOrMap(ref source, out var iterator, out requestHint); + if (requestHint != 0) + { + return default; + } + + if (!type.IsMap) + { + JsonThrow.IsNotObject(source.Consumed); + } + + return iterator; + } + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoder.Nullables.cs b/src/MsgPack.Json/Json/JsonDecoder.Nullables.cs new file mode 100644 index 000000000..6351dfc13 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoder.Nullables.cs @@ -0,0 +1,173 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + partial class JsonDecoder + { + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Byte? DecodeNullableByte(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeByteCore(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override SByte? DecodeNullableSByte(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeSByteCore(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Int16? DecodeNullableInt16(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeInt16Core(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override UInt16? DecodeNullableUInt16(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeUInt16Core(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Int32? DecodeNullableInt32(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeInt32Core(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override UInt32? DecodeNullableUInt32(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeUInt32Core(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Int64? DecodeNullableInt64(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeInt64Core(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override UInt64? DecodeNullableUInt64(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeUInt64Core(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Single? DecodeNullableSingle(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeSingleCore(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Double? DecodeNullableDouble(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeDoubleCore(ref source, out requestHint); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Boolean? DecodeNullableBoolean(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeBooleanCore(ref source, out requestHint); + } + + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoder.Nullables.tt b/src/MsgPack.Json/Json/JsonDecoder.Nullables.tt new file mode 100644 index 000000000..ce9e49b17 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoder.Nullables.tt @@ -0,0 +1,57 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ output extension=".cs" #> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + partial class JsonDecoder + { +<# +foreach (var type in + new [] + { + "Byte", + "SByte", + "Int16", + "UInt16", + "Int32", + "UInt32", + "Int64", + "UInt64", + "Single", + "Double", + "Boolean" + } +) +{ +#> + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override <#= type #>? DecodeNullable<#= type #>(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.Decode<#= type #>Core(ref source, out requestHint); + } + +<# +} // foreach (var type) +#> + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoder.NumberCore.cs b/src/MsgPack.Json/Json/JsonDecoder.NumberCore.cs new file mode 100644 index 000000000..d2d0b8f13 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoder.NumberCore.cs @@ -0,0 +1,289 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Buffers.Text; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + public partial class JsonDecoder + { + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override bool DecodeBoolean(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + return this.DecodeBooleanCore(ref source, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private bool DecodeBooleanCore(ref SequenceReader source, out int requestHint) + { + ReadOnlySpan @true; + ReadOnlySpan @false; + unsafe + { + byte* pTrue = stackalloc byte[] { (byte)'t', (byte)'r', (byte)'u', (byte)'e', }; + @true = new ReadOnlySpan(pTrue, 4); + byte* pFalse = stackalloc byte[] { (byte)'f', (byte)'a', (byte)'l', (byte)'s', (byte)'e', }; + @false = new ReadOnlySpan(pFalse, 5); + } + + requestHint = 0; + if (source.IsNext(@true, advancePast: true)) + { + return true; + } + else if (source.IsNext(@false, advancePast: true)) + { + return false; + } + else + { + requestHint = GetRequestHintForBoolean(ref source); + return default; + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int GetRequestHintForBoolean(ref SequenceReader source) + { + if (!source.TryPeek(out var b)) + { + return 4; + } + + switch (b) + { + case (byte)'t': + { + ReadOnlySpan @true; + unsafe + { + byte* pTrue = stackalloc byte[] { (byte)'t', (byte)'r', (byte)'u', (byte)'e' }; + @true = new ReadOnlySpan(pTrue, 4); + } + + return GetRequestHintForBoolean(ref source, offset: 1, @true); + } + case (byte)'f': + { + ReadOnlySpan @false; + unsafe + { + byte* pFalse = stackalloc byte[] { (byte)'f', (byte)'a', (byte)'l', (byte)'s', (byte)'e' }; + @false = new ReadOnlySpan(pFalse, 5); + } + + return GetRequestHintForBoolean(ref source, offset: 1, @false); + } + } + + ThrowNonBooleanUtfSequence(ref source, source.Consumed); + // never + return default; + } + + private static int GetRequestHintForBoolean(ref SequenceReader source, int offset, ReadOnlySpan expected) + { + Span buffer = stackalloc byte[(int)Math.Min(expected.Length, source.Remaining)]; + source.TryCopyTo(buffer); + + for (var i = 0; i < buffer.Length; i++) + { + if (buffer[i] != expected[i]) + { + ThrowNonBooleanUtfSequence(ref source, source.Consumed + i); + } + } + + return expected.Length - buffer.Length; + } + + private static void ThrowNonBooleanUtfSequence(ref SequenceReader source, long position) + { + if (TryGetUtf8Unit(ref source, out var unit) == Utf8UnitStatus.Invalid) + { + JsonThrow.MalformedUtf8(unit, position); + } + else + { + JsonThrow.IsNotType(typeof(bool), unit, position); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private double DecodeNumber(ref SequenceReader source, out int requestHint) + { + var lengthReader = source; + ReadOnlySpan plusMinusSigns; + ReadOnlySpan nonZeroDigits; + ReadOnlySpan digits; + unsafe + { + byte* pPlusMinusSigns = stackalloc[] { (byte)'+', (byte)'-' }; + plusMinusSigns = new ReadOnlySpan(pPlusMinusSigns, 2); + byte* pDigits = stackalloc[] { (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', (byte)'6', (byte)'7', (byte)'8', (byte)'9' }; + digits = new ReadOnlySpan(pDigits, 9); + nonZeroDigits = digits.Slice(1); + } + + var length = 0L; + // -? (0 | [1-9] \d*) (\. \d+ ([eE] [+-] \d+)? )? + // sign + length += lengthReader.AdvancePastAny((byte)'+', (byte)'-'); + + // 0 + if (lengthReader.IsNext((byte)'0', advancePast: true)) + { + length++; + } + // [1-9] + else + { + var nonZeroLength = lengthReader.AdvancePastAny(nonZeroDigits); + if (nonZeroLength > 0) + { + length += nonZeroLength; + // \d* + length += lengthReader.AdvancePastAny(digits); + } + else + { + ThrowInvalidNumber(ref lengthReader); + } + } + + // Check early for integer portion. + if (length > this.Options.MaxNumberLengthInBytes) + { + JsonThrow.TooLongNumber(length, this.Options.MaxNumberLengthInBytes, source.Consumed); + } + + // (\. ...)? + if (lengthReader.IsNext((byte)'.', advancePast: true)) + { + length++; + + if (lengthReader.End) + { + requestHint = 1; + return default; + } + + // \d+ + length += ReadAtLeastOneDigits(ref lengthReader, digits, out requestHint); + if (requestHint != 0) + { + return default; + } + + ReadOnlySpan exponentialIndicators; + unsafe + { + byte* pExponentialIndicators = stackalloc byte[] { (byte)'E', (byte)'e' }; + exponentialIndicators = new ReadOnlySpan(pExponentialIndicators, 2); + } + + // ([eE] ..)? + if (lengthReader.IsNext(exponentialIndicators, advancePast: true)) + { + // [+-] + if (lengthReader.IsNext(plusMinusSigns, advancePast: true)) + { + // \d+ + length += ReadAtLeastOneDigits(ref lengthReader, digits, out requestHint); + if (requestHint != 0) + { + return default; + } + } + else + { + ThrowInvalidNumber(ref lengthReader); + } + } + } + + if (length > this.Options.MaxNumberLengthInBytes) + { + JsonThrow.TooLongNumber(length, this.Options.MaxNumberLengthInBytes, source.Consumed); + } + + if (source.UnreadSpan.Length >= length) + { + var shouldBeTrue = Utf8Parser.TryParse(source.UnreadSpan, out double result, out var consumed); + Debug.Assert(shouldBeTrue, "Utf8Parser.TryParse returns false!"); + Debug.Assert(consumed == length, $"Utf8Parser.TryParse outputs ({consumed}:#,0) is not {length:#,0}!"); + source.Advance(consumed); + requestHint = 0; + return result; + } + + var length32 = (int)length; + byte[]? arrayBuffer = default; + if (length > 32) + { + // length should be Int32 because MaxNumberLength is Int32 type. + arrayBuffer = this.Options.ByteBufferPool.Rent(length32); + } + + try + { + Span buffer = arrayBuffer ?? (stackalloc byte[length32]); + source.TryCopyTo(buffer); + + var shouldBeTrue = Utf8Parser.TryParse(buffer, out double result, out var consumed); + Debug.Assert(shouldBeTrue, "Utf8Parser.TryParse returns false!"); + Debug.Assert(consumed == length, $"Utf8Parser.TryParse outputs ({consumed}:#,0) is not {length:#,0}!"); + source.Advance(consumed); + requestHint = 0; + return result; + } + finally + { + if (arrayBuffer != null) + { + this.Options.ByteBufferPool.Return(arrayBuffer, this.Options.ClearsBuffer); + } + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private static long ReadAtLeastOneDigits(ref SequenceReader source, ReadOnlySpan digits, out int requestHint) + { + if (source.End) + { + requestHint = 1; + return default; + } + + var decimalLength = source.AdvancePastAny(digits); + if (decimalLength == 0) + { + ThrowInvalidNumber(ref source); + } + + requestHint = 0; + return decimalLength; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ThrowInvalidNumber(ref SequenceReader source) + { + var position = source.Consumed; + if (TryGetUtf8Unit(ref source, out var unit) == Utf8UnitStatus.Valid) + { + JsonThrow.IsNotType(typeof(double), unit, position); + } + else + { + JsonThrow.MalformedUtf8(unit, position); + } + } + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoder.Numbers.cs b/src/MsgPack.Json/Json/JsonDecoder.Numbers.cs new file mode 100644 index 000000000..0be7af301 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoder.Numbers.cs @@ -0,0 +1,139 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + partial class JsonDecoder + { + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Byte DecodeByte(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + return this.DecodeByteCore(ref source, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private Byte DecodeByteCore(ref SequenceReader source, out int requestHint) + => (Byte)this.DecodeNumber(ref source, out requestHint); + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override SByte DecodeSByte(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + return this.DecodeSByteCore(ref source, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private SByte DecodeSByteCore(ref SequenceReader source, out int requestHint) + => (SByte)this.DecodeNumber(ref source, out requestHint); + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Int16 DecodeInt16(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + return this.DecodeInt16Core(ref source, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private Int16 DecodeInt16Core(ref SequenceReader source, out int requestHint) + => (Int16)this.DecodeNumber(ref source, out requestHint); + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override UInt16 DecodeUInt16(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + return this.DecodeUInt16Core(ref source, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private UInt16 DecodeUInt16Core(ref SequenceReader source, out int requestHint) + => (UInt16)this.DecodeNumber(ref source, out requestHint); + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Int32 DecodeInt32(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + return this.DecodeInt32Core(ref source, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private Int32 DecodeInt32Core(ref SequenceReader source, out int requestHint) + => (Int32)this.DecodeNumber(ref source, out requestHint); + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override UInt32 DecodeUInt32(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + return this.DecodeUInt32Core(ref source, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private UInt32 DecodeUInt32Core(ref SequenceReader source, out int requestHint) + => (UInt32)this.DecodeNumber(ref source, out requestHint); + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Int64 DecodeInt64(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + return this.DecodeInt64Core(ref source, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private Int64 DecodeInt64Core(ref SequenceReader source, out int requestHint) + => (Int64)this.DecodeNumber(ref source, out requestHint); + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override UInt64 DecodeUInt64(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + return this.DecodeUInt64Core(ref source, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private UInt64 DecodeUInt64Core(ref SequenceReader source, out int requestHint) + => (UInt64)this.DecodeNumber(ref source, out requestHint); + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Single DecodeSingle(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + return this.DecodeSingleCore(ref source, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private Single DecodeSingleCore(ref SequenceReader source, out int requestHint) + => (Single)this.DecodeNumber(ref source, out requestHint); + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override Double DecodeDouble(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + return this.DecodeDoubleCore(ref source, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private Double DecodeDoubleCore(ref SequenceReader source, out int requestHint) + => (Double)this.DecodeNumber(ref source, out requestHint); + + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoder.Numbers.tt b/src/MsgPack.Json/Json/JsonDecoder.Numbers.tt new file mode 100644 index 000000000..5372860bd --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoder.Numbers.tt @@ -0,0 +1,54 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ output extension=".cs" #> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + partial class JsonDecoder + { +<# +foreach (var type in + new [] + { + "Byte", + "SByte", + "Int16", + "UInt16", + "Int32", + "UInt32", + "Int64", + "UInt64", + "Single", + "Double" + } +) +{ +#> + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override <#= type #> Decode<#= type #>(ref SequenceReader source, out int requestHint) + { + this.ReadTrivia(ref source); + return this.Decode<#= type #>Core(ref source, out requestHint); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private <#= type #> Decode<#= type #>Core(ref SequenceReader source, out int requestHint) + => (<#= type #>)this.DecodeNumber(ref source, out requestHint); + +<# +} // foreach (var type) +#> + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoder.SkipDrain.cs b/src/MsgPack.Json/Json/JsonDecoder.SkipDrain.cs new file mode 100644 index 000000000..d45e0c7d5 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoder.SkipDrain.cs @@ -0,0 +1,47 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Threading; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + public partial class JsonDecoder + { + public override void Drain(ref SequenceReader source, in CollectionContext collectionContext, long itemsCount, out int requestHint, CancellationToken cancellationToken = default) + => JsonThrow.DrainIsNotSupported(out requestHint); + + public override void Skip(ref SequenceReader source, in CollectionContext collectionContext, out int requestHint, CancellationToken cancellationToken = default) + { + var originalPosition = source.Consumed; + + if(!this.DecodeItem(ref source, out var decodeItemResult, cancellationToken)) + { + requestHint = (int)(decodeItemResult.RequestHint & Int32.MaxValue); + return; + } + + switch (decodeItemResult.ElementType) + { + case ElementType.Array: + case ElementType.Map: + { + // Skip current collection with CollectionIterator.Drain() + var iterator = decodeItemResult.CollectionIterator; + if (!iterator.Drain(ref source, out requestHint)) + { + source.Rewind(source.Consumed - originalPosition); + return; + } + + break; + } + } + + requestHint = 0; + } + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoder.Strings.cs b/src/MsgPack.Json/Json/JsonDecoder.Strings.cs new file mode 100644 index 000000000..e728da428 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoder.Strings.cs @@ -0,0 +1,508 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Buffers.Text; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + public partial class JsonDecoder + { + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override bool GetRawString(ref SequenceReader source, out ReadOnlySpan rawString, out int requestHint, CancellationToken cancellationToken = default) + { + if (!this.GetRawStringCore(ref source, out var rawStringSequence, out requestHint, cancellationToken)) + { + rawString = default; + return false; + } + + if (rawStringSequence.IsSingleSegment) + { + rawString = rawStringSequence.FirstSpan; + } + else + { + // length should be Int32 because max length option is Int32 type. + Span result = new byte[(int)rawStringSequence.Length]; + rawStringSequence.CopyTo(result); + rawString = result; + } + + return true; + } + + private bool GetRawStringCore(ref SequenceReader source, out ReadOnlySequence rawString, out int requestHint, CancellationToken cancellationToken = default) + { + this.ReadTrivia(ref source); + var startOffset = source.Consumed; + + var quotation = this.ReadStringStart(ref source, out requestHint); + if (requestHint != 0) + { + rawString = default; + return false; + } + + if (!source.TryReadTo(out ReadOnlySequence sequence, quotation, (byte)'\\', advancePastDelimiter: false)) + { + // EoF + requestHint = 1; + rawString = default; + source.Rewind(source.Consumed - startOffset); + return false; + } + + rawString = sequence; + source.Advance(1); // skip end quote + return true; + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override string? DecodeNullableString(ref SequenceReader source, out int requestHint, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeStringCore(ref source, out requestHint, cancellationToken); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override string? DecodeString(ref SequenceReader source, out int requestHint, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + this.ReadTrivia(ref source); + return this.DecodeStringCore(ref source, out requestHint, cancellationToken); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private string? DecodeStringCore(ref SequenceReader source, out int requestHint, CancellationToken cancellationToken = default) + { + var startOffset = source.Consumed; + + var quotation = this.ReadStringStart(ref source, out requestHint); + if (requestHint != 0) + { + return default; + } + + // fast-path + if (!source.TryReadTo(out ReadOnlySequence sequence, quotation, (byte)'\\', advancePastDelimiter: false) || !source.TryRead(out var delimiter)) + { + // EoF + requestHint = 1; + source.Rewind(source.Consumed - startOffset); + return default; + } + + if (sequence.Length > this.Options.MaxStringLengthInBytes) + { + Throw.StringLengthExceeded(source.Consumed - sequence.Length, sequence.Length, this.Options.MaxBinaryLengthInBytes); + } + + if (delimiter == quotation) + { + return Encoding.UTF8.GetString(sequence); + } + + return this.DecodeStringCoreSlow(ref source, out requestHint, startOffset, quotation, ref sequence, delimiter, cancellationToken); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private string? DecodeStringCoreSlow(ref SequenceReader source, out int requestHint, long startOffset, byte quotation, ref ReadOnlySequence sequence, byte delimiter, CancellationToken cancellationToken) + { + var length = sequence.Length; + using (var result = new StringBuilderBufferWriter(new StringBuilder(), this.Options)) + { + Encoding.UTF8.GetChars(sequence, result); + length += (int)sequence.Length; + + // We accept unescaped charactors except quotation even if the value is '\0' because we can handle them correctly. + while (true) + { + // Decode escape sequence + if (source.End) + { + // EoF + requestHint = 2; + break; + } + + if (!source.IsNext((byte)'u', advancePast: true)) + { + DecodeSpetialEscapeSequence(ref source, result); + length += 2; + } + else + { + DecodeUnicodeEscapceSequence(source, result, out requestHint); + if (requestHint != 0) + { + source.Rewind(source.Consumed - startOffset); + return default; + } + + length += 6; + } + + cancellationToken.ThrowIfCancellationRequested(); + + if (!source.TryReadTo(out sequence, quotation, (byte)'\\', advancePastDelimiter: false)) + { + // EoF + requestHint = 1; + source.Rewind(source.Consumed - startOffset); + break; + } + + if (sequence.Length + length > this.Options.MaxStringLengthInBytes) + { + Throw.StringLengthExceeded(source.Consumed - sequence.Length, sequence.Length + length, this.Options.MaxBinaryLengthInBytes); + } + + // Copy & UTF8 decoding existing. + Encoding.UTF8.GetChars(sequence, result); + length += (int)sequence.Length; + + // Handle delimiter + source.TryRead(out delimiter); + if (delimiter == quotation) + { + // End + requestHint = 0; + return result.ToString(); + } + } + } + + // No closing quotation. + return default; + } + + private byte ReadStringStart(ref SequenceReader source, out int requestHint) + { + if (source.End) + { + requestHint = 2; + return default; + } + + requestHint = 0; + + if (source.IsNext((byte)'"', advancePast: false)) + { + source.Advance(1); + return (byte)'"'; + } + else + { + if ((this.Options.ParseOptions & JsonParseOptions.AllowSingleQuotationString) != 0) + { + // ['"] + if (source.IsNext((byte)'\'', advancePast: false)) + { + source.Advance(1); + return (byte)'\''; + } + else + { + JsonThrow.IsNotStringStart(source.Consumed, JsonStringTokens.AnyQuotations); + // never + return default; + } + } + else + { + JsonThrow.IsNotStringStart(source.Consumed, JsonStringTokens.DoubleQuotation); + // never + return default; + } + } + } + + private static void DecodeSpetialEscapeSequence(ref SequenceReader source, StringBuilderBufferWriter result) + { + source.TryPeek(out var escaped); + char unescaped; + switch(escaped) + { + case (byte)'b': + { + unescaped = '\b'; + break; + } + case (byte)'t': + { + unescaped = '\t'; + break; + } + case (byte)'f': + { + unescaped = '\f'; + break; + } + case (byte)'r': + { + unescaped = '\r'; + break; + } + case (byte)'n': + { + unescaped = '\n'; + break; + } + case (byte)'"': + { + unescaped = '"'; + break; + } + case (byte)'\\': + { + unescaped = '\\'; + break; + } + case (byte)'/': + { + unescaped = '/'; + break; + } + default: + { + JsonThrow.InvalidEscapeSequence(source.Consumed - 1, escaped); + // never + unescaped = default; + break; + } + } + + result.AppendUtf16CodePoint(unescaped); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void DecodeUnicodeEscapceSequence(SequenceReader source, StringBuilderBufferWriter result, out int requestHint) + { + Span buffer = stackalloc byte[4]; + if (!source.TryCopyTo(buffer)) + { + requestHint = buffer.Length - (int)source.Remaining; + return; + } + + if (!Utf8Parser.TryParse(buffer, out int codePointOrHighSurrogate, out var consumed, standardFormat: 'X')) + { + JsonThrow.InvalidUnicodeEscapeSequence(source.Consumed - 2, buffer); + } + + Debug.Assert(consumed == 4, $"consumed ({consumed}) == 4 for '{BitConverter.ToString(buffer.ToArray())}'"); + + var positionOf1stSurrogate = source.Consumed - 2; + if (Unicode.IsLowSurrogate(codePointOrHighSurrogate)) + { + // 1st surrogte must be high surrogate. + JsonThrow.OrphanSurrogate(positionOf1stSurrogate, codePointOrHighSurrogate); + } + + source.Advance(consumed); + result.AppendUtf16CodePoint(codePointOrHighSurrogate); + + if (Unicode.IsHighSurrogate(codePointOrHighSurrogate)) + { + if (source.Remaining < 7) + { + requestHint = 7 - (int)source.Remaining; + return; + } + + // Search lower surrogate + if (!source.IsNext((byte)'\\', advancePast: true)) + { + // No paired low surrogate. + JsonThrow.OrphanSurrogate(positionOf1stSurrogate, codePointOrHighSurrogate); + } + + if (!source.IsNext((byte)'u', advancePast: true)) + { + // No paired low surrogate. + JsonThrow.OrphanSurrogate(positionOf1stSurrogate, codePointOrHighSurrogate); + } + + if(!source.TryCopyTo(buffer)) + { + requestHint = buffer.Length - (int)source.Remaining; + return; + } + + source.Advance(4); + + if (!Utf8Parser.TryParse(buffer, out int shouldBeLowSurrogate, out consumed, standardFormat: 'X')) + { + JsonThrow.InvalidUnicodeEscapeSequence(source.Consumed, buffer); + } + + if (!Unicode.IsLowSurrogate(shouldBeLowSurrogate)) + { + // No paired low surrogate. + JsonThrow.OrphanSurrogate(positionOf1stSurrogate, codePointOrHighSurrogate); + } + + Debug.Assert(consumed == 4, $"consumed ({consumed}) == 4 for '{BitConverter.ToString(buffer.ToArray())}'"); + + source.Advance(consumed); + result.AppendUtf16CodePoint(shouldBeLowSurrogate); + } + + requestHint = 0; + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override byte[]? DecodeNullableBinary(ref SequenceReader source, out int requestHint, CancellationToken cancellationToken = default) + { + this.ReadTrivia(ref source); + if (this.TryDecodeNull(ref source)) + { + requestHint = 0; + return null; + } + + return this.DecodeBinaryCore(ref source, out requestHint, cancellationToken); + } + + /// + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override byte[]? DecodeBinary(ref SequenceReader source, out int requestHint, CancellationToken cancellationToken = default) + { + this.ReadTrivia(ref source); + return this.DecodeBinaryCore(ref source, out requestHint, cancellationToken); + } + + private byte[]? DecodeBinaryCore(ref SequenceReader source, out int requestHint, CancellationToken cancellationToken = default) + { + var startOffset = source.Consumed; + + var quotation = this.ReadStringStart(ref source, out requestHint); + if (requestHint != 0) + { + return default; + } + + if (!source.TryReadTo(out ReadOnlySequence sequence, quotation, advancePastDelimiter: true)) + { + // EoF + requestHint = 1; + source.Rewind(source.Consumed - startOffset); + return default; + } + + if (sequence.Length > this.Options.MaxBinaryLengthInBytes) + { + Throw.BinaryLengthExceeded(startOffset + 1, sequence.Length, this.Options.MaxBinaryLengthInBytes); + } + + var resultLength = 0; + var inputBuffer = sequence.IsSingleSegment ? null : this.Options.ByteBufferPool.Rent(this.Options.MaxByteBufferLength); + try + { + var outputBuffer = this.Options.ByteBufferPool.Rent(this.Options.MaxByteBufferLength); + try + { + ReadOnlySpan inputSpan = sequence.FirstSpan; + Span outputSpan = outputBuffer; + var status = OperationStatus.DestinationTooSmall; + while (true) + { + status = Base64.DecodeFromUtf8(inputSpan, outputSpan, out var bytesConsumed, out var bytesWritten, isFinalBlock: sequence.IsSingleSegment); + inputSpan = inputSpan.Slice(bytesConsumed); + outputSpan = outputSpan.Slice(bytesWritten); + + resultLength += bytesWritten; + switch (status) + { + case OperationStatus.Done: + { + // OK + var result = new byte[resultLength]; + unsafe + { + fixed (byte* pBuffer = outputBuffer) + fixed (byte* pResult = result) + { + Buffer.MemoryCopy(pBuffer, pResult, result.Length, resultLength); + } + } + + return result; + } + case OperationStatus.DestinationTooSmall: + { + // Realloc buffer and set span head. + + var newBuffer = this.Options.ByteBufferPool.Rent(outputBuffer.Length * 2); + unsafe + { + fixed (byte* pBuffer = outputBuffer) + fixed (byte* pNewBuffer = newBuffer) + { + Buffer.MemoryCopy(pBuffer, pNewBuffer, outputBuffer.Length, outputBuffer.Length); + } + } + + outputSpan = newBuffer.AsSpan(outputBuffer.Length - outputSpan.Length); + this.Options.ByteBufferPool.Return(outputBuffer, this.Options.ClearsBuffer); + outputBuffer = newBuffer; + break; + } + case OperationStatus.NeedMoreData: + { + // In this case, ReadOnlySequence is multi segment. + Debug.Assert(!sequence.IsSingleSegment, "!sequence.IsSingleSegment"); + Debug.Assert(inputBuffer != null, "inputBuffer != null"); + + // Realloc input span. + + var consumed = inputBuffer.Length - inputSpan.Length; + + sequence = sequence.Slice(consumed); + var newInputSpanLength = (int)Math.Min(inputBuffer.Length, sequence.Length); + sequence.Slice(0, newInputSpanLength).CopyTo(inputBuffer); + inputSpan = inputBuffer.AsSpan(0, newInputSpanLength); + break; + } + default: + { + Debug.Assert(status == OperationStatus.InvalidData, $"status ({status}) == OperationStatus.InvalidData"); + JsonThrow.InvalidBase64(startOffset); + // never + return default; + } + } + } + } + finally + { + this.Options.ByteBufferPool.Return(outputBuffer, this.Options.ClearsBuffer); + } + } + finally + { + if (!(inputBuffer is null)) + { + this.Options.ByteBufferPool.Return(inputBuffer, this.Options.ClearsBuffer); + } + } + } + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoder.cs b/src/MsgPack.Json/Json/JsonDecoder.cs new file mode 100644 index 000000000..4de440a9e --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoder.cs @@ -0,0 +1,155 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Text; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + /// + /// A decoder for JSON format. + /// + public abstract partial class JsonDecoder : FormatDecoder + { + public new JsonDecoderOptions Options { get; } + + protected JsonDecoder(JsonDecoderOptions options) + : base(options) + { + this.Options = options; + } + + public static JsonDecoder Create(JsonDecoderOptions options) + { + if ((options?.ParseOptions).GetValueOrDefault() == JsonParseOptions.None) + { + return new SimpleJsonDecoder(options!); + } + else + { + return new FlexibleJsonDecoder(options!); + } + } + + private protected static Utf8UnitStatus TryGetUtf8Unit(ref SequenceReader source, out ReadOnlySequence unit) + { + if (source.UnreadSpan.IsEmpty) + { + unit = default; + return Utf8UnitStatus.TooShort; + } + + var utf80 = source.UnreadSpan[0]; + if ((utf80 & 0b_1000_0000) == 0) + { + // 1byte + unit = source.Sequence.Slice(source.Position, 1); + return Utf8UnitStatus.Valid; + } + else if ((utf80 & 0b_1110_0000) == 0b_1100_0000) + { + Span utf8 = stackalloc byte[2]; + if (!source.TryCopyTo(utf8)) + { + unit = source.Sequence.Slice(source.Position); + return Utf8UnitStatus.TooShort; + } + + unit = source.Sequence.Slice(source.Position, 2); + + // 2 bytes, 5 + 6 bits + var bits1 = utf8[0] & 0b_0001_1111; + var bits2 = utf8[1] & 0b_0011_1111; + + if ((bits1 & 0b_1_1110) == 0 + || (utf8[1] & 0b_1100_0000) != 0b_1000_0000) + { + return Utf8UnitStatus.Invalid; + } + + return Utf8UnitStatus.Valid; + } + else if ((utf80 & 0b_1111_0000) == 0b_1110_0000) + { + Span utf8 = stackalloc byte[3]; + if (!source.TryCopyTo(utf8)) + { + unit = source.Sequence.Slice(source.Position); + return Utf8UnitStatus.TooShort; + } + + unit = source.Sequence.Slice(source.Position, 3); + + if ((utf8[1] & 0b_1100_0000) != 0b_1000_0000 + || (utf8[2] & 0b_1100_0000) != 0b_1000_0000) + { + return Utf8UnitStatus.Invalid; + } + + // 3 bytes, 4 + 6 + 6 bits + var bits1 = utf8[0] & 0b_0000_1111; + var bits2 = utf8[1] & 0b_0011_1111; + var bits3 = utf8[2] & 0b_0011_1111; + + ushort codePoint = (ushort)((bits1 << 12) | (bits2 << 6) | bits3); + + if ((codePoint & 0b_1111_100000_000000) == 0 || !Rune.IsValid(codePoint)) + { + return Utf8UnitStatus.Invalid; + } + + return Utf8UnitStatus.Valid; + } + else if ((utf80 & 0b_1111_1000) == 0b_1111_0000) + { + Span utf8 = stackalloc byte[4]; + if (!source.TryCopyTo(utf8)) + { + unit = source.Sequence.Slice(source.Position); + return Utf8UnitStatus.TooShort; + } + + unit = source.Sequence.Slice(source.Position, 4); + + if ((utf8[1] & 0b_1100_0000) != 0b_1000_0000 + || (utf8[2] & 0b_1100_0000) != 0b_1000_0000 + || (utf8[3] & 0b_1100_0000) != 0b_1000_0000) + { + return Utf8UnitStatus.Invalid; + } + + // 4 bytes, 3 + 6 + 6 + 6 bits + var bits1 = (utf8[0] & 0b_0000_0111) << 18; + var bits2 = (utf8[1] & 0b_0011_1111) << 12; + var bits3 = (utf8[2] & 0b_0011_1111) << 6; + var bits4 = (utf8[3] & 0b_0011_1111); + + int codePoint = ((bits1 << 18) | (bits2 << 12) | (bits1 << 6) | bits4); + + if ((codePoint & 0b_111_110000_000000_000000) == 0) + { + return Utf8UnitStatus.Invalid; + } + + return Utf8UnitStatus.Valid; + } + else + { + unit = source.Sequence.Slice(source.Position, 1); + return Utf8UnitStatus.Invalid; + } + } + + protected abstract long ReadTrivia(ref SequenceReader source); + + protected enum Utf8UnitStatus + { + TooShort = 0, + Valid = 1, + Invalid = 2 + } + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoderOptions.cs b/src/MsgPack.Json/Json/JsonDecoderOptions.cs new file mode 100644 index 000000000..bbb295938 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoderOptions.cs @@ -0,0 +1,24 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using MsgPack.Internal; + +namespace MsgPack.Json +{ + /// + /// Defines decoder options for . + /// + public class JsonDecoderOptions : FormatDecoderOptions + { + public static JsonDecoderOptions Default { get; } = new JsonDecoderOptionsBuilder().Build(); + + public JsonParseOptions ParseOptions { get; } + + public JsonDecoderOptions(JsonDecoderOptionsBuilder builder) + : base(builder, JsonFormatFeatures.Value) + { + this.ParseOptions = builder.ParseOptions; + } + } +} diff --git a/src/MsgPack.Json/Json/JsonDecoderOptionsBuilder.cs b/src/MsgPack.Json/Json/JsonDecoderOptionsBuilder.cs new file mode 100644 index 000000000..7bb7bffcb --- /dev/null +++ b/src/MsgPack.Json/Json/JsonDecoderOptionsBuilder.cs @@ -0,0 +1,20 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using MsgPack.Internal; + +namespace MsgPack.Json +{ + /// + /// A builder object for immutable . + /// + public class JsonDecoderOptionsBuilder : FormatDecoderOptionsBuilder + { + public JsonParseOptions ParseOptions { get; set; } + + public JsonDecoderOptionsBuilder() { } + + public JsonDecoderOptions Build() => new JsonDecoderOptions(this); + } +} diff --git a/src/MsgPack.Json/Json/JsonEncoder.cs b/src/MsgPack.Json/Json/JsonEncoder.cs new file mode 100644 index 000000000..b4109419d --- /dev/null +++ b/src/MsgPack.Json/Json/JsonEncoder.cs @@ -0,0 +1,834 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Buffers.Text; +using System.Diagnostics; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + /// + /// An encoder for JSON format. + /// + public sealed class JsonEncoder : FormatEncoder + { + private readonly Action, JsonEncoderOptions> _singleInfinityFormatter; + private readonly Action, JsonEncoderOptions> _singleNanFormatter; + private readonly Action, JsonEncoderOptions> _doubleInfinityFormatter; + private readonly Action, JsonEncoderOptions> _doubleNanFormatter; + private readonly ReadOnlyMemory _indentChars; + private readonly ReadOnlyMemory _newLineChars; + private readonly ReadOnlyMemory _escapeTargetChars1Byte; + private readonly ReadOnlyMemory _escapeTargetChars2Byte; + private readonly ReadOnlyMemory _escapeTargetChars4Byte; + private readonly bool _isPrettyPrint; + private readonly JsonEncoderOptions _options; + + public JsonEncoder(JsonEncoderOptions options) + : base(options) + { + this._options = options; + this._singleInfinityFormatter = options.SingleInfinityFormatter; + this._singleNanFormatter = options.SingleNaNFormatter; + this._doubleInfinityFormatter = options.DoubleInfinityFormatter; + this._doubleNanFormatter = options.DoubleNaNFormatter; + this._indentChars = options.IndentChars; + this._newLineChars = options.NewLineChars; + this._isPrettyPrint = options.IsPrettyPrint; + this._escapeTargetChars1Byte = options.EscapeTargetChars1Byte; + this._escapeTargetChars2Byte = options.EscapeTargetChars2Byte; + this._escapeTargetChars4Byte = options.EscapeTargetChars4Byte; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeRawString(ReadOnlySpan rawString, int charLength, IBufferWriter buffer, CancellationToken cancellationToken = default) + => Ensure.NotNull(buffer).Write(rawString); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeInt32(int value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var span = buffer.GetSpan(11); + while (true) + { + if (!Utf8Formatter.TryFormat(value, span, out var used)) + { + span = buffer.GetSpan(span.Length * 2); + } + else + { + buffer.Advance(used); + break; + } + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeUInt32(uint value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var span = buffer.GetSpan(10); + while (true) + { + if (!Utf8Formatter.TryFormat(value, span, out var used)) + { + span = buffer.GetSpan(span.Length * 2); + } + else + { + buffer.Advance(used); + break; + } + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeInt64(long value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var span = buffer.GetSpan(20); + while (true) + { + if (!Utf8Formatter.TryFormat(value, span, out var used)) + { + span = buffer.GetSpan(span.Length * 2); + } + else + { + buffer.Advance(used); + break; + } + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeUInt64(ulong value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + var span = buffer.GetSpan(19); + while (true) + { + if (!Utf8Formatter.TryFormat(value, span, out var used)) + { + span = buffer.GetSpan(span.Length * 2); + } + else + { + buffer.Advance(used); + break; + } + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeSingle(float value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + if (Single.IsNaN(value)) + { + this._singleNanFormatter(value, buffer, this._options); + } + else if (Single.IsInfinity(value)) + { + this._singleInfinityFormatter(value, buffer, this._options); + } + else + { + JsonFormatter.Format(value, buffer); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeDouble(double value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + if (Double.IsNaN(value)) + { + this._doubleNanFormatter(value, buffer, this._options); + } + else if (Double.IsInfinity(value)) + { + this._doubleInfinityFormatter(value, buffer, this._options); + } + else + { + JsonFormatter.Format(value, buffer); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeBoolean(bool value, IBufferWriter buffer) + { + buffer = Ensure.NotNull(buffer); + + if (value) + { + var span = buffer.GetSpan(4); + span[0] = (byte)'t'; + span[1] = (byte)'r'; + span[2] = (byte)'u'; + span[3] = (byte)'e'; + buffer.Advance(4); + } + else + { + var span = buffer.GetSpan(5); + span[0] = (byte)'f'; + span[1] = (byte)'a'; + span[2] = (byte)'l'; + span[3] = (byte)'s'; + span[4] = (byte)'e'; + buffer.Advance(5); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeNull(IBufferWriter buffer) + => JsonFormatter.WriteNull(Ensure.NotNull(buffer)); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private void WriteIndent(IBufferWriter buffer, in CollectionContext collectionContext) + { + if (this._isPrettyPrint) + { + this.WriteIndentCore(buffer, collectionContext); + } + } + + private void WriteIndentCore(IBufferWriter buffer, CollectionContext collectionContext) + { + Debug.Assert(buffer != null); + buffer.Write(this._newLineChars.Span); + for (var i = 0; i < collectionContext.CurrentDepth; i++) + { + buffer.Write(this._indentChars.Span); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeArrayStart(int length, IBufferWriter buffer, in CollectionContext collectionContext) + { + buffer = Ensure.NotNull(buffer); + + buffer.GetSpan(1)[0] = (byte)'['; + buffer.Advance(1); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeArrayEnd(int length, IBufferWriter buffer, in CollectionContext collectionContext) + { + buffer = Ensure.NotNull(buffer); + + this.WriteIndent(buffer, collectionContext); + buffer.GetSpan(1)[0] = (byte)']'; + buffer.Advance(1); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeArrayItemStart(int index, IBufferWriter buffer, in CollectionContext collectionContext) + { + buffer = Ensure.NotNull(buffer); + + this.WriteIndent(buffer, collectionContext); + if (index > 0) + { + buffer.GetSpan(1)[0] = (byte)','; + buffer.Advance(1); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeArrayItemEnd(int index, IBufferWriter buffer, in CollectionContext collectionContext) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeMapStart(int length, IBufferWriter buffer, in CollectionContext collectionContext) + { + buffer = Ensure.NotNull(buffer); + + buffer.GetSpan(1)[0] = (byte)'{'; + buffer.Advance(1); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeMapEnd(int length, IBufferWriter buffer, in CollectionContext collectionContext) + { + buffer = Ensure.NotNull(buffer); + + this.WriteIndent(buffer, collectionContext); + buffer.GetSpan(1)[0] = (byte)'}'; + buffer.Advance(1); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeMapKeyStart(int index, IBufferWriter buffer, in CollectionContext collectionContext) + { + buffer = Ensure.NotNull(buffer); + + this.WriteIndent(buffer, collectionContext); + if (index > 0) + { + buffer.GetSpan(1)[0] = (byte)','; + buffer.Advance(1); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeMapKeyEnd(int index, IBufferWriter buffer, in CollectionContext collectionContext) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeMapValueStart(int index, IBufferWriter buffer, in CollectionContext collectionContext) + { + buffer = Ensure.NotNull(buffer); + + if (!this._isPrettyPrint) + { + buffer.GetSpan(1)[0] = (byte)':'; + buffer.Advance(1); + } + else + { + var span = buffer.GetSpan(2); + span[0] = (byte)':'; + span[1] = (byte)' '; + buffer.Advance(2); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeMapValueEnd(int index, IBufferWriter buffer, in CollectionContext collectionContext) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + private static ReadOnlySpan GetEscapeSequence(byte c, bool newLineAllowed, bool tabAllowed, bool escapeHtmlChars) + { + switch (c) + { + case (byte)'\b': + { + return JsonEscapeSequence.BackSpace; + } + case (byte)'\r': + { + if (newLineAllowed) + { + // Does not escape. + return JsonCharactor.CarriageReturn; + } + + return JsonEscapeSequence.CarriageReturn; + } + case (byte)'\f': + { + return JsonEscapeSequence.FormFeed; + } + case (byte)'\n': + { + if (newLineAllowed) + { + // Does not escape. + return JsonCharactor.LineFeed; + } + + return JsonEscapeSequence.LineFeed; + } + case (byte)'"': + { + if (escapeHtmlChars) + { + return ReadOnlySpan.Empty; + } + + return JsonEscapeSequence.Quatation; + } + case (byte)'\\': + { + return JsonEscapeSequence.ReverseSolidous; + } + case (byte)'\t': + { + if (newLineAllowed) + { + // Does not escape. + return JsonCharactor.Tab; + } + + return JsonEscapeSequence.Tab; + } + } + + // Use \uXXXX + return ReadOnlySpan.Empty; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override void EncodeString(ReadOnlySpan encodedValue, int charLength, IBufferWriter buffer, CancellationToken cancellationToken = default) + { + buffer = Ensure.NotNull(buffer); + + buffer.GetSpan(1)[0] = (byte)'"'; + buffer.Advance(1); + + var source = encodedValue; + this.EncodeStringBody(ref source, 0, buffer, out var requestHint); + if (requestHint > 0) + { + JsonThrow.TooShortUtf8(); + } + + // End quot + buffer.GetSpan(1)[0] = (byte)'"'; + buffer.Advance(1); + } + + private void EncodeStringBody(ref ReadOnlySpan encodedValue, long position, IBufferWriter buffer, out int requestHint) + { + var utf8 = encodedValue; + while (utf8.Length > 0) + { + if ((utf8[0] & 0b_1000_0000) == 0) + { + // 1 byte, 7bits + byte codePoint = utf8[0]; + + if (this._escapeTargetChars1Byte.Span.Contains(codePoint) + || codePoint < 0x20 // Control chars must be escaped + || codePoint == 0x7F // Control chars must be escaped (DEL) + ) + { + var escapeSequence = GetEscapeSequence(codePoint, newLineAllowed: false, tabAllowed: !this._options.EscapesHorizontalTab, this._options.EscapesHtmlChars); + if (escapeSequence.IsEmpty) + { + EscapeCodePoint(codePoint, buffer); + } + else + { + buffer.Write(escapeSequence); + } + } + else + { + buffer.Write(utf8.Slice(0, 1)); + } + + utf8 = utf8.Slice(1); + position += 1; + } + + else if ((utf8[0] & 0b_1110_0000) == 0b_1100_0000) + { + if (utf8.Length < 2) + { + requestHint = 1; + return; + } + + // 2 bytes, 5 + 6 bits + var bits1 = utf8[0] & 0b_0001_1111; + var bits2 = utf8[1] & 0b_0011_1111; + + if ((bits1 & 0b_1_1110) == 0 + || (utf8[1] & 0b_1100_0000) != 0b_1000_0000) + { + JsonThrow.MalformedUtf8(utf8, position); + } + + ushort codePoint = (ushort)((bits1 << 6) | (bits2 << 6)); + + if (codePoint < 0xA0 // Control chars must be escaped + || this._escapeTargetChars2Byte.Span.Contains(codePoint)) + { + EscapeCodePoint(codePoint, buffer); + } + else + { + buffer.Write(utf8.Slice(0, 1)); + } + + utf8 = utf8.Slice(2); + position += 2; + } + else if ((utf8[0] & 0b_1111_0000) == 0b_1110_0000) + { + if (utf8.Length < 3) + { + requestHint = 3 - utf8.Length; + return; + } + + if ((utf8[1] & 0b_1100_0000) != 0b_1000_0000 + || (utf8[2] & 0b_1100_0000) != 0b_1000_0000) + { + JsonThrow.MalformedUtf8(utf8, position); + } + + // 3 bytes, 4 + 6 + 6 bits + var bits1 = utf8[0] & 0b_0000_1111; + var bits2 = utf8[1] & 0b_0011_1111; + var bits3 = utf8[2] & 0b_0011_1111; + + ushort codePoint = (ushort)((bits1 << 12) | (bits2 << 6) | bits3); + + if ((codePoint & 0b_1111_100000_000000) == 0) + { + JsonThrow.MalformedUtf8(utf8, position); + } + + if (!Rune.IsValid(codePoint)) + { + JsonThrow.SurrogateCharInUtf8(position, codePoint); + } + + if (codePoint >= 0xFFFE // U+FFFE and U+FFFF are Reserved + || (this._options.EscapesPrivateUseCharactors && codePoint >= 0xE000 && codePoint <= 0xF8FF) // Privte Use + || this._escapeTargetChars2Byte.Span.Contains(codePoint)) + { + EscapeCodePoint(codePoint, buffer); + } + else + { + buffer.Write(utf8.Slice(0, 1)); + } + + utf8 = utf8.Slice(3); + position += 3; + } + else if ((utf8[0] & 0b_1111_1000) == 0b_1111_0000) + { + if (utf8.Length < 4) + { + requestHint = 4 - utf8.Length; + return; + } + + if ((utf8[1] & 0b_1100_0000) != 0b_1000_0000 + || (utf8[2] & 0b_1100_0000) != 0b_1000_0000 + || (utf8[3] & 0b_1100_0000) != 0b_1000_0000) + { + JsonThrow.MalformedUtf8(utf8, position); + } + + // 4 bytes, 3 + 6 + 6 + 6 bits + var bits1 = (utf8[0] & 0b_0000_0111) << 18; + var bits2 = (utf8[1] & 0b_0011_1111) << 12; + var bits3 = (utf8[2] & 0b_0011_1111) << 6; + var bits4 = (utf8[3] & 0b_0011_1111); + + int codePoint = ((bits1 << 18) | (bits2 << 12) | (bits3 << 6) | bits4); + + if ((codePoint & 0b_111_110000_000000_000000) == 0) + { + JsonThrow.MalformedUtf8(utf8, position); + } + + if ((this._options.EscapesPrivateUseCharactors && codePoint >= 0xF0000) // Private use + || this._escapeTargetChars4Byte.Span.Contains(codePoint)) + { + EscapeCodePoint(codePoint, buffer); + } + else + { + buffer.Write(utf8.Slice(0, 1)); + } + + utf8 = utf8.Slice(4); + position += 4; + } + else + { + JsonThrow.MalformedUtf8(utf8, position); + } + } + + requestHint = 0; + } + + private static void EscapeCodePoint(int codePoint, IBufferWriter buffer) + { + buffer.Write(JsonEscapeSequence.Unicode); + Span numbers = stackalloc byte[4]; + Utf8Formatter.TryFormat(codePoint, numbers, out _, JsonEscapeSequence.UnicodeFormat); + buffer.Write(numbers); + } + + public sealed override void EncodeString(in ReadOnlySequence encodedValue, int charLength, IBufferWriter buffer, CancellationToken cancellationToken = default) + { + buffer = Ensure.NotNull(buffer); + + // Start quot + buffer.GetSpan(1)[0] = (byte)'"'; + buffer.Advance(1); + + var source = new SequenceReader(encodedValue); + + while (!source.End) + { + var length = source.UnreadSpan.Length; + var span = source.UnreadSpan; + this.EncodeStringBody(ref span, source.Consumed, buffer, out var requestHint); + source.Advance(length - span.Length); + + if (requestHint > 0) + { + Span codePoints = stackalloc byte[span.Length + requestHint]; + if (!source.TryCopyTo(codePoints)) + { + JsonThrow.TooShortUtf8(); + } + + source.Advance(codePoints.Length); + + ReadOnlySpan readOnlyCodePoints = codePoints; + this.EncodeStringBody(ref readOnlyCodePoints, source.Consumed, buffer, out requestHint); + } + } + + // End quot + buffer.GetSpan(1)[0] = (byte)'"'; + buffer.Advance(1); + } + + public sealed override void EncodeString(ReadOnlySpan value, IBufferWriter buffer, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + buffer = Ensure.NotNull(buffer); + buffer.GetSpan(1)[0] = (byte)'"'; + buffer.Advance(1); + + var source = value; + this.EncodeStringBody(ref source, buffer, out var requestHint); + if (requestHint > 0) + { + JsonThrow.OrphanSurrogate(value.Length - source.Length, source[0]); + } + + buffer.GetSpan(1)[0] = (byte)'"'; + buffer.Advance(1); + } + + public sealed override void EncodeString(in ReadOnlySequence value, IBufferWriter buffer, Encoding? encoding = null, CancellationToken cancellationToken = default) + { + if (value.IsSingleSegment) + { + this.EncodeString(value.FirstSpan, buffer, encoding); + return; + } + else + { + buffer = Ensure.NotNull(buffer); + + // Start quot + buffer.GetSpan(1)[0] = (byte)'"'; + buffer.Advance(1); + + var source = new SequenceReader(value); + + while (!source.End) + { + var length = source.UnreadSpan.Length; + var span = source.UnreadSpan; + this.EncodeStringBody(ref span, buffer, out var requestHint); + source.Advance(length - span.Length); + + if (requestHint > 0) + { + Span codePoints = stackalloc char[span.Length + requestHint]; + if (!source.TryCopyTo(codePoints)) + { + JsonThrow.TooShortUtf8(); + } + + source.Advance(codePoints.Length); + + ReadOnlySpan readOnlyCodePoints = codePoints; + this.EncodeStringBody(ref readOnlyCodePoints, buffer, out requestHint); + } + } + + // End quot + buffer.GetSpan(1)[0] = (byte)'"'; + buffer.Advance(1); + } + } + + private void EncodeStringBody(ref ReadOnlySpan chars, IBufferWriter buffer, out int requestHint) + { + var position = 0; + for(var i = 0; i < chars.Length; i++) + { + var c = chars[i]; + if (c < 0x80) + { + var codePoint = (byte)c; + + if (this._escapeTargetChars1Byte.Span.Contains(codePoint) + || codePoint < 0x20 // Control chars must be escaped + || codePoint == 0x7F // Control chars must be escaped (DEL) + ) + { + Utf8EncodingNonBom.Instance.GetBytes(chars.Slice(0, i), buffer); + chars = chars.Slice(i + 1); + position += i + 1; + i = 0; + + var escapeSequence = GetEscapeSequence(codePoint, newLineAllowed: false, tabAllowed: !this._options.EscapesHorizontalTab, this._options.EscapesHtmlChars); + if (escapeSequence.IsEmpty) + { + EscapeCodePoint(codePoint, buffer); + } + else + { + buffer.Write(escapeSequence); + } + } + } + else if (c < 0x800) + { + ushort codePoint = c; + + if (codePoint < 0xA0 // Control chars must be escaped + || this._escapeTargetChars2Byte.Span.Contains(codePoint)) + { + Utf8EncodingNonBom.Instance.GetBytes(chars.Slice(0, i), buffer); + chars = chars.Slice(i); + position += i + 1; + i = 0; + + EscapeCodePoint(codePoint, buffer); + } + } + else if (!Char.IsSurrogate(c)) + { + ushort codePoint = c; + + if (codePoint >= 0xFFFE // U+FFFE and U+FFFF are Reserved + || (this._options.EscapesPrivateUseCharactors && codePoint >= 0xE000 && codePoint <= 0xF8FF) // Privte Use + || this._escapeTargetChars2Byte.Span.Contains(codePoint)) + { + Utf8EncodingNonBom.Instance.GetBytes(chars.Slice(0, i), buffer); + chars = chars.Slice(i); + position += i + 1; + i = 0; + + EscapeCodePoint(codePoint, buffer); + } + } + else + { + // Surrogate + if(chars.Length < i + 2) + { + Utf8EncodingNonBom.Instance.GetBytes(chars.Slice(0, i), buffer); + chars = chars.Slice(i); + position += i; + i = 0; + requestHint = 1; + return; + } + + if(!Char.IsSurrogatePair(c, chars[1])) + { + JsonThrow.OrphanSurrogate(position, c); + } + + var codePoint = Char.ConvertToUtf32(c, chars[1]); + + if ((this._options.EscapesPrivateUseCharactors && codePoint >= 0xF0000) // Private use + || this._escapeTargetChars4Byte.Span.Contains(codePoint)) + { + EscapeCodePoint(codePoint, buffer); + } + + position += 2; + i++; + } + } + + requestHint = 0; + Utf8EncodingNonBom.Instance.GetBytes(chars, buffer); + chars = ReadOnlySpan.Empty; + } + + public sealed override void EncodeBinary(ReadOnlySpan value, IBufferWriter buffer, CancellationToken cancellationToken = default) + { + // Start quot + buffer.GetSpan(1)[0] = (byte)'"'; + buffer.Advance(1); + + if (value.Length < this.Options.CancellationSupportThreshold) + { + // Enough space -- over 138% + var span = buffer.GetSpan((int)(value.Length * 1.38)); + Base64.EncodeToUtf8(value, span, out var bytesConsumed, out var bytesWritten, isFinalBlock: true); + buffer.Advance(bytesWritten); + } + else + { + EncodeBinarySlow(value, buffer, this.Options.CancellationSupportThreshold, cancellationToken); + } + + // End quot + buffer.GetSpan(1)[0] = (byte)'"'; + buffer.Advance(1); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void EncodeBinarySlow(ReadOnlySpan value, IBufferWriter buffer, int bufferSize, CancellationToken cancellationToken) + { + while (true) + { + var span = buffer.GetSpan(bufferSize); + if (Base64.EncodeToUtf8(value, span, out var bytesConsumed, out var bytesWritten, isFinalBlock: value.IsEmpty) == OperationStatus.Done) + { + break; + } + + buffer.Advance(bytesWritten); + value = value.Slice(bytesConsumed); + cancellationToken.ThrowIfCancellationRequested(); + } + } + + public sealed override void EncodeBinary(in ReadOnlySequence value, IBufferWriter buffer, CancellationToken cancellationToken = default) + { + if (value.IsSingleSegment) + { + // Fast-path + this.EncodeBinary(value.FirstSpan, buffer, cancellationToken); + return; + } + + // Start quot + buffer.GetSpan(1)[0] = (byte)'"'; + buffer.Advance(1); + + var reader = new SequenceReader(value); + while (true) + { + var span = buffer.GetSpan(this.Options.CancellationSupportThreshold); + if (Base64.EncodeToUtf8(reader.UnreadSpan, span, out var bytesConsumed, out var bytesWritten, isFinalBlock: reader.End) == OperationStatus.Done) + { + break; + } + + buffer.Advance(bytesWritten); + reader.Advance(bytesConsumed); + cancellationToken.ThrowIfCancellationRequested(); + } + + // End quot + buffer.GetSpan(1)[0] = (byte)'"'; + buffer.Advance(1); + } + } +} diff --git a/src/MsgPack.Json/Json/JsonEncoderOptions.cs b/src/MsgPack.Json/Json/JsonEncoderOptions.cs new file mode 100644 index 000000000..0399e13b9 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonEncoderOptions.cs @@ -0,0 +1,120 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Linq; +using System.Text; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + /// + /// Defines encoder options for . + /// + public sealed class JsonEncoderOptions : FormatEncoderOptions + { + public static JsonEncoderOptions Default { get; } = new JsonEncoderOptionsBuilder().Build(); + + private static readonly Action, JsonEncoderOptions> ErrorSingleInfinityFormatter = + (value, buffer, options) => throw new ArgumentException($"Cannot serialize infinity ({value}) to JSON.", "value"); + + private static readonly Action, JsonEncoderOptions> MinMaxSingleInfinityFormatter = + (value, buffer, _) => JsonFormatter.Format(Single.IsPositiveInfinity(value) ? Single.MaxValue : Single.MinValue, buffer); + + private static readonly Action, JsonEncoderOptions> ErrorDoubleInfinityFormatter = + (value, buffer, options) => throw new ArgumentException($"Cannot serialize infinity ({value}) to JSON.", "value"); + + private static readonly Action, JsonEncoderOptions> MinMaxDoubleInfinityFormatter = + (value, buffer, _) => JsonFormatter.Format(Double.IsPositiveInfinity(value) ? Double.MaxValue : Double.MinValue, buffer); + + private static readonly Action, JsonEncoderOptions> ErrorSingleNaNFormatter = + (value, buffer, options) => throw new ArgumentException($"Cannot serialize NaN to JSON.", "value"); + private static readonly Action, JsonEncoderOptions> NullSingleNaNFormatter = + (value, buffer, options) => JsonFormatter.WriteNull(buffer); + + private static readonly Action, JsonEncoderOptions> ErrorDoubleNaNFormatter = + (value, buffer, options) => throw new ArgumentException($"Cannot serialize NaN to JSON.", "value"); + + private static readonly Action, JsonEncoderOptions> NullDoubleNaNFormatter = + (value, buffer, options) => JsonFormatter.WriteNull(buffer); + + private readonly Action, JsonEncoderOptions>? _singleInfinityFormatter; + private readonly Action, JsonEncoderOptions>? _doubleInfinityFormatter; + private readonly Action, JsonEncoderOptions>? _singleNaNFormatter; + private readonly Action, JsonEncoderOptions>? _doubleNaNFormatter; + + public ReadOnlyMemory IndentChars { get; } + public ReadOnlyMemory NewLineChars { get; } + public bool IsPrettyPrint { get; } + public bool EscapesHorizontalTab { get; } + public bool EscapesHtmlChars { get; } + public bool EscapesPrivateUseCharactors { get; } + public ReadOnlyMemory EscapeTargetChars { get; } + internal ReadOnlyMemory EscapeTargetChars1Byte { get; } + internal ReadOnlyMemory EscapeTargetChars2Byte { get; } + internal ReadOnlyMemory EscapeTargetChars4Byte { get; } + + public JsonEncoderOptions(JsonEncoderOptionsBuilder builder) + : base(builder, JsonFormatFeatures.Value) + { + this._singleInfinityFormatter = + builder.InfinityHandling switch + { + InfinityHandling.Custom => builder.SingleInfinityFormatter, + InfinityHandling.Error => ErrorSingleInfinityFormatter, + InfinityHandling.MinMax => MinMaxSingleInfinityFormatter, + _ => default, + }; + this._doubleInfinityFormatter = + builder.InfinityHandling switch + { + InfinityHandling.Custom => builder.DoubleInfinityFormatter, + InfinityHandling.Error => ErrorDoubleInfinityFormatter, + InfinityHandling.MinMax => MinMaxDoubleInfinityFormatter, + _ => default, + }; + this._singleNaNFormatter = + builder.NaNHandling switch + { + NaNHandling.Custom => builder.SingleNaNFormatter, + NaNHandling.Error => ErrorSingleNaNFormatter, + NaNHandling.Null => NullSingleNaNFormatter, + _ => default, + }; + this._doubleNaNFormatter = + builder.NaNHandling switch + { + NaNHandling.Custom => builder.DoubleNaNFormatter, + NaNHandling.Error => ErrorDoubleNaNFormatter, + NaNHandling.Null => NullDoubleNaNFormatter, + _ => default, + }; + + this.IndentChars = builder.IndentChars; + this.NewLineChars = builder.NewLineChars; + this.IsPrettyPrint = builder.IsPrettyPrint; + var escapeTargetChars = builder.AdditionalEscapeTargetChars.ToArray().Concat(JsonCharactor.MustBeEscaped1Byte.Select(b => new Rune(b))).Concat(builder.EscapesHtmlChars ? JsonCharactor.ShouldBeEscaped.ToArray() : Array.Empty()).Distinct().OrderBy(r => r).ToArray(); + this.EscapeTargetChars = escapeTargetChars; + this.EscapeTargetChars1Byte = escapeTargetChars.Where(r => r.Value <= Byte.MaxValue).Select(r => (byte)r.Value).ToArray(); + this.EscapeTargetChars2Byte = escapeTargetChars.Where(r => r.Value > Byte.MaxValue && r.Value <= UInt16.MaxValue).Select(r => (ushort)r.Value).ToArray(); + this.EscapeTargetChars4Byte = escapeTargetChars.Where(r => r.Value > UInt16.MaxValue).Select(r => r.Value).ToArray(); + this.EscapesHtmlChars = builder.EscapesHtmlChars; + this.EscapesHorizontalTab = builder.EscapesHorizontalTab; + this.EscapesPrivateUseCharactors = builder.EscapesPrivateUseCharactors; + } + + internal Action, JsonEncoderOptions> SingleInfinityFormatter => + this._singleInfinityFormatter ?? MinMaxSingleInfinityFormatter; + + internal Action, JsonEncoderOptions> SingleNaNFormatter => + this._singleNaNFormatter ?? NullSingleNaNFormatter; + + internal Action, JsonEncoderOptions> DoubleInfinityFormatter => + this._doubleInfinityFormatter ?? MinMaxDoubleInfinityFormatter; + + internal Action, JsonEncoderOptions> DoubleNaNFormatter => + this._doubleNaNFormatter ?? NullDoubleNaNFormatter; + } +} diff --git a/src/MsgPack.Json/Json/JsonEncoderOptionsBuilder.cs b/src/MsgPack.Json/Json/JsonEncoderOptionsBuilder.cs new file mode 100644 index 000000000..04634193c --- /dev/null +++ b/src/MsgPack.Json/Json/JsonEncoderOptionsBuilder.cs @@ -0,0 +1,158 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Text; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + /// + /// A builder object to construct immutable object. + /// + public class JsonEncoderOptionsBuilder : FormatEncoderOptionsBuilder + { + private static readonly ReadOnlyMemory DefaultIndentChars = + new byte[] { (byte)' ', (byte)' ' }; + + private static readonly ReadOnlyMemory DefaultNewLineChars = + new byte[] { (byte)'\n' }; + + private InfinityHandling _infinityHandling = InfinityHandling.Default; + + public InfinityHandling InfinityHandling + { + get => this._infinityHandling; + set => this._infinityHandling = JsonOptionsValidation.EnsureKnownNonCustom(value); + } + + private NaNHandling _nanHandling; + + public NaNHandling NaNHandling + { + get => this._nanHandling; + set => this._nanHandling = JsonOptionsValidation.EnsureKnownNonCustom(value); + } + + private ReadOnlyMemory _indentChars = DefaultIndentChars; + public ReadOnlyMemory IndentChars + { + get => this._indentChars; + set + { + for (var i = 0; i < value.Length; i++) + { + switch (value.Span[i]) + { + case (byte)' ': + case (byte)'\t': + case (byte)'\r': + case (byte)'\n': + { + // OK + break; + } + default: + { + throw new ArgumentException( + $"IndentChars can only contains whitespace (U+0020), horizontal tab (U+0009), carriage return (U+000A), and line feed (U+000D) according to RFC8259, but char at {i} is U+00{value.Span[i]:X2}.", + "value" + ); + } + } + } + + this._indentChars = value; + } + } + + private ReadOnlyMemory _newLineChars = DefaultNewLineChars; + public ReadOnlyMemory NewLineChars + { + get => this._newLineChars; + set + { + for (var i = 0; i < value.Length; i++) + { + switch (value.Span[i]) + { + case (byte)'\r': + case (byte)'\n': + { + // OK + break; + } + default: + { + throw new ArgumentException( + $"NewLineChars can only contains carriage return (U+000A), and line feed (U+000D) according to RFC8259, but char at {i} is U+00{value.Span[i]:X2}.", + "value" + ); + } + } + } + + this._newLineChars = value; + } + } + + public bool IsPrettyPrint { get; set; } + + public ReadOnlyMemory AdditionalEscapeTargetChars { get; set; } + + public bool EscapesHorizontalTab { get; set; } = true; + public bool EscapesPrivateUseCharactors { get; set; } = false; + public bool EscapesHtmlChars { get; set; } = false; + + internal Action, JsonEncoderOptions>? SingleInfinityFormatter; + internal Action, JsonEncoderOptions>? DoubleInfinityFormatter; + internal Action, JsonEncoderOptions>? SingleNaNFormatter; + internal Action, JsonEncoderOptions>? DoubleNaNFormatter; + + public JsonEncoderOptionsBuilder() { } + + public JsonEncoderOptionsBuilder SetCustomInfinityFormatter( + Action, JsonEncoderOptions> singleInfinityFormatter, + Action, JsonEncoderOptions> doubleInfinityFormatter + ) + { + this.SingleInfinityFormatter = Ensure.NotNull(singleInfinityFormatter); + this.DoubleInfinityFormatter = Ensure.NotNull(doubleInfinityFormatter); + this._infinityHandling = InfinityHandling.Custom; + return this; + } + + public JsonEncoderOptionsBuilder SetCustomNaNFormatter( + Action, JsonEncoderOptions> singleNaNFormatter, + Action, JsonEncoderOptions> doubleNaNFormatter + ) + { + this.SingleNaNFormatter = Ensure.NotNull(singleNaNFormatter); + this.DoubleNaNFormatter = Ensure.NotNull(doubleNaNFormatter); + this._nanHandling = NaNHandling.Custom; + return this; + } + + public JsonEncoderOptionsBuilder WithHtmlCharactorEscaping() + { + this.EscapesHtmlChars = true; + return this; + } + + public JsonEncoderOptionsBuilder WithoutHorizontalTabEscaping() + { + this.EscapesHorizontalTab = false; + return this; + } + + public JsonEncoderOptionsBuilder WithoutPrivateUseCharsEscaping() + { + this.EscapesPrivateUseCharactors = false; + return this; + } + + public JsonEncoderOptions Build() => new JsonEncoderOptions(this); + } +} diff --git a/src/MsgPack.Json/Json/JsonEscapeSequence.EscapeTables.cs b/src/MsgPack.Json/Json/JsonEscapeSequence.EscapeTables.cs new file mode 100644 index 000000000..bd248b274 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonEscapeSequence.EscapeTables.cs @@ -0,0 +1,2186 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Runtime.InteropServices; + +namespace MsgPack.Json +{ + partial class JsonEscapeSequence + { + + public static ReadOnlySpan BaseEscapeTable => + BitConverter.IsLittleEndian ? + MemoryMarshal.Cast(BaseEscapeTableLittleEndian) : + MemoryMarshal.Cast(BaseEscapeTableBigEndian); + + public static ReadOnlySpan BaseEscapeLengthTable => + BitConverter.IsLittleEndian ? + MemoryMarshal.Cast(BaseEscapeLengthTableLittleEndian) : + MemoryMarshal.Cast(BaseEscapeLengthTableBigEndian); + + public static ReadOnlySpan BaseEscapeTableBigEndian => + new byte[] + { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000000000, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000010000, + 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000020000, + 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000030000, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000040000, + 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000050000, + 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000060000, + 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000070000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x5C, // 0x5C62000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, 0x5C, // 0x5C74000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6E, 0x5C, // 0x5C6E000000000000, + 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000000B0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x5C, // 0x5C66000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x5C, // 0x5C72000000000000, + 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000000E0000, + 0x00, 0x00, 0x0F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000000F0000, + 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000100000, + 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000110000, + 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000120000, + 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000130000, + 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000140000, + 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000150000, + 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000160000, + 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000170000, + 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000180000, + 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000190000, + 0x00, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001A0000, + 0x00, 0x00, 0x1B, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001B0000, + 0x00, 0x00, 0x1C, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001C0000, + 0x00, 0x00, 0x1D, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001D0000, + 0x00, 0x00, 0x1E, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001E0000, + 0x00, 0x00, 0x1F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001F0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, // 0x2000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, // 0x2100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x5C, // 0x5C22000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, // 0x2300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, // 0x2400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, // 0x2500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, // 0x2600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, // 0x2700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, // 0x2800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, // 0x2900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2A, // 0x2A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2B, // 0x2B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2C, // 0x2C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2D, // 0x2D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2E, // 0x2E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2F, // 0x2F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, // 0x3000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, // 0x3100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, // 0x3200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, // 0x3300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, // 0x3400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, // 0x3500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, // 0x3600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, // 0x3700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, // 0x3800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, // 0x3900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3A, // 0x3A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3B, // 0x3B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3C, // 0x3C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3D, // 0x3D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3E, // 0x3E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3F, // 0x3F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, // 0x4000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, // 0x4100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, // 0x4200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, // 0x4300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, // 0x4400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, // 0x4500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, // 0x4600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, // 0x4700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, // 0x4800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, // 0x4900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4A, // 0x4A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4B, // 0x4B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4C, // 0x4C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4D, // 0x4D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4E, // 0x4E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4F, // 0x4F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, // 0x5000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x51, // 0x5100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, // 0x5200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, // 0x5300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, // 0x5400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, // 0x5500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, // 0x5600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, // 0x5700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, // 0x5800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, // 0x5900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5A, // 0x5A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5B, // 0x5B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5C, 0x5C, // 0x5C5C000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5D, // 0x5D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5E, // 0x5E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5F, // 0x5F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, // 0x6000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, // 0x6100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, // 0x6200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, // 0x6300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, // 0x6400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, // 0x6500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, // 0x6600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, // 0x6700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, // 0x6800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, // 0x6900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6A, // 0x6A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6B, // 0x6B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6C, // 0x6C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6D, // 0x6D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6E, // 0x6E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6F, // 0x6F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, // 0x7000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, // 0x7100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, // 0x7200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, // 0x7300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, // 0x7400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, // 0x7500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, // 0x7600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, // 0x7700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, // 0x7800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, // 0x7900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7A, // 0x7A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7B, // 0x7B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, // 0x7C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7D, // 0x7D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7E, // 0x7E00000000000000, + 0x00, 0x00, 0x7F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000007F0000, + }; + + public static ReadOnlySpan BaseEscapeTableLittleEndian => + new byte[] + { + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, // 0x000001000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, // 0x000002000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, // 0x000003000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, // 0x000004000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, // 0x000005000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, // 0x000006000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, // 0x000007000000755C, + 0x5C, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000625C, + 0x5C, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000745C, + 0x5C, 0x6E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000006E5C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, // 0x00000B000000755C, + 0x5C, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000665C, + 0x5C, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000725C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, // 0x00000E000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x0F, 0x00, 0x00, // 0x00000F000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, // 0x000010000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, // 0x000011000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, // 0x000012000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, // 0x000013000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, // 0x000014000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, // 0x000015000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, // 0x000016000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, // 0x000017000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, // 0x000018000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, // 0x000019000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1A, 0x00, 0x00, // 0x00001A000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1B, 0x00, 0x00, // 0x00001B000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1C, 0x00, 0x00, // 0x00001C000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1D, 0x00, 0x00, // 0x00001D000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x00, // 0x00001E000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1F, 0x00, 0x00, // 0x00001F000000755C, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000020, + 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000021, + 0x5C, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000225C, + 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000023, + 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000024, + 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000025, + 0x26, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000026, + 0x27, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000027, + 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000028, + 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000029, + 0x2A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002A, + 0x2B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002B, + 0x2C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002C, + 0x2D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002D, + 0x2E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002E, + 0x2F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002F, + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000030, + 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000031, + 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000032, + 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000033, + 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000034, + 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000035, + 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000036, + 0x37, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000037, + 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000038, + 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000039, + 0x3A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003A, + 0x3B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003B, + 0x3C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003C, + 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003D, + 0x3E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003E, + 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003F, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000040, + 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000041, + 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000042, + 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000043, + 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000044, + 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000045, + 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000046, + 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000047, + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000048, + 0x49, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000049, + 0x4A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004A, + 0x4B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004B, + 0x4C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004C, + 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004D, + 0x4E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004E, + 0x4F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004F, + 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000050, + 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000051, + 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000052, + 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000053, + 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000054, + 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000055, + 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000056, + 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000057, + 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000058, + 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000059, + 0x5A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005A, + 0x5B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005B, + 0x5C, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000005C5C, + 0x5D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005D, + 0x5E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005E, + 0x5F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005F, + 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000060, + 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000061, + 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000062, + 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000063, + 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000064, + 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000065, + 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000066, + 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000067, + 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000068, + 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000069, + 0x6A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006A, + 0x6B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006B, + 0x6C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006C, + 0x6D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006D, + 0x6E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006E, + 0x6F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006F, + 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000070, + 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000071, + 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000072, + 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000073, + 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000074, + 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000075, + 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000076, + 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000077, + 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000078, + 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000079, + 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007A, + 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007B, + 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007C, + 0x7D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007D, + 0x7E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007E, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x00, // 0x00007F000000755C, + }; + + public static ReadOnlySpan BaseEscapeLengthTableBigEndian => + new byte[] + { + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-00), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-01), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-02), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-03), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-04), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-05), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-06), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-07), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-62), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-74), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-6E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-0B), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-66), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-72), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-0E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-0F), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-10), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-11), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-12), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-13), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-14), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-15), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-16), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-17), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-18), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-19), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1A), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1B), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1C), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1D), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1F), + 0x00, 0x00, 0x00, 0x01, // 1 (20), + 0x00, 0x00, 0x00, 0x01, // 1 (21), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-22), + 0x00, 0x00, 0x00, 0x01, // 1 (23), + 0x00, 0x00, 0x00, 0x01, // 1 (24), + 0x00, 0x00, 0x00, 0x01, // 1 (25), + 0x00, 0x00, 0x00, 0x01, // 1 (26), + 0x00, 0x00, 0x00, 0x01, // 1 (27), + 0x00, 0x00, 0x00, 0x01, // 1 (28), + 0x00, 0x00, 0x00, 0x01, // 1 (29), + 0x00, 0x00, 0x00, 0x01, // 1 (2A), + 0x00, 0x00, 0x00, 0x01, // 1 (2B), + 0x00, 0x00, 0x00, 0x01, // 1 (2C), + 0x00, 0x00, 0x00, 0x01, // 1 (2D), + 0x00, 0x00, 0x00, 0x01, // 1 (2E), + 0x00, 0x00, 0x00, 0x01, // 1 (2F), + 0x00, 0x00, 0x00, 0x01, // 1 (30), + 0x00, 0x00, 0x00, 0x01, // 1 (31), + 0x00, 0x00, 0x00, 0x01, // 1 (32), + 0x00, 0x00, 0x00, 0x01, // 1 (33), + 0x00, 0x00, 0x00, 0x01, // 1 (34), + 0x00, 0x00, 0x00, 0x01, // 1 (35), + 0x00, 0x00, 0x00, 0x01, // 1 (36), + 0x00, 0x00, 0x00, 0x01, // 1 (37), + 0x00, 0x00, 0x00, 0x01, // 1 (38), + 0x00, 0x00, 0x00, 0x01, // 1 (39), + 0x00, 0x00, 0x00, 0x01, // 1 (3A), + 0x00, 0x00, 0x00, 0x01, // 1 (3B), + 0x00, 0x00, 0x00, 0x01, // 1 (3C), + 0x00, 0x00, 0x00, 0x01, // 1 (3D), + 0x00, 0x00, 0x00, 0x01, // 1 (3E), + 0x00, 0x00, 0x00, 0x01, // 1 (3F), + 0x00, 0x00, 0x00, 0x01, // 1 (40), + 0x00, 0x00, 0x00, 0x01, // 1 (41), + 0x00, 0x00, 0x00, 0x01, // 1 (42), + 0x00, 0x00, 0x00, 0x01, // 1 (43), + 0x00, 0x00, 0x00, 0x01, // 1 (44), + 0x00, 0x00, 0x00, 0x01, // 1 (45), + 0x00, 0x00, 0x00, 0x01, // 1 (46), + 0x00, 0x00, 0x00, 0x01, // 1 (47), + 0x00, 0x00, 0x00, 0x01, // 1 (48), + 0x00, 0x00, 0x00, 0x01, // 1 (49), + 0x00, 0x00, 0x00, 0x01, // 1 (4A), + 0x00, 0x00, 0x00, 0x01, // 1 (4B), + 0x00, 0x00, 0x00, 0x01, // 1 (4C), + 0x00, 0x00, 0x00, 0x01, // 1 (4D), + 0x00, 0x00, 0x00, 0x01, // 1 (4E), + 0x00, 0x00, 0x00, 0x01, // 1 (4F), + 0x00, 0x00, 0x00, 0x01, // 1 (50), + 0x00, 0x00, 0x00, 0x01, // 1 (51), + 0x00, 0x00, 0x00, 0x01, // 1 (52), + 0x00, 0x00, 0x00, 0x01, // 1 (53), + 0x00, 0x00, 0x00, 0x01, // 1 (54), + 0x00, 0x00, 0x00, 0x01, // 1 (55), + 0x00, 0x00, 0x00, 0x01, // 1 (56), + 0x00, 0x00, 0x00, 0x01, // 1 (57), + 0x00, 0x00, 0x00, 0x01, // 1 (58), + 0x00, 0x00, 0x00, 0x01, // 1 (59), + 0x00, 0x00, 0x00, 0x01, // 1 (5A), + 0x00, 0x00, 0x00, 0x01, // 1 (5B), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-5C), + 0x00, 0x00, 0x00, 0x01, // 1 (5D), + 0x00, 0x00, 0x00, 0x01, // 1 (5E), + 0x00, 0x00, 0x00, 0x01, // 1 (5F), + 0x00, 0x00, 0x00, 0x01, // 1 (60), + 0x00, 0x00, 0x00, 0x01, // 1 (61), + 0x00, 0x00, 0x00, 0x01, // 1 (62), + 0x00, 0x00, 0x00, 0x01, // 1 (63), + 0x00, 0x00, 0x00, 0x01, // 1 (64), + 0x00, 0x00, 0x00, 0x01, // 1 (65), + 0x00, 0x00, 0x00, 0x01, // 1 (66), + 0x00, 0x00, 0x00, 0x01, // 1 (67), + 0x00, 0x00, 0x00, 0x01, // 1 (68), + 0x00, 0x00, 0x00, 0x01, // 1 (69), + 0x00, 0x00, 0x00, 0x01, // 1 (6A), + 0x00, 0x00, 0x00, 0x01, // 1 (6B), + 0x00, 0x00, 0x00, 0x01, // 1 (6C), + 0x00, 0x00, 0x00, 0x01, // 1 (6D), + 0x00, 0x00, 0x00, 0x01, // 1 (6E), + 0x00, 0x00, 0x00, 0x01, // 1 (6F), + 0x00, 0x00, 0x00, 0x01, // 1 (70), + 0x00, 0x00, 0x00, 0x01, // 1 (71), + 0x00, 0x00, 0x00, 0x01, // 1 (72), + 0x00, 0x00, 0x00, 0x01, // 1 (73), + 0x00, 0x00, 0x00, 0x01, // 1 (74), + 0x00, 0x00, 0x00, 0x01, // 1 (75), + 0x00, 0x00, 0x00, 0x01, // 1 (76), + 0x00, 0x00, 0x00, 0x01, // 1 (77), + 0x00, 0x00, 0x00, 0x01, // 1 (78), + 0x00, 0x00, 0x00, 0x01, // 1 (79), + 0x00, 0x00, 0x00, 0x01, // 1 (7A), + 0x00, 0x00, 0x00, 0x01, // 1 (7B), + 0x00, 0x00, 0x00, 0x01, // 1 (7C), + 0x00, 0x00, 0x00, 0x01, // 1 (7D), + 0x00, 0x00, 0x00, 0x01, // 1 (7E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-7F), + }; + + public static ReadOnlySpan BaseEscapeLengthTableLittleEndian => + new byte[] + { + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-00), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-01), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-02), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-03), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-04), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-05), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-06), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-07), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-62), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-74), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-6E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-0B), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-66), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-72), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-0E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-0F), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-10), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-11), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-12), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-13), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-14), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-15), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-16), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-17), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-18), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-19), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1A), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1B), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1C), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1D), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1F), + 0x01, 0x00, 0x00, 0x00, // 1 (20), + 0x01, 0x00, 0x00, 0x00, // 1 (21), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-22), + 0x01, 0x00, 0x00, 0x00, // 1 (23), + 0x01, 0x00, 0x00, 0x00, // 1 (24), + 0x01, 0x00, 0x00, 0x00, // 1 (25), + 0x01, 0x00, 0x00, 0x00, // 1 (26), + 0x01, 0x00, 0x00, 0x00, // 1 (27), + 0x01, 0x00, 0x00, 0x00, // 1 (28), + 0x01, 0x00, 0x00, 0x00, // 1 (29), + 0x01, 0x00, 0x00, 0x00, // 1 (2A), + 0x01, 0x00, 0x00, 0x00, // 1 (2B), + 0x01, 0x00, 0x00, 0x00, // 1 (2C), + 0x01, 0x00, 0x00, 0x00, // 1 (2D), + 0x01, 0x00, 0x00, 0x00, // 1 (2E), + 0x01, 0x00, 0x00, 0x00, // 1 (2F), + 0x01, 0x00, 0x00, 0x00, // 1 (30), + 0x01, 0x00, 0x00, 0x00, // 1 (31), + 0x01, 0x00, 0x00, 0x00, // 1 (32), + 0x01, 0x00, 0x00, 0x00, // 1 (33), + 0x01, 0x00, 0x00, 0x00, // 1 (34), + 0x01, 0x00, 0x00, 0x00, // 1 (35), + 0x01, 0x00, 0x00, 0x00, // 1 (36), + 0x01, 0x00, 0x00, 0x00, // 1 (37), + 0x01, 0x00, 0x00, 0x00, // 1 (38), + 0x01, 0x00, 0x00, 0x00, // 1 (39), + 0x01, 0x00, 0x00, 0x00, // 1 (3A), + 0x01, 0x00, 0x00, 0x00, // 1 (3B), + 0x01, 0x00, 0x00, 0x00, // 1 (3C), + 0x01, 0x00, 0x00, 0x00, // 1 (3D), + 0x01, 0x00, 0x00, 0x00, // 1 (3E), + 0x01, 0x00, 0x00, 0x00, // 1 (3F), + 0x01, 0x00, 0x00, 0x00, // 1 (40), + 0x01, 0x00, 0x00, 0x00, // 1 (41), + 0x01, 0x00, 0x00, 0x00, // 1 (42), + 0x01, 0x00, 0x00, 0x00, // 1 (43), + 0x01, 0x00, 0x00, 0x00, // 1 (44), + 0x01, 0x00, 0x00, 0x00, // 1 (45), + 0x01, 0x00, 0x00, 0x00, // 1 (46), + 0x01, 0x00, 0x00, 0x00, // 1 (47), + 0x01, 0x00, 0x00, 0x00, // 1 (48), + 0x01, 0x00, 0x00, 0x00, // 1 (49), + 0x01, 0x00, 0x00, 0x00, // 1 (4A), + 0x01, 0x00, 0x00, 0x00, // 1 (4B), + 0x01, 0x00, 0x00, 0x00, // 1 (4C), + 0x01, 0x00, 0x00, 0x00, // 1 (4D), + 0x01, 0x00, 0x00, 0x00, // 1 (4E), + 0x01, 0x00, 0x00, 0x00, // 1 (4F), + 0x01, 0x00, 0x00, 0x00, // 1 (50), + 0x01, 0x00, 0x00, 0x00, // 1 (51), + 0x01, 0x00, 0x00, 0x00, // 1 (52), + 0x01, 0x00, 0x00, 0x00, // 1 (53), + 0x01, 0x00, 0x00, 0x00, // 1 (54), + 0x01, 0x00, 0x00, 0x00, // 1 (55), + 0x01, 0x00, 0x00, 0x00, // 1 (56), + 0x01, 0x00, 0x00, 0x00, // 1 (57), + 0x01, 0x00, 0x00, 0x00, // 1 (58), + 0x01, 0x00, 0x00, 0x00, // 1 (59), + 0x01, 0x00, 0x00, 0x00, // 1 (5A), + 0x01, 0x00, 0x00, 0x00, // 1 (5B), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-5C), + 0x01, 0x00, 0x00, 0x00, // 1 (5D), + 0x01, 0x00, 0x00, 0x00, // 1 (5E), + 0x01, 0x00, 0x00, 0x00, // 1 (5F), + 0x01, 0x00, 0x00, 0x00, // 1 (60), + 0x01, 0x00, 0x00, 0x00, // 1 (61), + 0x01, 0x00, 0x00, 0x00, // 1 (62), + 0x01, 0x00, 0x00, 0x00, // 1 (63), + 0x01, 0x00, 0x00, 0x00, // 1 (64), + 0x01, 0x00, 0x00, 0x00, // 1 (65), + 0x01, 0x00, 0x00, 0x00, // 1 (66), + 0x01, 0x00, 0x00, 0x00, // 1 (67), + 0x01, 0x00, 0x00, 0x00, // 1 (68), + 0x01, 0x00, 0x00, 0x00, // 1 (69), + 0x01, 0x00, 0x00, 0x00, // 1 (6A), + 0x01, 0x00, 0x00, 0x00, // 1 (6B), + 0x01, 0x00, 0x00, 0x00, // 1 (6C), + 0x01, 0x00, 0x00, 0x00, // 1 (6D), + 0x01, 0x00, 0x00, 0x00, // 1 (6E), + 0x01, 0x00, 0x00, 0x00, // 1 (6F), + 0x01, 0x00, 0x00, 0x00, // 1 (70), + 0x01, 0x00, 0x00, 0x00, // 1 (71), + 0x01, 0x00, 0x00, 0x00, // 1 (72), + 0x01, 0x00, 0x00, 0x00, // 1 (73), + 0x01, 0x00, 0x00, 0x00, // 1 (74), + 0x01, 0x00, 0x00, 0x00, // 1 (75), + 0x01, 0x00, 0x00, 0x00, // 1 (76), + 0x01, 0x00, 0x00, 0x00, // 1 (77), + 0x01, 0x00, 0x00, 0x00, // 1 (78), + 0x01, 0x00, 0x00, 0x00, // 1 (79), + 0x01, 0x00, 0x00, 0x00, // 1 (7A), + 0x01, 0x00, 0x00, 0x00, // 1 (7B), + 0x01, 0x00, 0x00, 0x00, // 1 (7C), + 0x01, 0x00, 0x00, 0x00, // 1 (7D), + 0x01, 0x00, 0x00, 0x00, // 1 (7E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-7F), + }; + + public static ReadOnlySpan BaseWithoutTabEscapeTable => + BitConverter.IsLittleEndian ? + MemoryMarshal.Cast(BaseWithoutTabEscapeTableLittleEndian) : + MemoryMarshal.Cast(BaseWithoutTabEscapeTableBigEndian); + + public static ReadOnlySpan BaseWithoutTabEscapeLengthTable => + BitConverter.IsLittleEndian ? + MemoryMarshal.Cast(BaseWithoutTabEscapeLengthTableLittleEndian) : + MemoryMarshal.Cast(BaseWithoutTabEscapeLengthTableBigEndian); + + public static ReadOnlySpan BaseWithoutTabEscapeTableBigEndian => + new byte[] + { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000000000, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000010000, + 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000020000, + 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000030000, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000040000, + 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000050000, + 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000060000, + 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000070000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x5C, // 0x5C62000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, // 0x0900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6E, 0x5C, // 0x5C6E000000000000, + 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000000B0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x5C, // 0x5C66000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x5C, // 0x5C72000000000000, + 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000000E0000, + 0x00, 0x00, 0x0F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000000F0000, + 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000100000, + 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000110000, + 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000120000, + 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000130000, + 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000140000, + 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000150000, + 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000160000, + 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000170000, + 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000180000, + 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000190000, + 0x00, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001A0000, + 0x00, 0x00, 0x1B, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001B0000, + 0x00, 0x00, 0x1C, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001C0000, + 0x00, 0x00, 0x1D, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001D0000, + 0x00, 0x00, 0x1E, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001E0000, + 0x00, 0x00, 0x1F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001F0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, // 0x2000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, // 0x2100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x5C, // 0x5C22000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, // 0x2300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, // 0x2400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, // 0x2500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, // 0x2600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, // 0x2700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, // 0x2800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, // 0x2900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2A, // 0x2A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2B, // 0x2B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2C, // 0x2C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2D, // 0x2D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2E, // 0x2E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2F, // 0x2F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, // 0x3000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, // 0x3100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, // 0x3200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, // 0x3300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, // 0x3400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, // 0x3500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, // 0x3600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, // 0x3700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, // 0x3800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, // 0x3900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3A, // 0x3A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3B, // 0x3B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3C, // 0x3C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3D, // 0x3D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3E, // 0x3E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3F, // 0x3F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, // 0x4000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, // 0x4100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, // 0x4200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, // 0x4300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, // 0x4400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, // 0x4500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, // 0x4600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, // 0x4700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, // 0x4800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, // 0x4900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4A, // 0x4A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4B, // 0x4B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4C, // 0x4C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4D, // 0x4D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4E, // 0x4E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4F, // 0x4F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, // 0x5000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x51, // 0x5100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, // 0x5200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, // 0x5300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, // 0x5400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, // 0x5500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, // 0x5600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, // 0x5700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, // 0x5800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, // 0x5900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5A, // 0x5A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5B, // 0x5B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5C, 0x5C, // 0x5C5C000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5D, // 0x5D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5E, // 0x5E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5F, // 0x5F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, // 0x6000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, // 0x6100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, // 0x6200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, // 0x6300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, // 0x6400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, // 0x6500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, // 0x6600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, // 0x6700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, // 0x6800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, // 0x6900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6A, // 0x6A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6B, // 0x6B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6C, // 0x6C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6D, // 0x6D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6E, // 0x6E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6F, // 0x6F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, // 0x7000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, // 0x7100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, // 0x7200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, // 0x7300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, // 0x7400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, // 0x7500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, // 0x7600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, // 0x7700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, // 0x7800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, // 0x7900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7A, // 0x7A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7B, // 0x7B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, // 0x7C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7D, // 0x7D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7E, // 0x7E00000000000000, + 0x00, 0x00, 0x7F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000007F0000, + }; + + public static ReadOnlySpan BaseWithoutTabEscapeTableLittleEndian => + new byte[] + { + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, // 0x000001000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, // 0x000002000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, // 0x000003000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, // 0x000004000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, // 0x000005000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, // 0x000006000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, // 0x000007000000755C, + 0x5C, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000625C, + 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000009, + 0x5C, 0x6E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000006E5C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, // 0x00000B000000755C, + 0x5C, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000665C, + 0x5C, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000725C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, // 0x00000E000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x0F, 0x00, 0x00, // 0x00000F000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, // 0x000010000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, // 0x000011000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, // 0x000012000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, // 0x000013000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, // 0x000014000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, // 0x000015000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, // 0x000016000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, // 0x000017000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, // 0x000018000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, // 0x000019000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1A, 0x00, 0x00, // 0x00001A000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1B, 0x00, 0x00, // 0x00001B000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1C, 0x00, 0x00, // 0x00001C000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1D, 0x00, 0x00, // 0x00001D000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x00, // 0x00001E000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1F, 0x00, 0x00, // 0x00001F000000755C, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000020, + 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000021, + 0x5C, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000225C, + 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000023, + 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000024, + 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000025, + 0x26, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000026, + 0x27, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000027, + 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000028, + 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000029, + 0x2A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002A, + 0x2B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002B, + 0x2C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002C, + 0x2D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002D, + 0x2E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002E, + 0x2F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002F, + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000030, + 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000031, + 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000032, + 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000033, + 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000034, + 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000035, + 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000036, + 0x37, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000037, + 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000038, + 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000039, + 0x3A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003A, + 0x3B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003B, + 0x3C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003C, + 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003D, + 0x3E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003E, + 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003F, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000040, + 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000041, + 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000042, + 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000043, + 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000044, + 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000045, + 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000046, + 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000047, + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000048, + 0x49, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000049, + 0x4A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004A, + 0x4B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004B, + 0x4C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004C, + 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004D, + 0x4E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004E, + 0x4F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004F, + 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000050, + 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000051, + 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000052, + 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000053, + 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000054, + 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000055, + 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000056, + 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000057, + 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000058, + 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000059, + 0x5A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005A, + 0x5B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005B, + 0x5C, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000005C5C, + 0x5D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005D, + 0x5E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005E, + 0x5F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005F, + 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000060, + 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000061, + 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000062, + 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000063, + 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000064, + 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000065, + 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000066, + 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000067, + 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000068, + 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000069, + 0x6A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006A, + 0x6B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006B, + 0x6C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006C, + 0x6D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006D, + 0x6E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006E, + 0x6F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006F, + 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000070, + 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000071, + 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000072, + 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000073, + 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000074, + 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000075, + 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000076, + 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000077, + 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000078, + 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000079, + 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007A, + 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007B, + 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007C, + 0x7D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007D, + 0x7E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007E, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x00, // 0x00007F000000755C, + }; + + public static ReadOnlySpan BaseWithoutTabEscapeLengthTableBigEndian => + new byte[] + { + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-00), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-01), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-02), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-03), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-04), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-05), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-06), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-07), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-62), + 0x00, 0x00, 0x00, 0x01, // 1 (09), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-6E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-0B), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-66), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-72), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-0E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-0F), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-10), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-11), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-12), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-13), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-14), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-15), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-16), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-17), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-18), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-19), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1A), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1B), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1C), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1D), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1F), + 0x00, 0x00, 0x00, 0x01, // 1 (20), + 0x00, 0x00, 0x00, 0x01, // 1 (21), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-22), + 0x00, 0x00, 0x00, 0x01, // 1 (23), + 0x00, 0x00, 0x00, 0x01, // 1 (24), + 0x00, 0x00, 0x00, 0x01, // 1 (25), + 0x00, 0x00, 0x00, 0x01, // 1 (26), + 0x00, 0x00, 0x00, 0x01, // 1 (27), + 0x00, 0x00, 0x00, 0x01, // 1 (28), + 0x00, 0x00, 0x00, 0x01, // 1 (29), + 0x00, 0x00, 0x00, 0x01, // 1 (2A), + 0x00, 0x00, 0x00, 0x01, // 1 (2B), + 0x00, 0x00, 0x00, 0x01, // 1 (2C), + 0x00, 0x00, 0x00, 0x01, // 1 (2D), + 0x00, 0x00, 0x00, 0x01, // 1 (2E), + 0x00, 0x00, 0x00, 0x01, // 1 (2F), + 0x00, 0x00, 0x00, 0x01, // 1 (30), + 0x00, 0x00, 0x00, 0x01, // 1 (31), + 0x00, 0x00, 0x00, 0x01, // 1 (32), + 0x00, 0x00, 0x00, 0x01, // 1 (33), + 0x00, 0x00, 0x00, 0x01, // 1 (34), + 0x00, 0x00, 0x00, 0x01, // 1 (35), + 0x00, 0x00, 0x00, 0x01, // 1 (36), + 0x00, 0x00, 0x00, 0x01, // 1 (37), + 0x00, 0x00, 0x00, 0x01, // 1 (38), + 0x00, 0x00, 0x00, 0x01, // 1 (39), + 0x00, 0x00, 0x00, 0x01, // 1 (3A), + 0x00, 0x00, 0x00, 0x01, // 1 (3B), + 0x00, 0x00, 0x00, 0x01, // 1 (3C), + 0x00, 0x00, 0x00, 0x01, // 1 (3D), + 0x00, 0x00, 0x00, 0x01, // 1 (3E), + 0x00, 0x00, 0x00, 0x01, // 1 (3F), + 0x00, 0x00, 0x00, 0x01, // 1 (40), + 0x00, 0x00, 0x00, 0x01, // 1 (41), + 0x00, 0x00, 0x00, 0x01, // 1 (42), + 0x00, 0x00, 0x00, 0x01, // 1 (43), + 0x00, 0x00, 0x00, 0x01, // 1 (44), + 0x00, 0x00, 0x00, 0x01, // 1 (45), + 0x00, 0x00, 0x00, 0x01, // 1 (46), + 0x00, 0x00, 0x00, 0x01, // 1 (47), + 0x00, 0x00, 0x00, 0x01, // 1 (48), + 0x00, 0x00, 0x00, 0x01, // 1 (49), + 0x00, 0x00, 0x00, 0x01, // 1 (4A), + 0x00, 0x00, 0x00, 0x01, // 1 (4B), + 0x00, 0x00, 0x00, 0x01, // 1 (4C), + 0x00, 0x00, 0x00, 0x01, // 1 (4D), + 0x00, 0x00, 0x00, 0x01, // 1 (4E), + 0x00, 0x00, 0x00, 0x01, // 1 (4F), + 0x00, 0x00, 0x00, 0x01, // 1 (50), + 0x00, 0x00, 0x00, 0x01, // 1 (51), + 0x00, 0x00, 0x00, 0x01, // 1 (52), + 0x00, 0x00, 0x00, 0x01, // 1 (53), + 0x00, 0x00, 0x00, 0x01, // 1 (54), + 0x00, 0x00, 0x00, 0x01, // 1 (55), + 0x00, 0x00, 0x00, 0x01, // 1 (56), + 0x00, 0x00, 0x00, 0x01, // 1 (57), + 0x00, 0x00, 0x00, 0x01, // 1 (58), + 0x00, 0x00, 0x00, 0x01, // 1 (59), + 0x00, 0x00, 0x00, 0x01, // 1 (5A), + 0x00, 0x00, 0x00, 0x01, // 1 (5B), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-5C), + 0x00, 0x00, 0x00, 0x01, // 1 (5D), + 0x00, 0x00, 0x00, 0x01, // 1 (5E), + 0x00, 0x00, 0x00, 0x01, // 1 (5F), + 0x00, 0x00, 0x00, 0x01, // 1 (60), + 0x00, 0x00, 0x00, 0x01, // 1 (61), + 0x00, 0x00, 0x00, 0x01, // 1 (62), + 0x00, 0x00, 0x00, 0x01, // 1 (63), + 0x00, 0x00, 0x00, 0x01, // 1 (64), + 0x00, 0x00, 0x00, 0x01, // 1 (65), + 0x00, 0x00, 0x00, 0x01, // 1 (66), + 0x00, 0x00, 0x00, 0x01, // 1 (67), + 0x00, 0x00, 0x00, 0x01, // 1 (68), + 0x00, 0x00, 0x00, 0x01, // 1 (69), + 0x00, 0x00, 0x00, 0x01, // 1 (6A), + 0x00, 0x00, 0x00, 0x01, // 1 (6B), + 0x00, 0x00, 0x00, 0x01, // 1 (6C), + 0x00, 0x00, 0x00, 0x01, // 1 (6D), + 0x00, 0x00, 0x00, 0x01, // 1 (6E), + 0x00, 0x00, 0x00, 0x01, // 1 (6F), + 0x00, 0x00, 0x00, 0x01, // 1 (70), + 0x00, 0x00, 0x00, 0x01, // 1 (71), + 0x00, 0x00, 0x00, 0x01, // 1 (72), + 0x00, 0x00, 0x00, 0x01, // 1 (73), + 0x00, 0x00, 0x00, 0x01, // 1 (74), + 0x00, 0x00, 0x00, 0x01, // 1 (75), + 0x00, 0x00, 0x00, 0x01, // 1 (76), + 0x00, 0x00, 0x00, 0x01, // 1 (77), + 0x00, 0x00, 0x00, 0x01, // 1 (78), + 0x00, 0x00, 0x00, 0x01, // 1 (79), + 0x00, 0x00, 0x00, 0x01, // 1 (7A), + 0x00, 0x00, 0x00, 0x01, // 1 (7B), + 0x00, 0x00, 0x00, 0x01, // 1 (7C), + 0x00, 0x00, 0x00, 0x01, // 1 (7D), + 0x00, 0x00, 0x00, 0x01, // 1 (7E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-7F), + }; + + public static ReadOnlySpan BaseWithoutTabEscapeLengthTableLittleEndian => + new byte[] + { + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-00), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-01), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-02), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-03), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-04), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-05), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-06), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-07), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-62), + 0x01, 0x00, 0x00, 0x00, // 1 (09), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-6E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-0B), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-66), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-72), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-0E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-0F), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-10), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-11), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-12), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-13), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-14), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-15), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-16), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-17), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-18), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-19), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1A), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1B), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1C), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1D), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1F), + 0x01, 0x00, 0x00, 0x00, // 1 (20), + 0x01, 0x00, 0x00, 0x00, // 1 (21), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-22), + 0x01, 0x00, 0x00, 0x00, // 1 (23), + 0x01, 0x00, 0x00, 0x00, // 1 (24), + 0x01, 0x00, 0x00, 0x00, // 1 (25), + 0x01, 0x00, 0x00, 0x00, // 1 (26), + 0x01, 0x00, 0x00, 0x00, // 1 (27), + 0x01, 0x00, 0x00, 0x00, // 1 (28), + 0x01, 0x00, 0x00, 0x00, // 1 (29), + 0x01, 0x00, 0x00, 0x00, // 1 (2A), + 0x01, 0x00, 0x00, 0x00, // 1 (2B), + 0x01, 0x00, 0x00, 0x00, // 1 (2C), + 0x01, 0x00, 0x00, 0x00, // 1 (2D), + 0x01, 0x00, 0x00, 0x00, // 1 (2E), + 0x01, 0x00, 0x00, 0x00, // 1 (2F), + 0x01, 0x00, 0x00, 0x00, // 1 (30), + 0x01, 0x00, 0x00, 0x00, // 1 (31), + 0x01, 0x00, 0x00, 0x00, // 1 (32), + 0x01, 0x00, 0x00, 0x00, // 1 (33), + 0x01, 0x00, 0x00, 0x00, // 1 (34), + 0x01, 0x00, 0x00, 0x00, // 1 (35), + 0x01, 0x00, 0x00, 0x00, // 1 (36), + 0x01, 0x00, 0x00, 0x00, // 1 (37), + 0x01, 0x00, 0x00, 0x00, // 1 (38), + 0x01, 0x00, 0x00, 0x00, // 1 (39), + 0x01, 0x00, 0x00, 0x00, // 1 (3A), + 0x01, 0x00, 0x00, 0x00, // 1 (3B), + 0x01, 0x00, 0x00, 0x00, // 1 (3C), + 0x01, 0x00, 0x00, 0x00, // 1 (3D), + 0x01, 0x00, 0x00, 0x00, // 1 (3E), + 0x01, 0x00, 0x00, 0x00, // 1 (3F), + 0x01, 0x00, 0x00, 0x00, // 1 (40), + 0x01, 0x00, 0x00, 0x00, // 1 (41), + 0x01, 0x00, 0x00, 0x00, // 1 (42), + 0x01, 0x00, 0x00, 0x00, // 1 (43), + 0x01, 0x00, 0x00, 0x00, // 1 (44), + 0x01, 0x00, 0x00, 0x00, // 1 (45), + 0x01, 0x00, 0x00, 0x00, // 1 (46), + 0x01, 0x00, 0x00, 0x00, // 1 (47), + 0x01, 0x00, 0x00, 0x00, // 1 (48), + 0x01, 0x00, 0x00, 0x00, // 1 (49), + 0x01, 0x00, 0x00, 0x00, // 1 (4A), + 0x01, 0x00, 0x00, 0x00, // 1 (4B), + 0x01, 0x00, 0x00, 0x00, // 1 (4C), + 0x01, 0x00, 0x00, 0x00, // 1 (4D), + 0x01, 0x00, 0x00, 0x00, // 1 (4E), + 0x01, 0x00, 0x00, 0x00, // 1 (4F), + 0x01, 0x00, 0x00, 0x00, // 1 (50), + 0x01, 0x00, 0x00, 0x00, // 1 (51), + 0x01, 0x00, 0x00, 0x00, // 1 (52), + 0x01, 0x00, 0x00, 0x00, // 1 (53), + 0x01, 0x00, 0x00, 0x00, // 1 (54), + 0x01, 0x00, 0x00, 0x00, // 1 (55), + 0x01, 0x00, 0x00, 0x00, // 1 (56), + 0x01, 0x00, 0x00, 0x00, // 1 (57), + 0x01, 0x00, 0x00, 0x00, // 1 (58), + 0x01, 0x00, 0x00, 0x00, // 1 (59), + 0x01, 0x00, 0x00, 0x00, // 1 (5A), + 0x01, 0x00, 0x00, 0x00, // 1 (5B), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-5C), + 0x01, 0x00, 0x00, 0x00, // 1 (5D), + 0x01, 0x00, 0x00, 0x00, // 1 (5E), + 0x01, 0x00, 0x00, 0x00, // 1 (5F), + 0x01, 0x00, 0x00, 0x00, // 1 (60), + 0x01, 0x00, 0x00, 0x00, // 1 (61), + 0x01, 0x00, 0x00, 0x00, // 1 (62), + 0x01, 0x00, 0x00, 0x00, // 1 (63), + 0x01, 0x00, 0x00, 0x00, // 1 (64), + 0x01, 0x00, 0x00, 0x00, // 1 (65), + 0x01, 0x00, 0x00, 0x00, // 1 (66), + 0x01, 0x00, 0x00, 0x00, // 1 (67), + 0x01, 0x00, 0x00, 0x00, // 1 (68), + 0x01, 0x00, 0x00, 0x00, // 1 (69), + 0x01, 0x00, 0x00, 0x00, // 1 (6A), + 0x01, 0x00, 0x00, 0x00, // 1 (6B), + 0x01, 0x00, 0x00, 0x00, // 1 (6C), + 0x01, 0x00, 0x00, 0x00, // 1 (6D), + 0x01, 0x00, 0x00, 0x00, // 1 (6E), + 0x01, 0x00, 0x00, 0x00, // 1 (6F), + 0x01, 0x00, 0x00, 0x00, // 1 (70), + 0x01, 0x00, 0x00, 0x00, // 1 (71), + 0x01, 0x00, 0x00, 0x00, // 1 (72), + 0x01, 0x00, 0x00, 0x00, // 1 (73), + 0x01, 0x00, 0x00, 0x00, // 1 (74), + 0x01, 0x00, 0x00, 0x00, // 1 (75), + 0x01, 0x00, 0x00, 0x00, // 1 (76), + 0x01, 0x00, 0x00, 0x00, // 1 (77), + 0x01, 0x00, 0x00, 0x00, // 1 (78), + 0x01, 0x00, 0x00, 0x00, // 1 (79), + 0x01, 0x00, 0x00, 0x00, // 1 (7A), + 0x01, 0x00, 0x00, 0x00, // 1 (7B), + 0x01, 0x00, 0x00, 0x00, // 1 (7C), + 0x01, 0x00, 0x00, 0x00, // 1 (7D), + 0x01, 0x00, 0x00, 0x00, // 1 (7E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-7F), + }; + + public static ReadOnlySpan BaseWithHtmlCharsEscapeTable => + BitConverter.IsLittleEndian ? + MemoryMarshal.Cast(BaseWithHtmlCharsEscapeTableLittleEndian) : + MemoryMarshal.Cast(BaseWithHtmlCharsEscapeTableBigEndian); + + public static ReadOnlySpan BaseWithHtmlCharsEscapeLengthTable => + BitConverter.IsLittleEndian ? + MemoryMarshal.Cast(BaseWithHtmlCharsEscapeLengthTableLittleEndian) : + MemoryMarshal.Cast(BaseWithHtmlCharsEscapeLengthTableBigEndian); + + public static ReadOnlySpan BaseWithHtmlCharsEscapeTableBigEndian => + new byte[] + { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000000000, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000010000, + 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000020000, + 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000030000, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000040000, + 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000050000, + 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000060000, + 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000070000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x5C, // 0x5C62000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, 0x5C, // 0x5C74000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6E, 0x5C, // 0x5C6E000000000000, + 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000000B0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x5C, // 0x5C66000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x5C, // 0x5C72000000000000, + 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000000E0000, + 0x00, 0x00, 0x0F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000000F0000, + 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000100000, + 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000110000, + 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000120000, + 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000130000, + 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000140000, + 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000150000, + 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000160000, + 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000170000, + 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000180000, + 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000190000, + 0x00, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001A0000, + 0x00, 0x00, 0x1B, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001B0000, + 0x00, 0x00, 0x1C, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001C0000, + 0x00, 0x00, 0x1D, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001D0000, + 0x00, 0x00, 0x1E, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001E0000, + 0x00, 0x00, 0x1F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001F0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, // 0x2000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, // 0x2100000000000000, + 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000220000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, // 0x2300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, // 0x2400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, // 0x2500000000000000, + 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000260000, + 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000270000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, // 0x2800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, // 0x2900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2A, // 0x2A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2B, // 0x2B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2C, // 0x2C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2D, // 0x2D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2E, // 0x2E00000000000000, + 0x00, 0x00, 0x2F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000002F0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, // 0x3000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, // 0x3100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, // 0x3200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, // 0x3300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, // 0x3400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, // 0x3500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, // 0x3600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, // 0x3700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, // 0x3800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, // 0x3900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3A, // 0x3A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3B, // 0x3B00000000000000, + 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000003C0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3D, // 0x3D00000000000000, + 0x00, 0x00, 0x3E, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000003E0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3F, // 0x3F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, // 0x4000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, // 0x4100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, // 0x4200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, // 0x4300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, // 0x4400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, // 0x4500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, // 0x4600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, // 0x4700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, // 0x4800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, // 0x4900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4A, // 0x4A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4B, // 0x4B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4C, // 0x4C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4D, // 0x4D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4E, // 0x4E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4F, // 0x4F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, // 0x5000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x51, // 0x5100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, // 0x5200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, // 0x5300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, // 0x5400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, // 0x5500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, // 0x5600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, // 0x5700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, // 0x5800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, // 0x5900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5A, // 0x5A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5B, // 0x5B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5C, 0x5C, // 0x5C5C000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5D, // 0x5D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5E, // 0x5E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5F, // 0x5F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, // 0x6000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, // 0x6100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, // 0x6200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, // 0x6300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, // 0x6400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, // 0x6500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, // 0x6600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, // 0x6700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, // 0x6800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, // 0x6900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6A, // 0x6A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6B, // 0x6B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6C, // 0x6C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6D, // 0x6D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6E, // 0x6E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6F, // 0x6F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, // 0x7000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, // 0x7100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, // 0x7200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, // 0x7300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, // 0x7400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, // 0x7500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, // 0x7600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, // 0x7700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, // 0x7800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, // 0x7900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7A, // 0x7A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7B, // 0x7B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, // 0x7C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7D, // 0x7D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7E, // 0x7E00000000000000, + 0x00, 0x00, 0x7F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000007F0000, + }; + + public static ReadOnlySpan BaseWithHtmlCharsEscapeTableLittleEndian => + new byte[] + { + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, // 0x000001000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, // 0x000002000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, // 0x000003000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, // 0x000004000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, // 0x000005000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, // 0x000006000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, // 0x000007000000755C, + 0x5C, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000625C, + 0x5C, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000745C, + 0x5C, 0x6E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000006E5C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, // 0x00000B000000755C, + 0x5C, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000665C, + 0x5C, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000725C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, // 0x00000E000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x0F, 0x00, 0x00, // 0x00000F000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, // 0x000010000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, // 0x000011000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, // 0x000012000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, // 0x000013000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, // 0x000014000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, // 0x000015000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, // 0x000016000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, // 0x000017000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, // 0x000018000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, // 0x000019000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1A, 0x00, 0x00, // 0x00001A000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1B, 0x00, 0x00, // 0x00001B000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1C, 0x00, 0x00, // 0x00001C000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1D, 0x00, 0x00, // 0x00001D000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x00, // 0x00001E000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1F, 0x00, 0x00, // 0x00001F000000755C, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000020, + 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000021, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // 0x000022000000755C, + 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000023, + 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000024, + 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000025, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, // 0x000026000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, // 0x000027000000755C, + 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000028, + 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000029, + 0x2A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002A, + 0x2B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002B, + 0x2C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002C, + 0x2D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002D, + 0x2E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002E, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x2F, 0x00, 0x00, // 0x00002F000000755C, + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000030, + 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000031, + 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000032, + 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000033, + 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000034, + 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000035, + 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000036, + 0x37, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000037, + 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000038, + 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000039, + 0x3A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003A, + 0x3B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003B, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, // 0x00003C000000755C, + 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003D, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x00, // 0x00003E000000755C, + 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003F, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000040, + 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000041, + 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000042, + 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000043, + 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000044, + 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000045, + 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000046, + 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000047, + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000048, + 0x49, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000049, + 0x4A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004A, + 0x4B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004B, + 0x4C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004C, + 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004D, + 0x4E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004E, + 0x4F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004F, + 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000050, + 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000051, + 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000052, + 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000053, + 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000054, + 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000055, + 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000056, + 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000057, + 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000058, + 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000059, + 0x5A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005A, + 0x5B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005B, + 0x5C, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000005C5C, + 0x5D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005D, + 0x5E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005E, + 0x5F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005F, + 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000060, + 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000061, + 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000062, + 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000063, + 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000064, + 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000065, + 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000066, + 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000067, + 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000068, + 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000069, + 0x6A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006A, + 0x6B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006B, + 0x6C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006C, + 0x6D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006D, + 0x6E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006E, + 0x6F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006F, + 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000070, + 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000071, + 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000072, + 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000073, + 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000074, + 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000075, + 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000076, + 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000077, + 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000078, + 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000079, + 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007A, + 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007B, + 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007C, + 0x7D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007D, + 0x7E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007E, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x00, // 0x00007F000000755C, + }; + + public static ReadOnlySpan BaseWithHtmlCharsEscapeLengthTableBigEndian => + new byte[] + { + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-00), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-01), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-02), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-03), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-04), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-05), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-06), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-07), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-62), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-74), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-6E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-0B), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-66), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-72), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-0E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-0F), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-10), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-11), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-12), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-13), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-14), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-15), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-16), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-17), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-18), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-19), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1A), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1B), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1C), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1D), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1F), + 0x00, 0x00, 0x00, 0x01, // 1 (20), + 0x00, 0x00, 0x00, 0x01, // 1 (21), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-22), + 0x00, 0x00, 0x00, 0x01, // 1 (23), + 0x00, 0x00, 0x00, 0x01, // 1 (24), + 0x00, 0x00, 0x00, 0x01, // 1 (25), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-26), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-27), + 0x00, 0x00, 0x00, 0x01, // 1 (28), + 0x00, 0x00, 0x00, 0x01, // 1 (29), + 0x00, 0x00, 0x00, 0x01, // 1 (2A), + 0x00, 0x00, 0x00, 0x01, // 1 (2B), + 0x00, 0x00, 0x00, 0x01, // 1 (2C), + 0x00, 0x00, 0x00, 0x01, // 1 (2D), + 0x00, 0x00, 0x00, 0x01, // 1 (2E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-2F), + 0x00, 0x00, 0x00, 0x01, // 1 (30), + 0x00, 0x00, 0x00, 0x01, // 1 (31), + 0x00, 0x00, 0x00, 0x01, // 1 (32), + 0x00, 0x00, 0x00, 0x01, // 1 (33), + 0x00, 0x00, 0x00, 0x01, // 1 (34), + 0x00, 0x00, 0x00, 0x01, // 1 (35), + 0x00, 0x00, 0x00, 0x01, // 1 (36), + 0x00, 0x00, 0x00, 0x01, // 1 (37), + 0x00, 0x00, 0x00, 0x01, // 1 (38), + 0x00, 0x00, 0x00, 0x01, // 1 (39), + 0x00, 0x00, 0x00, 0x01, // 1 (3A), + 0x00, 0x00, 0x00, 0x01, // 1 (3B), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-3C), + 0x00, 0x00, 0x00, 0x01, // 1 (3D), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-3E), + 0x00, 0x00, 0x00, 0x01, // 1 (3F), + 0x00, 0x00, 0x00, 0x01, // 1 (40), + 0x00, 0x00, 0x00, 0x01, // 1 (41), + 0x00, 0x00, 0x00, 0x01, // 1 (42), + 0x00, 0x00, 0x00, 0x01, // 1 (43), + 0x00, 0x00, 0x00, 0x01, // 1 (44), + 0x00, 0x00, 0x00, 0x01, // 1 (45), + 0x00, 0x00, 0x00, 0x01, // 1 (46), + 0x00, 0x00, 0x00, 0x01, // 1 (47), + 0x00, 0x00, 0x00, 0x01, // 1 (48), + 0x00, 0x00, 0x00, 0x01, // 1 (49), + 0x00, 0x00, 0x00, 0x01, // 1 (4A), + 0x00, 0x00, 0x00, 0x01, // 1 (4B), + 0x00, 0x00, 0x00, 0x01, // 1 (4C), + 0x00, 0x00, 0x00, 0x01, // 1 (4D), + 0x00, 0x00, 0x00, 0x01, // 1 (4E), + 0x00, 0x00, 0x00, 0x01, // 1 (4F), + 0x00, 0x00, 0x00, 0x01, // 1 (50), + 0x00, 0x00, 0x00, 0x01, // 1 (51), + 0x00, 0x00, 0x00, 0x01, // 1 (52), + 0x00, 0x00, 0x00, 0x01, // 1 (53), + 0x00, 0x00, 0x00, 0x01, // 1 (54), + 0x00, 0x00, 0x00, 0x01, // 1 (55), + 0x00, 0x00, 0x00, 0x01, // 1 (56), + 0x00, 0x00, 0x00, 0x01, // 1 (57), + 0x00, 0x00, 0x00, 0x01, // 1 (58), + 0x00, 0x00, 0x00, 0x01, // 1 (59), + 0x00, 0x00, 0x00, 0x01, // 1 (5A), + 0x00, 0x00, 0x00, 0x01, // 1 (5B), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-5C), + 0x00, 0x00, 0x00, 0x01, // 1 (5D), + 0x00, 0x00, 0x00, 0x01, // 1 (5E), + 0x00, 0x00, 0x00, 0x01, // 1 (5F), + 0x00, 0x00, 0x00, 0x01, // 1 (60), + 0x00, 0x00, 0x00, 0x01, // 1 (61), + 0x00, 0x00, 0x00, 0x01, // 1 (62), + 0x00, 0x00, 0x00, 0x01, // 1 (63), + 0x00, 0x00, 0x00, 0x01, // 1 (64), + 0x00, 0x00, 0x00, 0x01, // 1 (65), + 0x00, 0x00, 0x00, 0x01, // 1 (66), + 0x00, 0x00, 0x00, 0x01, // 1 (67), + 0x00, 0x00, 0x00, 0x01, // 1 (68), + 0x00, 0x00, 0x00, 0x01, // 1 (69), + 0x00, 0x00, 0x00, 0x01, // 1 (6A), + 0x00, 0x00, 0x00, 0x01, // 1 (6B), + 0x00, 0x00, 0x00, 0x01, // 1 (6C), + 0x00, 0x00, 0x00, 0x01, // 1 (6D), + 0x00, 0x00, 0x00, 0x01, // 1 (6E), + 0x00, 0x00, 0x00, 0x01, // 1 (6F), + 0x00, 0x00, 0x00, 0x01, // 1 (70), + 0x00, 0x00, 0x00, 0x01, // 1 (71), + 0x00, 0x00, 0x00, 0x01, // 1 (72), + 0x00, 0x00, 0x00, 0x01, // 1 (73), + 0x00, 0x00, 0x00, 0x01, // 1 (74), + 0x00, 0x00, 0x00, 0x01, // 1 (75), + 0x00, 0x00, 0x00, 0x01, // 1 (76), + 0x00, 0x00, 0x00, 0x01, // 1 (77), + 0x00, 0x00, 0x00, 0x01, // 1 (78), + 0x00, 0x00, 0x00, 0x01, // 1 (79), + 0x00, 0x00, 0x00, 0x01, // 1 (7A), + 0x00, 0x00, 0x00, 0x01, // 1 (7B), + 0x00, 0x00, 0x00, 0x01, // 1 (7C), + 0x00, 0x00, 0x00, 0x01, // 1 (7D), + 0x00, 0x00, 0x00, 0x01, // 1 (7E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-7F), + }; + + public static ReadOnlySpan BaseWithHtmlCharsEscapeLengthTableLittleEndian => + new byte[] + { + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-00), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-01), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-02), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-03), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-04), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-05), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-06), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-07), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-62), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-74), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-6E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-0B), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-66), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-72), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-0E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-0F), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-10), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-11), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-12), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-13), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-14), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-15), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-16), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-17), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-18), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-19), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1A), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1B), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1C), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1D), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1F), + 0x01, 0x00, 0x00, 0x00, // 1 (20), + 0x01, 0x00, 0x00, 0x00, // 1 (21), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-22), + 0x01, 0x00, 0x00, 0x00, // 1 (23), + 0x01, 0x00, 0x00, 0x00, // 1 (24), + 0x01, 0x00, 0x00, 0x00, // 1 (25), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-26), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-27), + 0x01, 0x00, 0x00, 0x00, // 1 (28), + 0x01, 0x00, 0x00, 0x00, // 1 (29), + 0x01, 0x00, 0x00, 0x00, // 1 (2A), + 0x01, 0x00, 0x00, 0x00, // 1 (2B), + 0x01, 0x00, 0x00, 0x00, // 1 (2C), + 0x01, 0x00, 0x00, 0x00, // 1 (2D), + 0x01, 0x00, 0x00, 0x00, // 1 (2E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-2F), + 0x01, 0x00, 0x00, 0x00, // 1 (30), + 0x01, 0x00, 0x00, 0x00, // 1 (31), + 0x01, 0x00, 0x00, 0x00, // 1 (32), + 0x01, 0x00, 0x00, 0x00, // 1 (33), + 0x01, 0x00, 0x00, 0x00, // 1 (34), + 0x01, 0x00, 0x00, 0x00, // 1 (35), + 0x01, 0x00, 0x00, 0x00, // 1 (36), + 0x01, 0x00, 0x00, 0x00, // 1 (37), + 0x01, 0x00, 0x00, 0x00, // 1 (38), + 0x01, 0x00, 0x00, 0x00, // 1 (39), + 0x01, 0x00, 0x00, 0x00, // 1 (3A), + 0x01, 0x00, 0x00, 0x00, // 1 (3B), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-3C), + 0x01, 0x00, 0x00, 0x00, // 1 (3D), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-3E), + 0x01, 0x00, 0x00, 0x00, // 1 (3F), + 0x01, 0x00, 0x00, 0x00, // 1 (40), + 0x01, 0x00, 0x00, 0x00, // 1 (41), + 0x01, 0x00, 0x00, 0x00, // 1 (42), + 0x01, 0x00, 0x00, 0x00, // 1 (43), + 0x01, 0x00, 0x00, 0x00, // 1 (44), + 0x01, 0x00, 0x00, 0x00, // 1 (45), + 0x01, 0x00, 0x00, 0x00, // 1 (46), + 0x01, 0x00, 0x00, 0x00, // 1 (47), + 0x01, 0x00, 0x00, 0x00, // 1 (48), + 0x01, 0x00, 0x00, 0x00, // 1 (49), + 0x01, 0x00, 0x00, 0x00, // 1 (4A), + 0x01, 0x00, 0x00, 0x00, // 1 (4B), + 0x01, 0x00, 0x00, 0x00, // 1 (4C), + 0x01, 0x00, 0x00, 0x00, // 1 (4D), + 0x01, 0x00, 0x00, 0x00, // 1 (4E), + 0x01, 0x00, 0x00, 0x00, // 1 (4F), + 0x01, 0x00, 0x00, 0x00, // 1 (50), + 0x01, 0x00, 0x00, 0x00, // 1 (51), + 0x01, 0x00, 0x00, 0x00, // 1 (52), + 0x01, 0x00, 0x00, 0x00, // 1 (53), + 0x01, 0x00, 0x00, 0x00, // 1 (54), + 0x01, 0x00, 0x00, 0x00, // 1 (55), + 0x01, 0x00, 0x00, 0x00, // 1 (56), + 0x01, 0x00, 0x00, 0x00, // 1 (57), + 0x01, 0x00, 0x00, 0x00, // 1 (58), + 0x01, 0x00, 0x00, 0x00, // 1 (59), + 0x01, 0x00, 0x00, 0x00, // 1 (5A), + 0x01, 0x00, 0x00, 0x00, // 1 (5B), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-5C), + 0x01, 0x00, 0x00, 0x00, // 1 (5D), + 0x01, 0x00, 0x00, 0x00, // 1 (5E), + 0x01, 0x00, 0x00, 0x00, // 1 (5F), + 0x01, 0x00, 0x00, 0x00, // 1 (60), + 0x01, 0x00, 0x00, 0x00, // 1 (61), + 0x01, 0x00, 0x00, 0x00, // 1 (62), + 0x01, 0x00, 0x00, 0x00, // 1 (63), + 0x01, 0x00, 0x00, 0x00, // 1 (64), + 0x01, 0x00, 0x00, 0x00, // 1 (65), + 0x01, 0x00, 0x00, 0x00, // 1 (66), + 0x01, 0x00, 0x00, 0x00, // 1 (67), + 0x01, 0x00, 0x00, 0x00, // 1 (68), + 0x01, 0x00, 0x00, 0x00, // 1 (69), + 0x01, 0x00, 0x00, 0x00, // 1 (6A), + 0x01, 0x00, 0x00, 0x00, // 1 (6B), + 0x01, 0x00, 0x00, 0x00, // 1 (6C), + 0x01, 0x00, 0x00, 0x00, // 1 (6D), + 0x01, 0x00, 0x00, 0x00, // 1 (6E), + 0x01, 0x00, 0x00, 0x00, // 1 (6F), + 0x01, 0x00, 0x00, 0x00, // 1 (70), + 0x01, 0x00, 0x00, 0x00, // 1 (71), + 0x01, 0x00, 0x00, 0x00, // 1 (72), + 0x01, 0x00, 0x00, 0x00, // 1 (73), + 0x01, 0x00, 0x00, 0x00, // 1 (74), + 0x01, 0x00, 0x00, 0x00, // 1 (75), + 0x01, 0x00, 0x00, 0x00, // 1 (76), + 0x01, 0x00, 0x00, 0x00, // 1 (77), + 0x01, 0x00, 0x00, 0x00, // 1 (78), + 0x01, 0x00, 0x00, 0x00, // 1 (79), + 0x01, 0x00, 0x00, 0x00, // 1 (7A), + 0x01, 0x00, 0x00, 0x00, // 1 (7B), + 0x01, 0x00, 0x00, 0x00, // 1 (7C), + 0x01, 0x00, 0x00, 0x00, // 1 (7D), + 0x01, 0x00, 0x00, 0x00, // 1 (7E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-7F), + }; + + public static ReadOnlySpan BaseWithHtmlCharsWithoutTabEscapeTable => + BitConverter.IsLittleEndian ? + MemoryMarshal.Cast(BaseWithHtmlCharsWithoutTabEscapeTableLittleEndian) : + MemoryMarshal.Cast(BaseWithHtmlCharsWithoutTabEscapeTableBigEndian); + + public static ReadOnlySpan BaseWithHtmlCharsWithoutTabEscapeLengthTable => + BitConverter.IsLittleEndian ? + MemoryMarshal.Cast(BaseWithHtmlCharsWithoutTabEscapeLengthTableLittleEndian) : + MemoryMarshal.Cast(BaseWithHtmlCharsWithoutTabEscapeLengthTableBigEndian); + + public static ReadOnlySpan BaseWithHtmlCharsWithoutTabEscapeTableBigEndian => + new byte[] + { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000000000, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000010000, + 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000020000, + 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000030000, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000040000, + 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000050000, + 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000060000, + 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000070000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x5C, // 0x5C62000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, // 0x0900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6E, 0x5C, // 0x5C6E000000000000, + 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000000B0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x5C, // 0x5C66000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x5C, // 0x5C72000000000000, + 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000000E0000, + 0x00, 0x00, 0x0F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000000F0000, + 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000100000, + 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000110000, + 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000120000, + 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000130000, + 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000140000, + 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000150000, + 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000160000, + 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000170000, + 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000180000, + 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000190000, + 0x00, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001A0000, + 0x00, 0x00, 0x1B, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001B0000, + 0x00, 0x00, 0x1C, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001C0000, + 0x00, 0x00, 0x1D, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001D0000, + 0x00, 0x00, 0x1E, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001E0000, + 0x00, 0x00, 0x1F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000001F0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, // 0x2000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, // 0x2100000000000000, + 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000220000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, // 0x2300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, // 0x2400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, // 0x2500000000000000, + 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000260000, + 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C75000000270000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, // 0x2800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, // 0x2900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2A, // 0x2A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2B, // 0x2B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2C, // 0x2C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2D, // 0x2D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2E, // 0x2E00000000000000, + 0x00, 0x00, 0x2F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000002F0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, // 0x3000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, // 0x3100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, // 0x3200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, // 0x3300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, // 0x3400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x35, // 0x3500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, // 0x3600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, // 0x3700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, // 0x3800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, // 0x3900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3A, // 0x3A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3B, // 0x3B00000000000000, + 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000003C0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3D, // 0x3D00000000000000, + 0x00, 0x00, 0x3E, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000003E0000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3F, // 0x3F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, // 0x4000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, // 0x4100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, // 0x4200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, // 0x4300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, // 0x4400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, // 0x4500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46, // 0x4600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, // 0x4700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, // 0x4800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49, // 0x4900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4A, // 0x4A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4B, // 0x4B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4C, // 0x4C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4D, // 0x4D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4E, // 0x4E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4F, // 0x4F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, // 0x5000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x51, // 0x5100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x52, // 0x5200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x53, // 0x5300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, // 0x5400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, // 0x5500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, // 0x5600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, // 0x5700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, // 0x5800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, // 0x5900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5A, // 0x5A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5B, // 0x5B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5C, 0x5C, // 0x5C5C000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5D, // 0x5D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5E, // 0x5E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5F, // 0x5F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, // 0x6000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, // 0x6100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, // 0x6200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, // 0x6300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, // 0x6400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, // 0x6500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, // 0x6600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, // 0x6700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, // 0x6800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, // 0x6900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6A, // 0x6A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6B, // 0x6B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6C, // 0x6C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6D, // 0x6D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6E, // 0x6E00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6F, // 0x6F00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, // 0x7000000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, // 0x7100000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, // 0x7200000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, // 0x7300000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x74, // 0x7400000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, // 0x7500000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, // 0x7600000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, // 0x7700000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, // 0x7800000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, // 0x7900000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7A, // 0x7A00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7B, // 0x7B00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, // 0x7C00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7D, // 0x7D00000000000000, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7E, // 0x7E00000000000000, + 0x00, 0x00, 0x7F, 0x00, 0x00, 0x00, 0x75, 0x5C, // 0x5C750000007F0000, + }; + + public static ReadOnlySpan BaseWithHtmlCharsWithoutTabEscapeTableLittleEndian => + new byte[] + { + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, // 0x000001000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, // 0x000002000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, // 0x000003000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, // 0x000004000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, // 0x000005000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, // 0x000006000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, // 0x000007000000755C, + 0x5C, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000625C, + 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000009, + 0x5C, 0x6E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000006E5C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, // 0x00000B000000755C, + 0x5C, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000665C, + 0x5C, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000725C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, // 0x00000E000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x0F, 0x00, 0x00, // 0x00000F000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, // 0x000010000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, // 0x000011000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, // 0x000012000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, // 0x000013000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, // 0x000014000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, // 0x000015000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, // 0x000016000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, // 0x000017000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, // 0x000018000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, // 0x000019000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1A, 0x00, 0x00, // 0x00001A000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1B, 0x00, 0x00, // 0x00001B000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1C, 0x00, 0x00, // 0x00001C000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1D, 0x00, 0x00, // 0x00001D000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x00, // 0x00001E000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x1F, 0x00, 0x00, // 0x00001F000000755C, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000020, + 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000021, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // 0x000022000000755C, + 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000023, + 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000024, + 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000025, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, // 0x000026000000755C, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, // 0x000027000000755C, + 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000028, + 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000029, + 0x2A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002A, + 0x2B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002B, + 0x2C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002C, + 0x2D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002D, + 0x2E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000002E, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x2F, 0x00, 0x00, // 0x00002F000000755C, + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000030, + 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000031, + 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000032, + 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000033, + 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000034, + 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000035, + 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000036, + 0x37, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000037, + 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000038, + 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000039, + 0x3A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003A, + 0x3B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003B, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, // 0x00003C000000755C, + 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003D, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x00, // 0x00003E000000755C, + 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000003F, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000040, + 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000041, + 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000042, + 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000043, + 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000044, + 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000045, + 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000046, + 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000047, + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000048, + 0x49, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000049, + 0x4A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004A, + 0x4B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004B, + 0x4C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004C, + 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004D, + 0x4E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004E, + 0x4F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000004F, + 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000050, + 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000051, + 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000052, + 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000053, + 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000054, + 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000055, + 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000056, + 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000057, + 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000058, + 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000059, + 0x5A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005A, + 0x5B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005B, + 0x5C, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000005C5C, + 0x5D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005D, + 0x5E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005E, + 0x5F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000005F, + 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000060, + 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000061, + 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000062, + 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000063, + 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000064, + 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000065, + 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000066, + 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000067, + 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000068, + 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000069, + 0x6A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006A, + 0x6B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006B, + 0x6C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006C, + 0x6D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006D, + 0x6E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006E, + 0x6F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000006F, + 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000070, + 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000071, + 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000072, + 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000073, + 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000074, + 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000075, + 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000076, + 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000077, + 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000078, + 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x0000000000000079, + 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007A, + 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007B, + 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007C, + 0x7D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007D, + 0x7E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x000000000000007E, + 0x5C, 0x75, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x00, // 0x00007F000000755C, + }; + + public static ReadOnlySpan BaseWithHtmlCharsWithoutTabEscapeLengthTableBigEndian => + new byte[] + { + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-00), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-01), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-02), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-03), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-04), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-05), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-06), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-07), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-62), + 0x00, 0x00, 0x00, 0x01, // 1 (09), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-6E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-0B), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-66), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-72), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-0E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-0F), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-10), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-11), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-12), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-13), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-14), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-15), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-16), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-17), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-18), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-19), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1A), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1B), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1C), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1D), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-1F), + 0x00, 0x00, 0x00, 0x01, // 1 (20), + 0x00, 0x00, 0x00, 0x01, // 1 (21), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-22), + 0x00, 0x00, 0x00, 0x01, // 1 (23), + 0x00, 0x00, 0x00, 0x01, // 1 (24), + 0x00, 0x00, 0x00, 0x01, // 1 (25), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-26), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-27), + 0x00, 0x00, 0x00, 0x01, // 1 (28), + 0x00, 0x00, 0x00, 0x01, // 1 (29), + 0x00, 0x00, 0x00, 0x01, // 1 (2A), + 0x00, 0x00, 0x00, 0x01, // 1 (2B), + 0x00, 0x00, 0x00, 0x01, // 1 (2C), + 0x00, 0x00, 0x00, 0x01, // 1 (2D), + 0x00, 0x00, 0x00, 0x01, // 1 (2E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-2F), + 0x00, 0x00, 0x00, 0x01, // 1 (30), + 0x00, 0x00, 0x00, 0x01, // 1 (31), + 0x00, 0x00, 0x00, 0x01, // 1 (32), + 0x00, 0x00, 0x00, 0x01, // 1 (33), + 0x00, 0x00, 0x00, 0x01, // 1 (34), + 0x00, 0x00, 0x00, 0x01, // 1 (35), + 0x00, 0x00, 0x00, 0x01, // 1 (36), + 0x00, 0x00, 0x00, 0x01, // 1 (37), + 0x00, 0x00, 0x00, 0x01, // 1 (38), + 0x00, 0x00, 0x00, 0x01, // 1 (39), + 0x00, 0x00, 0x00, 0x01, // 1 (3A), + 0x00, 0x00, 0x00, 0x01, // 1 (3B), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-3C), + 0x00, 0x00, 0x00, 0x01, // 1 (3D), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-3E), + 0x00, 0x00, 0x00, 0x01, // 1 (3F), + 0x00, 0x00, 0x00, 0x01, // 1 (40), + 0x00, 0x00, 0x00, 0x01, // 1 (41), + 0x00, 0x00, 0x00, 0x01, // 1 (42), + 0x00, 0x00, 0x00, 0x01, // 1 (43), + 0x00, 0x00, 0x00, 0x01, // 1 (44), + 0x00, 0x00, 0x00, 0x01, // 1 (45), + 0x00, 0x00, 0x00, 0x01, // 1 (46), + 0x00, 0x00, 0x00, 0x01, // 1 (47), + 0x00, 0x00, 0x00, 0x01, // 1 (48), + 0x00, 0x00, 0x00, 0x01, // 1 (49), + 0x00, 0x00, 0x00, 0x01, // 1 (4A), + 0x00, 0x00, 0x00, 0x01, // 1 (4B), + 0x00, 0x00, 0x00, 0x01, // 1 (4C), + 0x00, 0x00, 0x00, 0x01, // 1 (4D), + 0x00, 0x00, 0x00, 0x01, // 1 (4E), + 0x00, 0x00, 0x00, 0x01, // 1 (4F), + 0x00, 0x00, 0x00, 0x01, // 1 (50), + 0x00, 0x00, 0x00, 0x01, // 1 (51), + 0x00, 0x00, 0x00, 0x01, // 1 (52), + 0x00, 0x00, 0x00, 0x01, // 1 (53), + 0x00, 0x00, 0x00, 0x01, // 1 (54), + 0x00, 0x00, 0x00, 0x01, // 1 (55), + 0x00, 0x00, 0x00, 0x01, // 1 (56), + 0x00, 0x00, 0x00, 0x01, // 1 (57), + 0x00, 0x00, 0x00, 0x01, // 1 (58), + 0x00, 0x00, 0x00, 0x01, // 1 (59), + 0x00, 0x00, 0x00, 0x01, // 1 (5A), + 0x00, 0x00, 0x00, 0x01, // 1 (5B), + 0x00, 0x00, 0x00, 0x02, // 2 (5C-5C), + 0x00, 0x00, 0x00, 0x01, // 1 (5D), + 0x00, 0x00, 0x00, 0x01, // 1 (5E), + 0x00, 0x00, 0x00, 0x01, // 1 (5F), + 0x00, 0x00, 0x00, 0x01, // 1 (60), + 0x00, 0x00, 0x00, 0x01, // 1 (61), + 0x00, 0x00, 0x00, 0x01, // 1 (62), + 0x00, 0x00, 0x00, 0x01, // 1 (63), + 0x00, 0x00, 0x00, 0x01, // 1 (64), + 0x00, 0x00, 0x00, 0x01, // 1 (65), + 0x00, 0x00, 0x00, 0x01, // 1 (66), + 0x00, 0x00, 0x00, 0x01, // 1 (67), + 0x00, 0x00, 0x00, 0x01, // 1 (68), + 0x00, 0x00, 0x00, 0x01, // 1 (69), + 0x00, 0x00, 0x00, 0x01, // 1 (6A), + 0x00, 0x00, 0x00, 0x01, // 1 (6B), + 0x00, 0x00, 0x00, 0x01, // 1 (6C), + 0x00, 0x00, 0x00, 0x01, // 1 (6D), + 0x00, 0x00, 0x00, 0x01, // 1 (6E), + 0x00, 0x00, 0x00, 0x01, // 1 (6F), + 0x00, 0x00, 0x00, 0x01, // 1 (70), + 0x00, 0x00, 0x00, 0x01, // 1 (71), + 0x00, 0x00, 0x00, 0x01, // 1 (72), + 0x00, 0x00, 0x00, 0x01, // 1 (73), + 0x00, 0x00, 0x00, 0x01, // 1 (74), + 0x00, 0x00, 0x00, 0x01, // 1 (75), + 0x00, 0x00, 0x00, 0x01, // 1 (76), + 0x00, 0x00, 0x00, 0x01, // 1 (77), + 0x00, 0x00, 0x00, 0x01, // 1 (78), + 0x00, 0x00, 0x00, 0x01, // 1 (79), + 0x00, 0x00, 0x00, 0x01, // 1 (7A), + 0x00, 0x00, 0x00, 0x01, // 1 (7B), + 0x00, 0x00, 0x00, 0x01, // 1 (7C), + 0x00, 0x00, 0x00, 0x01, // 1 (7D), + 0x00, 0x00, 0x00, 0x01, // 1 (7E), + 0x00, 0x00, 0x00, 0x06, // 6 (5C-75-00-00-00-7F), + }; + + public static ReadOnlySpan BaseWithHtmlCharsWithoutTabEscapeLengthTableLittleEndian => + new byte[] + { + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-00), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-01), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-02), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-03), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-04), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-05), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-06), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-07), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-62), + 0x01, 0x00, 0x00, 0x00, // 1 (09), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-6E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-0B), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-66), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-72), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-0E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-0F), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-10), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-11), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-12), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-13), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-14), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-15), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-16), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-17), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-18), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-19), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1A), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1B), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1C), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1D), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-1F), + 0x01, 0x00, 0x00, 0x00, // 1 (20), + 0x01, 0x00, 0x00, 0x00, // 1 (21), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-22), + 0x01, 0x00, 0x00, 0x00, // 1 (23), + 0x01, 0x00, 0x00, 0x00, // 1 (24), + 0x01, 0x00, 0x00, 0x00, // 1 (25), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-26), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-27), + 0x01, 0x00, 0x00, 0x00, // 1 (28), + 0x01, 0x00, 0x00, 0x00, // 1 (29), + 0x01, 0x00, 0x00, 0x00, // 1 (2A), + 0x01, 0x00, 0x00, 0x00, // 1 (2B), + 0x01, 0x00, 0x00, 0x00, // 1 (2C), + 0x01, 0x00, 0x00, 0x00, // 1 (2D), + 0x01, 0x00, 0x00, 0x00, // 1 (2E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-2F), + 0x01, 0x00, 0x00, 0x00, // 1 (30), + 0x01, 0x00, 0x00, 0x00, // 1 (31), + 0x01, 0x00, 0x00, 0x00, // 1 (32), + 0x01, 0x00, 0x00, 0x00, // 1 (33), + 0x01, 0x00, 0x00, 0x00, // 1 (34), + 0x01, 0x00, 0x00, 0x00, // 1 (35), + 0x01, 0x00, 0x00, 0x00, // 1 (36), + 0x01, 0x00, 0x00, 0x00, // 1 (37), + 0x01, 0x00, 0x00, 0x00, // 1 (38), + 0x01, 0x00, 0x00, 0x00, // 1 (39), + 0x01, 0x00, 0x00, 0x00, // 1 (3A), + 0x01, 0x00, 0x00, 0x00, // 1 (3B), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-3C), + 0x01, 0x00, 0x00, 0x00, // 1 (3D), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-3E), + 0x01, 0x00, 0x00, 0x00, // 1 (3F), + 0x01, 0x00, 0x00, 0x00, // 1 (40), + 0x01, 0x00, 0x00, 0x00, // 1 (41), + 0x01, 0x00, 0x00, 0x00, // 1 (42), + 0x01, 0x00, 0x00, 0x00, // 1 (43), + 0x01, 0x00, 0x00, 0x00, // 1 (44), + 0x01, 0x00, 0x00, 0x00, // 1 (45), + 0x01, 0x00, 0x00, 0x00, // 1 (46), + 0x01, 0x00, 0x00, 0x00, // 1 (47), + 0x01, 0x00, 0x00, 0x00, // 1 (48), + 0x01, 0x00, 0x00, 0x00, // 1 (49), + 0x01, 0x00, 0x00, 0x00, // 1 (4A), + 0x01, 0x00, 0x00, 0x00, // 1 (4B), + 0x01, 0x00, 0x00, 0x00, // 1 (4C), + 0x01, 0x00, 0x00, 0x00, // 1 (4D), + 0x01, 0x00, 0x00, 0x00, // 1 (4E), + 0x01, 0x00, 0x00, 0x00, // 1 (4F), + 0x01, 0x00, 0x00, 0x00, // 1 (50), + 0x01, 0x00, 0x00, 0x00, // 1 (51), + 0x01, 0x00, 0x00, 0x00, // 1 (52), + 0x01, 0x00, 0x00, 0x00, // 1 (53), + 0x01, 0x00, 0x00, 0x00, // 1 (54), + 0x01, 0x00, 0x00, 0x00, // 1 (55), + 0x01, 0x00, 0x00, 0x00, // 1 (56), + 0x01, 0x00, 0x00, 0x00, // 1 (57), + 0x01, 0x00, 0x00, 0x00, // 1 (58), + 0x01, 0x00, 0x00, 0x00, // 1 (59), + 0x01, 0x00, 0x00, 0x00, // 1 (5A), + 0x01, 0x00, 0x00, 0x00, // 1 (5B), + 0x02, 0x00, 0x00, 0x00, // 2 (5C-5C), + 0x01, 0x00, 0x00, 0x00, // 1 (5D), + 0x01, 0x00, 0x00, 0x00, // 1 (5E), + 0x01, 0x00, 0x00, 0x00, // 1 (5F), + 0x01, 0x00, 0x00, 0x00, // 1 (60), + 0x01, 0x00, 0x00, 0x00, // 1 (61), + 0x01, 0x00, 0x00, 0x00, // 1 (62), + 0x01, 0x00, 0x00, 0x00, // 1 (63), + 0x01, 0x00, 0x00, 0x00, // 1 (64), + 0x01, 0x00, 0x00, 0x00, // 1 (65), + 0x01, 0x00, 0x00, 0x00, // 1 (66), + 0x01, 0x00, 0x00, 0x00, // 1 (67), + 0x01, 0x00, 0x00, 0x00, // 1 (68), + 0x01, 0x00, 0x00, 0x00, // 1 (69), + 0x01, 0x00, 0x00, 0x00, // 1 (6A), + 0x01, 0x00, 0x00, 0x00, // 1 (6B), + 0x01, 0x00, 0x00, 0x00, // 1 (6C), + 0x01, 0x00, 0x00, 0x00, // 1 (6D), + 0x01, 0x00, 0x00, 0x00, // 1 (6E), + 0x01, 0x00, 0x00, 0x00, // 1 (6F), + 0x01, 0x00, 0x00, 0x00, // 1 (70), + 0x01, 0x00, 0x00, 0x00, // 1 (71), + 0x01, 0x00, 0x00, 0x00, // 1 (72), + 0x01, 0x00, 0x00, 0x00, // 1 (73), + 0x01, 0x00, 0x00, 0x00, // 1 (74), + 0x01, 0x00, 0x00, 0x00, // 1 (75), + 0x01, 0x00, 0x00, 0x00, // 1 (76), + 0x01, 0x00, 0x00, 0x00, // 1 (77), + 0x01, 0x00, 0x00, 0x00, // 1 (78), + 0x01, 0x00, 0x00, 0x00, // 1 (79), + 0x01, 0x00, 0x00, 0x00, // 1 (7A), + 0x01, 0x00, 0x00, 0x00, // 1 (7B), + 0x01, 0x00, 0x00, 0x00, // 1 (7C), + 0x01, 0x00, 0x00, 0x00, // 1 (7D), + 0x01, 0x00, 0x00, 0x00, // 1 (7E), + 0x06, 0x00, 0x00, 0x00, // 6 (5C-75-00-00-00-7F), + }; + } +} + diff --git a/src/MsgPack.Json/Json/JsonEscapeSequence.EscapeTables.tt b/src/MsgPack.Json/Json/JsonEscapeSequence.EscapeTables.tt new file mode 100644 index 000000000..5ed0fc745 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonEscapeSequence.EscapeTables.tt @@ -0,0 +1,218 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ output extension=".cs" #> +<#@ assembly name="System.Core" #> +<#@ import namespace="System.Linq" #> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Runtime.InteropServices; + +namespace MsgPack.Json +{ + partial class JsonEscapeSequence + { +<# +foreach (var spec in + new [] + { + new { Label = "Base", ExcludesTab = false, ExcludesHtmlChars = true }, + new { Label = "BaseWithoutTab", ExcludesTab = true, ExcludesHtmlChars = true }, + new { Label = "BaseWithHtmlChars", ExcludesTab = false, ExcludesHtmlChars = false }, + new { Label = "BaseWithHtmlCharsWithoutTab", ExcludesTab = true, ExcludesHtmlChars = false }, + } +) +{ +#> + + public static ReadOnlySpan <#= spec.Label #>EscapeTable => + BitConverter.IsLittleEndian ? + MemoryMarshal.Cast(<#= spec.Label #>EscapeTableLittleEndian) : + MemoryMarshal.Cast(<#= spec.Label #>EscapeTableBigEndian); + + public static ReadOnlySpan <#= spec.Label #>EscapeLengthTable => + BitConverter.IsLittleEndian ? + MemoryMarshal.Cast(<#= spec.Label #>EscapeLengthTableLittleEndian) : + MemoryMarshal.Cast(<#= spec.Label #>EscapeLengthTableBigEndian); + + public static ReadOnlySpan <#= spec.Label #>EscapeTableBigEndian => + new byte[] + { +<# + for (var i = 0; i < 0x80; i++) + { +#> + <#= Generate(i, spec.ExcludesTab, spec.ExcludesHtmlChars, forLittleEndian: false) #>, +<# + } +#> + }; + + public static ReadOnlySpan <#= spec.Label #>EscapeTableLittleEndian => + new byte[] + { +<# + for (var i = 0; i < 0x80; i++) + { +#> + <#= Generate(i, spec.ExcludesTab, spec.ExcludesHtmlChars, forLittleEndian: true) #>, +<# + } +#> + }; + + public static ReadOnlySpan <#= spec.Label #>EscapeLengthTableBigEndian => + new byte[] + { +<# + for (var i = 0; i < 0x80; i++) + { +#> + <#= GenerateLength(i, spec.ExcludesTab, spec.ExcludesHtmlChars, forLittleEndian: false) #>, +<# + } +#> + }; + + public static ReadOnlySpan <#= spec.Label #>EscapeLengthTableLittleEndian => + new byte[] + { +<# + for (var i = 0; i < 0x80; i++) + { +#> + <#= GenerateLength(i, spec.ExcludesTab, spec.ExcludesHtmlChars, forLittleEndian: true) #>, +<# + } +#> + }; +<# +} +#> + } +} + +<#+ +string Generate(int codePoint, bool excludesTab, bool excludesHtmlChars, bool forLittleEndian) +{ + var bytes = GetEscapeSequenceBytes(codePoint, excludesTab, excludesHtmlChars); + if (bytes.Length > 8) + { + throw new Exception($"U+{codePoint:X4} requires {bytes.Length} bytes."); + } + + var span = new byte[8]; + bytes.CopyTo(span, 0); + + if (BitConverter.IsLittleEndian != forLittleEndian) + { + Array.Reverse(span); + } + + var result = BitConverter.ToUInt64(span, 0); + + return $"{String.Join(", ", span.Select(b => $"0x{b:X2}"))}, // 0x{result:X16}"; +} + +string GenerateLength(int codePoint, bool excludesTab, bool excludesHtmlChars, bool forLittleEndian) +{ + var bytes = GetEscapeSequenceBytes(codePoint, excludesTab, excludesHtmlChars); + var span = BitConverter.GetBytes(bytes.Length); + + if (BitConverter.IsLittleEndian != forLittleEndian) + { + Array.Reverse(span); + } + + return $"{String.Join(", ", span.Select(b => $"0x{b:X2}"))}, // {bytes.Length} ({BitConverter.ToString(bytes)})"; +} + +byte[] GetEscapeSequenceBytes(int codePoint, bool excludesTab, bool excludesHtmlChars) +{ + if (codePoint < 0x20) + { + switch (codePoint) + { + case '\b': + { + return new [] { (byte)'\\', (byte)'b' }; + } + case '\f': + { + return new [] { (byte)'\\', (byte)'f' }; + } + case '\r': + { + return new [] { (byte)'\\', (byte)'r' }; + } + case '\n': + { + return new [] { (byte)'\\', (byte)'n' }; + } + case '\t': + { + if (excludesTab) + { + break; + } + + return new [] { (byte)'\\', (byte)'t' }; + } + default: + { + return GetUnicodeEscapeEquence(codePoint); + } + } + } + + if (codePoint == '\\') + { + return new [] { (byte)'\\', (byte)'\\' }; + } + + if (!excludesHtmlChars) + { + switch(codePoint) + { + case '"': + case '/': + case '&': + case '\'': + case '<': + case '>': + { + return GetUnicodeEscapeEquence(codePoint); + } + } + } + + if (codePoint == '"') + { + return new [] { (byte)'\\', (byte)'"' }; + } + + if (codePoint == 0x7F) + { + return GetUnicodeEscapeEquence(codePoint); + } + + return new [] { (byte)codePoint }; +} + +byte[] GetUnicodeEscapeEquence(int codePoint) +{ + var result = new byte[6]; + result[0] = (byte)'\\'; + result[1] = (byte)'u'; + result[2] = (byte)(codePoint >> 24 & 0xFF); + result[3] = (byte)(codePoint >> 16 & 0xFF); + result[4] = (byte)(codePoint >> 8 & 0xFF); + result[5] = (byte)(codePoint & 0xFF); + return result; +} +#> diff --git a/src/MsgPack.Json/Json/JsonEscapeSequence.cs b/src/MsgPack.Json/Json/JsonEscapeSequence.cs new file mode 100644 index 000000000..9a4c8baf6 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonEscapeSequence.cs @@ -0,0 +1,26 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; + +namespace MsgPack.Json +{ + /// + /// Defines known escape sequence int UTF-8 and related structures. + /// + internal static partial class JsonEscapeSequence + { + public static ReadOnlySpan Unicode => new[] { (byte)'\\', (byte)'u' }; + public static ReadOnlySpan ReverseSolidous => new[] { (byte)'\\', (byte)'\\' }; + public static ReadOnlySpan Quatation => new[] { (byte)'\\', (byte)'"' }; + public static ReadOnlySpan Tab => new[] { (byte)'\\', (byte)'t' }; + public static ReadOnlySpan CarriageReturn => new[] { (byte)'\\', (byte)'r' }; + public static ReadOnlySpan LineFeed => new[] { (byte)'\\', (byte)'n' }; + public static ReadOnlySpan BackSpace => new[] { (byte)'\\', (byte)'b' }; + public static ReadOnlySpan FormFeed => new[] { (byte)'\\', (byte)'f' }; + + public static readonly StandardFormat UnicodeFormat = new StandardFormat('X', 4); + } +} diff --git a/src/MsgPack.Json/Json/JsonFormatFeatures.cs b/src/MsgPack.Json/Json/JsonFormatFeatures.cs new file mode 100644 index 000000000..d17149e58 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonFormatFeatures.cs @@ -0,0 +1,23 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using MsgPack.Internal; +using MsgPack.Serialization; + +namespace MsgPack.Json +{ + internal static class JsonFormatFeatures + { + public static FormatFeatures Value { get; } = + new FormatFeaturesBuilder("Json") + { + CanCountCollectionItems = false, + CanSpecifyStringEncoding = false, + IsContextful = false, + PreferredSerializationMethod = SerializationMethod.Map, + AvailableSerializationMethods = AvailableSerializationMethods.Map, + SupportsExtensionTypes = false + }.Build(); + } +} diff --git a/src/MsgPack.Json/Json/JsonFormatter.cs b/src/MsgPack.Json/Json/JsonFormatter.cs new file mode 100644 index 000000000..61e62689f --- /dev/null +++ b/src/MsgPack.Json/Json/JsonFormatter.cs @@ -0,0 +1,48 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System.Buffers; +using System.Buffers.Text; +using System.Runtime.CompilerServices; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + /// + /// Defines common JSON formatting logic. + /// + internal static class JsonFormatter + { + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static void WriteNull(IBufferWriter buffer) + { + var span = buffer.GetSpan(4); + span[0] = (byte)'n'; + span[1] = (byte)'u'; + span[2] = (byte)'l'; + span[3] = (byte)'l'; + buffer.Advance(4); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static void Format(float value, IBufferWriter buffer) + { + var span = buffer.GetSpan(0); + while (!Utf8Formatter.TryFormat(value, span, out _)) + { + span = buffer.GetSpan((span.Length + 1) * 2); + } + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public static void Format(double value, IBufferWriter buffer) + { + var span = buffer.GetSpan(0); + while (!Utf8Formatter.TryFormat(value, span, out _)) + { + span = buffer.GetSpan((span.Length + 1) * 2); + } + } + } +} diff --git a/src/MsgPack.Json/Json/JsonOptionsValidation.cs b/src/MsgPack.Json/Json/JsonOptionsValidation.cs new file mode 100644 index 000000000..a74f29922 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonOptionsValidation.cs @@ -0,0 +1,61 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Runtime.CompilerServices; + +namespace MsgPack.Json +{ + /// + /// Defines common validation logic for option builders. + /// + internal static class JsonOptionsValidation + { + public static InfinityHandling EnsureKnownNonCustom(InfinityHandling value, [CallerArgumentExpression("value")]string paramName = null!) + { + switch (value) + { + case InfinityHandling.Default: + case InfinityHandling.Error: + case InfinityHandling.MinMax: + { + break; + } + case InfinityHandling.Custom: + { + throw new ArgumentException(paramName, $"Value 'Custom' cannot be set via property setter."); + } + default: + { + throw new ArgumentOutOfRangeException(paramName, $"Value '{value:d}' is not known enum value of enum type '{typeof(InfinityHandling)}'"); + } + } + + return value; + } + + public static NaNHandling EnsureKnownNonCustom(NaNHandling value, [CallerArgumentExpression("value")]string paramName = null!) + { + switch (value) + { + case NaNHandling.Default: + case NaNHandling.Error: + case NaNHandling.Null: + { + break; + } + case NaNHandling.Custom: + { + throw new ArgumentException(paramName, $"Value 'Custom' cannot be set via property setter."); + } + default: + { + throw new ArgumentOutOfRangeException(paramName, $"Value '{value:d}' is not known enum value of enum type '{typeof(NaNHandling)}'"); + } + } + + return value; + } + } +} diff --git a/src/MsgPack.Json/Json/JsonParseOptions.cs b/src/MsgPack.Json/Json/JsonParseOptions.cs new file mode 100644 index 000000000..fc7f5f6c1 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonParseOptions.cs @@ -0,0 +1,95 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; + +namespace MsgPack.Json +{ + /// + /// Defines JSON parser options in deserialization. + /// + [Flags] + public enum JsonParseOptions + { + /// + /// No options specified. This should be fastest. + /// + None = 0, + + /// + /// Allows single line comment which starts with # outside of string value. + /// + AllowHashSingleLineComment = 0x1, + + /// + /// Allows single line comment which starts with // outside of string value. + /// + AllowDoubleSolidousSingleLineComment = 0x2, + + /// + /// Allows multi line comment which starts with /* and ends with */ outside of string value. + /// + AllowMultilineComment = 0x4, + + /// + /// Allows Unicode whitespace chars such as 1/4 spacing and full width space, and treats them as ASCII whitespace. + /// + AllowUnicodeWhitespace = 0x10, + + /// + /// Allows all trivial tokens. + /// + AllowAllTrivias = 0xFF, + + /// + /// Allows +NaN and -NaN as valid number. + /// + AllowNaN = 0x100, + + /// + /// Allows +Infinity and -Infinity as valid number. + /// + AllowInfinity = 0x200, + + /// + /// Allows undefined as valid value, and treats it as null. + /// + AllowUndefined = 0x1000, + + /// + /// Allows all irregal values which is not valid in RFC 8259 but is valid as ECMA Script literal. + /// + AllowIrregalValues = 0xFF00, + + /// + /// Allows = as object key-valid separator as well as :. + /// + AllowEqualSignSeparator = 0x10000, + + /// + /// Allows ; as collection item delimiter as well as ,. + /// + AllowSemicolonDelimiter = 0x20000, + + /// + /// Allows extra trailing comma after last collection item. + /// + AllowExtraComma = 0x40000, + + /// + /// Allows ' for string value quotation instead of ". + /// + AllowSingleQuotationString = 0x100000, + + /// + /// Allows all known syntax errors. + /// + AllowWellknownSyntaxErrors = 0xFF0000, + + /// + /// Allows all grammer errors. + /// + AllowAllErrors = unchecked((int)0xFFFFFFFF) + } +} diff --git a/src/MsgPack.Json/Json/JsonStringTokens.cs b/src/MsgPack.Json/Json/JsonStringTokens.cs new file mode 100644 index 000000000..c3a353633 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonStringTokens.cs @@ -0,0 +1,17 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; + +namespace MsgPack.Json +{ + /// + /// Defines JSON string decoding related binary constants which are stored as native BLOB. + /// + internal static class JsonStringTokens + { + public static ReadOnlySpan AnyQuotations => new[] { (byte)'"', (byte)'\'' }; + public static ReadOnlySpan DoubleQuotation => AnyQuotations.Slice(0, 1); + } +} diff --git a/src/MsgPack.Json/Json/JsonThrow.cs b/src/MsgPack.Json/Json/JsonThrow.cs new file mode 100644 index 000000000..96beb6918 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonThrow.cs @@ -0,0 +1,117 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Globalization; +using System.Linq; +using System.Text; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + internal static class JsonThrow + { + public static void TooShortUtf8() + => throw new FormatException($"Input UTF-8 sequence is invalid. The sequence unexpectedly ends."); + + public static CollectionType CollectionHeaderDecodingIsNotSupported(out int itemsCount, out int requestHint) + => throw new NotSupportedException("JSON does not support collection length."); + + public static int CollectionHeaderDecodingIsNotSupported(out int requestHint) + => throw new NotSupportedException("JSON does not support collection length."); + + public static void DrainIsNotSupported(out int requestHint) + => throw new NotSupportedException("JSON does not support collection length."); + + public static void MalformedUtf8(in ReadOnlySpan sequence, long position) + => throw new FormatException($"Input UTF-8 has invalid sequence {BitConverter.ToString(sequence.ToArray())} at position {position:#,0}."); + + public static void MalformedUtf8(in ReadOnlySequence sequence, long position) + => throw new FormatException($"Input UTF-8 has invalid sequence {BitConverter.ToString(sequence.ToArray())} at position {position:#,0}."); + + public static void SurrogateCharInUtf8(long position, int codePoint) + => throw new FormatException($"Input UTF-8 has surrogate charactor U+{codePoint:X4} at position {position:#,0}."); + + public static void IsNotArrayNorObject(in ReadOnlySequence sequence, long position) + => throw new FormatException($"Char {Stringify(sequence)} at position {position:#,0} is not start of array nor object."); + + public static void IsNotArray(long position) + => throw new FormatException($"Char '{{' at position {position:#,0} is not start of array."); + + public static void IsNotObject(long position) + => throw new FormatException($"Char '[' at position {position:#,0} is not start of object."); + + public static void IsNotType(Type type, in ReadOnlySequence unit, long position) + => throw new FormatException($"Char {Stringify(unit)} at position {position:#,0} is not valid for {type} value."); + + public static void TooLongNumber(long numberLength, long maxLength, long position) + => throw new FormatException($"The number at position {position:#,0} has {numberLength:#,0} charactors, but maximum allowed length is {maxLength:#,0}."); + + public static void IsNotStringStart(long position, ReadOnlySpan validQuotations) + { + if (validQuotations.Length == 1) + { + throw new FormatException($"String must starts with '{(char)validQuotations[0]}' (U+00{validQuotations[0]:X2}) at {position:#,0}."); + } + else + { + throw new FormatException($"String must starts with one of [{String.Join(", ", validQuotations.ToArray().Select(b => $"'{(char)b}'(U + 00{b:X2})"))}] at {position:#,0}."); + } + } + + public static void InvalidEscapeSequence(long position, byte escaped) + => throw new FormatException($"Escape sequence '\\{(char)escaped}' at {position:#,0} is not valid."); + + public static void InvalidUnicodeEscapeSequence(long position, Span chars) + => throw new FormatException($"Escape sequence '\\u{Encoding.UTF8.GetString(chars)}' at {position:#,0} is not valid."); + + public static void OrphanSurrogate(long position, int codePoint) + => throw new FormatException($"Surrogate char U+{codePoint:X4} at {position:#,0} is not valid."); + + public static void InvalidBase64(long position) + => throw new FormatException($"String at {position:#,0} is not valid BASE64 sequence."); + + public static void UnexpectedToken(long position, byte token) + => throw new FormatException($"A token {(token >= 0x80 ? $"0x{token:X2}" : (token < 0x7F && token >= 0x20 ? $"'{(char)token}'" : $"U+00{token:X2}"))} is not expected at {position:#,0}."); + + private static string Stringify(in ReadOnlySequence unit) + { + var buffer = new StringBuilder(); + foreach (var rune in Encoding.UTF8.GetString(unit).EnumerateRunes()) + { + var category = Rune.GetUnicodeCategory(rune); + switch (category) + { + case UnicodeCategory.Control: + case UnicodeCategory.Format: + case UnicodeCategory.LineSeparator: + case UnicodeCategory.ModifierLetter: + case UnicodeCategory.ModifierSymbol: + case UnicodeCategory.NonSpacingMark: + case UnicodeCategory.OtherNotAssigned: + case UnicodeCategory.ParagraphSeparator: + case UnicodeCategory.SpacingCombiningMark: + case UnicodeCategory.Surrogate: + { + buffer.Append($"(U+{rune.Value:X4}, {category})"); + break; + } + case UnicodeCategory.SpaceSeparator: + { + buffer.Append($"'{rune}'(U+{rune.Value:X4}, {category})"); + break; + } + default: + { + buffer.Append('\'').Append(rune).Append('\''); + break; + } + } + } + + return buffer.ToString(); + } + } +} diff --git a/src/MsgPack.Json/Json/JsonTriviaTokens.cs b/src/MsgPack.Json/Json/JsonTriviaTokens.cs new file mode 100644 index 000000000..c2bc1b2b4 --- /dev/null +++ b/src/MsgPack.Json/Json/JsonTriviaTokens.cs @@ -0,0 +1,36 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +namespace MsgPack.Json +{ + /// + /// Defines JSON trivials decoding related binary constants which are stored as native BLOB. + /// + internal static class JsonTriviaTokens + { + public static readonly byte[][] MultiByteUnicodeWhitespaces = + new[] + { + new byte[] { 0xC2, 0x85 }, // U+0085 NEXT LINE (NEL) + new byte[] { 0xC2, 0xA0 }, // U+00A0 NO-BREAK SPACE + new byte[] { 0xE1, 0x9A, 0x80 }, // U+1680 OGHAM SPACE MARK + new byte[] { 0xE1, 0x9A, 0x80 }, // U+2000 EN QUAD + new byte[] { 0xE2, 0x80, 0x81 }, // U+2001 EM QUAD + new byte[] { 0xE2, 0x80, 0x82 }, // U+2002 EN SPACE + new byte[] { 0xE2, 0x80, 0x83 }, // U+2003 EM SPACE + new byte[] { 0xE2, 0x80, 0x84 }, // U+2004 THREE-PER-EM SPACE + new byte[] { 0xE2, 0x80, 0x85 }, // U+2005 FOUR-PER-EM SPACE + new byte[] { 0xE2, 0x80, 0x86 }, // U+2006 SIX-PER-EM SPACE + new byte[] { 0xE2, 0x80, 0x87 }, // U+2007 FIGURE SPACE + new byte[] { 0xE2, 0x80, 0x88 }, // U+2008 PUNCTUATION SPACE + new byte[] { 0xE2, 0x80, 0x89 }, // U+2009 THIN SPACE + new byte[] { 0xE2, 0x80, 0x8A }, // U+200A HAIR SPACE + new byte[] { 0xE2, 0x80, 0xA8 }, // U+2028 LINE SEPARATOR + new byte[] { 0xE2, 0x80, 0xA9 }, // U+2029 PARAGRAPH SEPARATOR + new byte[] { 0xE2, 0x80, 0xAF }, // U+202F NARROW NO-BREAK SPACE + new byte[] { 0xE2, 0x81, 0x9F }, // U+205F MEDIUM MATHEMATICAL SPACE + new byte[] { 0xEe, 0x80, 0x80 }, // U+3000 IDEOGRAPHIC SPACE + }; + } +} diff --git a/src/MsgPack.Json/Json/NaNHandling.cs b/src/MsgPack.Json/Json/NaNHandling.cs new file mode 100644 index 000000000..fe7e95b66 --- /dev/null +++ b/src/MsgPack.Json/Json/NaNHandling.cs @@ -0,0 +1,32 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +namespace MsgPack.Json +{ + /// + /// Defines NaN handling in JSON serialization to be compliant with RFC 8259. + /// + public enum NaNHandling + { + /// + /// Use system default setting. See for current default. + /// + Default = 0, + + /// + /// Use null for NaN. + /// + Null = 1, + + /// + /// Throws exception conservatively. + /// + Error = 2, + + /// + /// Use own custom formatting logic. + /// + Custom = 3 + } +} diff --git a/src/MsgPack.Json/Json/SimpleJsonDecoder.cs b/src/MsgPack.Json/Json/SimpleJsonDecoder.cs new file mode 100644 index 000000000..a5373ef1c --- /dev/null +++ b/src/MsgPack.Json/Json/SimpleJsonDecoder.cs @@ -0,0 +1,41 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + /// + /// Simple implementation which is optimized for RFC compliant JSON parsing. + /// + internal sealed class SimpleJsonDecoder : JsonDecoder + { + public SimpleJsonDecoder(JsonDecoderOptions options) + : base(options) { } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + protected sealed override long ReadTrivia(ref SequenceReader source) + => source.AdvancePastAny((byte)' ', (byte)'\t', (byte)'\r', (byte)'\n'); + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public sealed override unsafe bool TryDecodeNull(ref SequenceReader source, out int requestHint) + { + byte* pNull = stackalloc byte[] { (byte)'n', (byte)'u', (byte)'l', (byte)'l' }; + ReadOnlySpan @null = new ReadOnlySpan(pNull, 4); + + if (source.IsNext(@null, advancePast: true)) + { + requestHint = 0; + source.Advance(4); + return true; + } + + requestHint = Math.Max(0, 4 - (int)source.Remaining); + return false; + } + } +} diff --git a/src/MsgPack.Json/Json/StringBuilderBufferWriter.cs b/src/MsgPack.Json/Json/StringBuilderBufferWriter.cs new file mode 100644 index 000000000..3d6330da9 --- /dev/null +++ b/src/MsgPack.Json/Json/StringBuilderBufferWriter.cs @@ -0,0 +1,142 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using MsgPack.Internal; + +namespace MsgPack.Json +{ + /// + /// wrapping . + /// + /// + /// method returns internal char buffer to the pool, so it is important to ensure calling it. + /// + internal sealed class StringBuilderBufferWriter : IBufferWriter, IDisposable + { + private readonly ArrayPool _bufferPool; + private readonly bool _clearsArray; + private readonly StringBuilder _stringBuilder; + private char[] _buffer; + private int _currentOffset; + + private int CurrentBufferSize => this._buffer.Length - this._currentOffset; + + public StringBuilderBufferWriter(StringBuilder stringBuilder, JsonDecoderOptions options) + { + this._stringBuilder = stringBuilder; + this._bufferPool = options.CharBufferPool; + this._clearsArray = options.ClearsBuffer; +#warning TODO: OPTION + this._buffer = this._bufferPool.Rent(64/*options.MaxCharBufferLength*/); + } + + public void Dispose() + { + var array = Interlocked.Exchange(ref this._buffer, null!); + if (array != null) + { + this._bufferPool.Return(array, this._clearsArray); + } + } + + public override string ToString() + { + this.Flush(this._currentOffset); + return this._stringBuilder.ToString(); + } + + private unsafe void Flush(int length) + { + Debug.Assert(length <= this._buffer.Length, $"length ({length}) <= this._buffer.Length ({this._buffer.Length})"); + Debug.Assert(length <= this._currentOffset, $"length ({length}) <= this._currentOffset ({this._currentOffset})"); + + if (length == 0) + { + return; + } + + this._stringBuilder.Append(this._buffer, 0, length); + if (this._currentOffset > length) + { + // Compact + fixed (char* source = this._buffer) + fixed (char* destination = this._buffer) + { + Buffer.MemoryCopy(source + length, destination, this._buffer.Length, this._currentOffset - length); + } + } + + this._currentOffset = 0; + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void Advance(int count) + { + if (this.CurrentBufferSize > count) + { + // Just advance offset. + this._currentOffset += count; + return; + } + + this.AdvanceToNewBuffer(count); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void AdvanceToNewBuffer(int count) + { + // We must prepare new buffer. + var remainingCount = count - this._currentOffset; + this.Flush(this._currentOffset); + if (remainingCount <= 0) + { + return; + } + + // Fill StringBuilder with zero. + this._stringBuilder.Append('\0', remainingCount); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Memory GetMemory(int sizeHint = 0) + { + if (sizeHint <= this.CurrentBufferSize) + { + return this._buffer.AsMemory(this._currentOffset); + } + + return this.GetMemorySlow(sizeHint); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private Memory GetMemorySlow(int sizeHint) + { + this.Flush(this._currentOffset); + Debug.Assert(this._currentOffset == 0, "this._currentOffset == 0"); + if (sizeHint <= this._buffer.Length) + { + return this._buffer.AsMemory(); + } + + // sizeHint is too large. So realloc buffer. + this._bufferPool.Return(this._buffer, this._clearsArray); + this._buffer = this._bufferPool.Rent(sizeHint); + return this._buffer.AsMemory(); + } + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public Span GetSpan(int sizeHint = 0) + => this.GetMemory(sizeHint).Span; + + [MethodImpl(MethodImplOptionsShim.AggressiveInlining)] + public void AppendUtf16CodePoint(int utf16CodePoint) + => this.GetSpan(1)[0] = (char)utf16CodePoint; + } +} diff --git a/src/MsgPack.Json/Json/Unicode.cs b/src/MsgPack.Json/Json/Unicode.cs new file mode 100644 index 000000000..281448db7 --- /dev/null +++ b/src/MsgPack.Json/Json/Unicode.cs @@ -0,0 +1,18 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +namespace MsgPack.Json +{ + /// + /// Defines Unicode related utilities. + /// + internal static class Unicode + { + public static bool IsLowSurrogate(int cp) + => cp >= 0xDC00u && cp <= 0xDFFFu; + + public static bool IsHighSurrogate(int cp) + => cp >= 0xD800u && cp <= 0xDBFFu; + } +} diff --git a/src/MsgPack.Json/MsgPack.Json.csproj b/src/MsgPack.Json/MsgPack.Json.csproj new file mode 100644 index 000000000..d7dfa0faa --- /dev/null +++ b/src/MsgPack.Json/MsgPack.Json.csproj @@ -0,0 +1,67 @@ + + + + netcoreapp3.1 + true + + + + + + + + + + + + + + + True + True + JsonEscapeSequence.EscapeTables.tt + + + + + + TextTemplatingFileGenerator + JsonDecoder.Nullables.cs + + + TextTemplatingFileGenerator + JsonDecoder.Numbers.cs + + + TextTemplatingFileGenerator + JsonEscapeSequence.EscapeTables.cs + + + + + + + + + + True + True + JsonDecoder.Nullables.tt + + + True + True + JsonDecoder.Numbers.tt + + + True + True + JsonEscapeSequence.EscapeTables.tt + + + + + + + + diff --git a/src/MsgPack.Serialization.Extensions/MsgPack.Serialization.Extensions.csproj b/src/MsgPack.Serialization.Extensions/MsgPack.Serialization.Extensions.csproj new file mode 100644 index 000000000..cb6319069 --- /dev/null +++ b/src/MsgPack.Serialization.Extensions/MsgPack.Serialization.Extensions.csproj @@ -0,0 +1,7 @@ + + + + netcoreapp3.1 + + + diff --git a/src/MsgPack.Serialization.ILGeneration/MsgPack.Serialization.ILGeneration.csproj b/src/MsgPack.Serialization.ILGeneration/MsgPack.Serialization.ILGeneration.csproj new file mode 100644 index 000000000..336aa7704 --- /dev/null +++ b/src/MsgPack.Serialization.ILGeneration/MsgPack.Serialization.ILGeneration.csproj @@ -0,0 +1,42 @@ + + + + netcoreapp3.1 + true + + + + + + + + + + + + True + True + ILTranslator.Parse.tt + + + + + + TextTemplatingFileGenerator + ILTranslator.Parse.cs + + + + + + + + + + True + True + ILTranslator.Parse.tt + + + + diff --git a/src/MsgPack.Serialization.ILGeneration/Serialization/Reflection/ILGenerationContext.cs b/src/MsgPack.Serialization.ILGeneration/Serialization/Reflection/ILGenerationContext.cs new file mode 100644 index 000000000..15d6eaf6e --- /dev/null +++ b/src/MsgPack.Serialization.ILGeneration/Serialization/Reflection/ILGenerationContext.cs @@ -0,0 +1,434 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace MsgPack.Serialization.Reflection +{ + internal sealed class ILGenerationContext + { + private struct TypeEntry + { + public TypeBuilder TypeBuilder { get; } + public IReadOnlyDictionary Fields { get; } + public IReadOnlyDictionary Constructors { get; } + public IReadOnlyDictionary Methods { get; } + + public TypeEntry( + TypeBuilder typeBuilder, + Dictionary fields, + Dictionary constructors, + Dictionary methods + ) + { + this.TypeBuilder = typeBuilder; + this.Fields = fields; + this.Constructors = constructors; + this.Methods = methods; + } + } + + private readonly AssemblyBuilder _assemblyBuilder; + private readonly ModuleBuilder _moduleBuilder; + private readonly Dictionary _typeBuilders = new Dictionary(); + private readonly Dictionary _globalMethods = new Dictionary(); + private readonly Dictionary _globalFields = new Dictionary(); + + public ILGenerationContext(string assemblyName, AssemblyBuilderAccess access) + { + this._assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName(assemblyName), access); + this._moduleBuilder = this._assemblyBuilder.DefineDynamicModule($"{assemblyName}.dll"); + } + +#warning TODO: ILGeneration + + public IEnumerable MakeTypes() + { + foreach (var typeBuilder in this._typeBuilders) + { + if (typeBuilder.Value.TypeBuilder.IsCreated()) + { + continue; + } + + var type = typeBuilder.Value.TypeBuilder.CreateType(); + // Type may be null if typeB + if (type != null) + { + yield return type; + } + } + } + + private static CustomAttributeBuilder ToAttributeBuilder(CustomAttributeData attributeData) + { + if (attributeData.NamedArguments.Count == 0) + { + return new CustomAttributeBuilder(attributeData.Constructor, attributeData.ConstructorArguments.Select(a => a.Value).ToArray()); + } + else + { + var fields = new List(); + var fieldValues = new List(); + var properties = new List(); + var propertyValues = new List(); + + foreach (var argument in attributeData.NamedArguments) + { + if (argument.TypedValue.Value is null) + { + continue; + } + + if (argument.IsField) + { + fields.Add((argument.MemberInfo as FieldInfo)!); + fieldValues.Add(argument.TypedValue.Value); + } + else + { + properties.Add((argument.MemberInfo as PropertyInfo)!); + propertyValues.Add(argument.TypedValue.Value); + } + } + + return + new CustomAttributeBuilder( + attributeData.Constructor, + attributeData.ConstructorArguments.Select(a => a.Value).ToArray(), + properties.ToArray(), + propertyValues.ToArray(), + fields.ToArray(), + fieldValues.ToArray() + ); + } + } + + private TypeEntry GetSharedType(Type type) + { + const BindingFlags AllFlags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly; + + if (this._typeBuilders.TryGetValue(type.FullName!, out var typeEntry)) + { + return typeEntry; + } + + var fields = new Dictionary(); + var constructors = new Dictionary(); + var methods = new Dictionary(); + + var typeBuilder = + this._moduleBuilder.DefineType( + type.FullName!, + type.Attributes, + type.BaseType, + type.GetInterfaces() + ); + + foreach (var attribute in type.GetCustomAttributesData()) + { + typeBuilder.SetCustomAttribute(ToAttributeBuilder(attribute)); + } + + foreach (var field in type.GetFields(AllFlags)) + { + if ((field.Attributes & FieldAttributes.HasFieldRVA) == 0) + { + // normal field + var builder = typeBuilder.DefineField(field.Name, field.FieldType, field.GetRequiredCustomModifiers(), field.GetOptionalCustomModifiers(), field.Attributes); + var constant = field.GetRawConstantValue(); + if (constant != null) + { + builder.SetConstant(constant); + } + + foreach (var attribute in field.GetCustomAttributesData()) + { + builder.SetCustomAttribute(ToAttributeBuilder(attribute)); + } + + fields[field.Name] = builder; + } + else + { + // RVA field -- for example, back-end of ReadOnlySpan S => new byte[] { 1, 2, 3 } to avoid newarr. + var value = field.GetValue(null); + if (value == null) + { + throw new NotSupportedException($"RVA field must have value type object."); + } + + // Value is non public struct which has size that is equal to .sdata section bytes. + // So, reinterpret it as byte span and get data as byte array thorugh the span. + + // First, pin boxed value object in GC heap. + var valueHandle = GCHandle.Alloc(value, GCHandleType.Pinned); + try + { + ReadOnlySpan data; + unsafe + { + // Reinterpret pinned object reference as pointer. + void* pValue = valueHandle.AddrOfPinnedObject().ToPointer(); + // Reinterpret boxed object reference as byte span for unmanaged memory. + // Note that value type's size can be gotten from Marshal.SizeOf(object). + data = MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(pValue), Marshal.SizeOf(value)); + } + + var builder = typeBuilder.DefineInitializedData(field.Name, data.ToArray(), field.Attributes); + foreach (var attribute in field.GetCustomAttributesData()) + { + builder.SetCustomAttribute(ToAttributeBuilder(attribute)); + } + + fields[field.Name] = builder; + } + finally + { + valueHandle.Free(); + } + } + } + + foreach (var constructor in type.GetConstructors(AllFlags)) + { + var builder = + typeBuilder.DefineConstructor( + constructor.Attributes, + constructor.CallingConvention, + constructor.GetParameters().Select(p => p.ParameterType).ToArray() + ); + builder.SetImplementationFlags(builder.GetMethodImplementationFlags()); + + foreach (var attribute in builder.GetCustomAttributesData()) + { + builder.SetCustomAttribute(ToAttributeBuilder(attribute)); + } + + constructors[new MethodSignature(constructor)] = (constructor, builder); + } + + foreach (var method in type.GetMethods(AllFlags)) + { + var parameters = method.GetParameters(); + var builder = + typeBuilder.DefineMethod( + method.Name, + method.Attributes, + method.CallingConvention, + method.ReturnParameter.ParameterType, + method.ReturnParameter.GetRequiredCustomModifiers(), + method.ReturnParameter.GetOptionalCustomModifiers(), + parameters.Select(p => p.ParameterType).ToArray(), + parameters.Select(p => p.GetRequiredCustomModifiers()).ToArray(), + parameters.Select(p => p.GetOptionalCustomModifiers()).ToArray() + ); + builder.SetImplementationFlags(builder.GetMethodImplementationFlags()); + + foreach (var attribute in builder.GetCustomAttributesData()) + { + builder.SetCustomAttribute(ToAttributeBuilder(attribute)); + } + + methods[new MethodSignature(method)] = (method, builder); + } + + foreach (var property in type.GetProperties(AllFlags)) + { + var parameters = property.GetIndexParameters(); + var builder = + typeBuilder.DefineProperty( + property.Name, + property.Attributes, + property.PropertyType, + property.GetRequiredCustomModifiers(), + property.GetOptionalCustomModifiers(), + parameters.Select(p => p.ParameterType).ToArray(), + parameters.Select(p => p.GetRequiredCustomModifiers()).ToArray(), + parameters.Select(p => p.GetOptionalCustomModifiers()).ToArray() + ); + + var constant = property.GetRawConstantValue(); + if (constant != null) + { + builder.SetConstant(constant); + } + + foreach (var attribute in property.GetCustomAttributesData()) + { + builder.SetCustomAttribute(ToAttributeBuilder(attribute)); + } + + var getter = property.GetGetMethod(nonPublic: true); + if (getter != null) + { + builder.SetGetMethod(methods[new MethodSignature(getter)].Builder); + } + + var setter = property.GetSetMethod(nonPublic: true); + if (setter != null) + { + builder.SetGetMethod(methods[new MethodSignature(setter)].Builder); + } + } + + + foreach (var @event in type.GetEvents(AllFlags)) + { + var builder = + typeBuilder.DefineEvent( + @event.Name, + @event.Attributes, + @event.EventHandlerType! + ); + + foreach (var attribute in @event.GetCustomAttributesData()) + { + builder.SetCustomAttribute(ToAttributeBuilder(attribute)); + } + + var getter = @event.GetAddMethod(nonPublic: true); + if (getter != null) + { + builder.SetAddOnMethod(methods[new MethodSignature(getter)].Builder); + } + + var setter = @event.GetRemoveMethod(nonPublic: true); + if (setter != null) + { + builder.SetRemoveOnMethod(methods[new MethodSignature(setter)].Builder); + } + + var raise = @event.GetRaiseMethod(nonPublic: true); + if (raise != null) + { + builder.SetRaiseMethod(methods[new MethodSignature(raise)].Builder); + } + + foreach (var other in @event.GetOtherMethods(nonPublic: true)) + { + builder.AddOtherMethod(methods[new MethodSignature(other)].Builder); + } + } + + typeEntry = new TypeEntry(typeBuilder, fields, constructors, methods); + this._typeBuilders[type.FullName!] = typeEntry; + + return typeEntry; + } + + public MethodInfo ResolveMethod(MethodBase declaringMethod, int token) + { + var typeArguments = declaringMethod.DeclaringType?.GetGenericArguments(); + var methodArguments = declaringMethod.GetGenericArguments(); + var result = declaringMethod.Module.ResolveMethod(token, (typeArguments ?? Array.Empty()).Length == 0 ? null : typeArguments, methodArguments.Length == 0 ? null : methodArguments); + if (result == null) + { + // result may be null when the token indicates unresolvable DynamicMethod + throw new NotSupportedException($"Cannot resolve dynamic method token 0x{token:X8} in module '{declaringMethod.Module}'."); + } + + if (!result.IsPublic && !result.IsFamily) + { + throw new InvalidOperationException($"{result.DeclaringType}.{result} is not public."); + } + + if (!(result is MethodInfo method)) + { + throw new InvalidOperationException($"{result.DeclaringType}.{result} is not a method."); + } + + var declaringType = result.DeclaringType; + + if (declaringType == null) + { + throw new NotImplementedException("Global function is not implemented."); + } + + if (declaringType.IsPublic) + { + return method; + } + + return this.GetSharedType(declaringType).Methods[new MethodSignature(method)].Builder; + } + + public ConstructorInfo ResolveConstructor(MethodBase declaringMethod, int token) + { + var typeArguments = declaringMethod.DeclaringType?.GetGenericArguments(); + var methodArguments = declaringMethod.GetGenericArguments(); + var result = declaringMethod.Module.ResolveMethod(token, (typeArguments ?? Array.Empty()).Length == 0 ? null : typeArguments, methodArguments.Length == 0 ? null : methodArguments); + if (!(result is ConstructorInfo constructor)) + { + throw new InvalidOperationException($"Method token 0x{token:X8} in module '{declaringMethod.Module}' may be dynamic method, it is not a constructor."); + } + + if (!result.IsPublic && !result.IsFamily) + { + throw new InvalidOperationException($"{result.DeclaringType}.{result} is not public."); + } + + var declaringType = constructor.DeclaringType!; + + if (declaringType.IsPublic) + { + return constructor; + } + + return this.GetSharedType(declaringType).Constructors[new MethodSignature(constructor)].Builder; + } + + public FieldInfo ResolveField(MethodBase declaringMethod, int token) + { + var typeArguments = declaringMethod.DeclaringType?.GetGenericArguments(); + var methodArguments = declaringMethod.GetGenericArguments(); + var result = declaringMethod.Module.ResolveField(token, (typeArguments ?? Array.Empty()).Length == 0 ? null : typeArguments, methodArguments.Length == 0 ? null : methodArguments); + + // FieldInfo should not be null here. + if (!result!.IsPublic && !result.IsFamily) + { + throw new InvalidOperationException($"{result.DeclaringType}.{result} is not public."); + } + + var declaringType = result.DeclaringType; + + if (declaringType == null) + { + throw new NotImplementedException("Global field is not implemented."); + } + + if (declaringType.IsPublic) + { + return result; + } + + return this.GetSharedType(declaringType).Fields[result.Name]; + } + + public Type ResolveType(MethodBase declaringMethod, int token) + { + var typeArguments = declaringMethod.DeclaringType?.GetGenericArguments(); + var methodArguments = declaringMethod.GetGenericArguments(); + var result = declaringMethod.Module.ResolveType(token, (typeArguments ?? Array.Empty()).Length == 0 ? null : typeArguments, methodArguments.Length == 0 ? null : methodArguments); + + if (result.IsNested) + { + throw new NotImplementedException("Not implemented yet."); + } + + if (result.IsPublic) + { + return result; + } + + return this.GetSharedType(result).TypeBuilder; + } + } +} diff --git a/src/MsgPack.Serialization.ILGeneration/Serialization/Reflection/ILTranslator.Parse.cs b/src/MsgPack.Serialization.ILGeneration/Serialization/Reflection/ILTranslator.Parse.cs new file mode 100644 index 000000000..5ce88fa2f --- /dev/null +++ b/src/MsgPack.Serialization.ILGeneration/Serialization/Reflection/ILTranslator.Parse.cs @@ -0,0 +1,907 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers.Binary; +using System.Reflection.Emit; + +namespace MsgPack.Serialization.Reflection +{ + partial class ILTranspiler + { + private static OpCode Parse(Span il) + { + switch(il[0]) + { + case 0x00: + { + return OpCodes.Nop; // nop InlineNone + } + case 0x01: + { + return OpCodes.Break; // break InlineNone + } + case 0x02: + { + return OpCodes.Ldarg_0; // ldarg.0 InlineNone + } + case 0x03: + { + return OpCodes.Ldarg_1; // ldarg.1 InlineNone + } + case 0x04: + { + return OpCodes.Ldarg_2; // ldarg.2 InlineNone + } + case 0x05: + { + return OpCodes.Ldarg_3; // ldarg.3 InlineNone + } + case 0x06: + { + return OpCodes.Ldloc_0; // ldloc.0 InlineNone + } + case 0x07: + { + return OpCodes.Ldloc_1; // ldloc.1 InlineNone + } + case 0x08: + { + return OpCodes.Ldloc_2; // ldloc.2 InlineNone + } + case 0x09: + { + return OpCodes.Ldloc_3; // ldloc.3 InlineNone + } + case 0x0A: + { + return OpCodes.Stloc_0; // stloc.0 InlineNone + } + case 0x0B: + { + return OpCodes.Stloc_1; // stloc.1 InlineNone + } + case 0x0C: + { + return OpCodes.Stloc_2; // stloc.2 InlineNone + } + case 0x0D: + { + return OpCodes.Stloc_3; // stloc.3 InlineNone + } + case 0x0E: + { + return OpCodes.Ldarg_S; // ldarg.s ShortInlineVar + } + case 0x0F: + { + return OpCodes.Ldarga_S; // ldarga.s ShortInlineVar + } + case 0x10: + { + return OpCodes.Starg_S; // starg.s ShortInlineVar + } + case 0x11: + { + return OpCodes.Ldloc_S; // ldloc.s ShortInlineVar + } + case 0x12: + { + return OpCodes.Ldloca_S; // ldloca.s ShortInlineVar + } + case 0x13: + { + return OpCodes.Stloc_S; // stloc.s ShortInlineVar + } + case 0x14: + { + return OpCodes.Ldnull; // ldnull InlineNone + } + case 0x15: + { + return OpCodes.Ldc_I4_M1; // ldc.i4.m1 InlineNone + } + case 0x16: + { + return OpCodes.Ldc_I4_0; // ldc.i4.0 InlineNone + } + case 0x17: + { + return OpCodes.Ldc_I4_1; // ldc.i4.1 InlineNone + } + case 0x18: + { + return OpCodes.Ldc_I4_2; // ldc.i4.2 InlineNone + } + case 0x19: + { + return OpCodes.Ldc_I4_3; // ldc.i4.3 InlineNone + } + case 0x1A: + { + return OpCodes.Ldc_I4_4; // ldc.i4.4 InlineNone + } + case 0x1B: + { + return OpCodes.Ldc_I4_5; // ldc.i4.5 InlineNone + } + case 0x1C: + { + return OpCodes.Ldc_I4_6; // ldc.i4.6 InlineNone + } + case 0x1D: + { + return OpCodes.Ldc_I4_7; // ldc.i4.7 InlineNone + } + case 0x1E: + { + return OpCodes.Ldc_I4_8; // ldc.i4.8 InlineNone + } + case 0x1F: + { + return OpCodes.Ldc_I4_S; // ldc.i4.s ShortInlineI + } + case 0x20: + { + return OpCodes.Ldc_I4; // ldc.i4 InlineI + } + case 0x21: + { + return OpCodes.Ldc_I8; // ldc.i8 InlineI8 + } + case 0x22: + { + return OpCodes.Ldc_R4; // ldc.r4 ShortInlineR + } + case 0x23: + { + return OpCodes.Ldc_R8; // ldc.r8 InlineR + } + case 0x25: + { + return OpCodes.Dup; // dup InlineNone + } + case 0x26: + { + return OpCodes.Pop; // pop InlineNone + } + case 0x27: + { + return OpCodes.Jmp; // jmp InlineMethod + } + case 0x28: + { + return OpCodes.Call; // call InlineMethod + } + case 0x29: + { + return OpCodes.Calli; // calli InlineSig + } + case 0x2A: + { + return OpCodes.Ret; // ret InlineNone + } + case 0x2B: + { + return OpCodes.Br_S; // br.s ShortInlineBrTarget + } + case 0x2C: + { + return OpCodes.Brfalse_S; // brfalse.s ShortInlineBrTarget + } + case 0x2D: + { + return OpCodes.Brtrue_S; // brtrue.s ShortInlineBrTarget + } + case 0x2E: + { + return OpCodes.Beq_S; // beq.s ShortInlineBrTarget + } + case 0x2F: + { + return OpCodes.Bge_S; // bge.s ShortInlineBrTarget + } + case 0x30: + { + return OpCodes.Bgt_S; // bgt.s ShortInlineBrTarget + } + case 0x31: + { + return OpCodes.Ble_S; // ble.s ShortInlineBrTarget + } + case 0x32: + { + return OpCodes.Blt_S; // blt.s ShortInlineBrTarget + } + case 0x33: + { + return OpCodes.Bne_Un_S; // bne.un.s ShortInlineBrTarget + } + case 0x34: + { + return OpCodes.Bge_Un_S; // bge.un.s ShortInlineBrTarget + } + case 0x35: + { + return OpCodes.Bgt_Un_S; // bgt.un.s ShortInlineBrTarget + } + case 0x36: + { + return OpCodes.Ble_Un_S; // ble.un.s ShortInlineBrTarget + } + case 0x37: + { + return OpCodes.Blt_Un_S; // blt.un.s ShortInlineBrTarget + } + case 0x38: + { + return OpCodes.Br; // br InlineBrTarget + } + case 0x39: + { + return OpCodes.Brfalse; // brfalse InlineBrTarget + } + case 0x3A: + { + return OpCodes.Brtrue; // brtrue InlineBrTarget + } + case 0x3B: + { + return OpCodes.Beq; // beq InlineBrTarget + } + case 0x3C: + { + return OpCodes.Bge; // bge InlineBrTarget + } + case 0x3D: + { + return OpCodes.Bgt; // bgt InlineBrTarget + } + case 0x3E: + { + return OpCodes.Ble; // ble InlineBrTarget + } + case 0x3F: + { + return OpCodes.Blt; // blt InlineBrTarget + } + case 0x40: + { + return OpCodes.Bne_Un; // bne.un InlineBrTarget + } + case 0x41: + { + return OpCodes.Bge_Un; // bge.un InlineBrTarget + } + case 0x42: + { + return OpCodes.Bgt_Un; // bgt.un InlineBrTarget + } + case 0x43: + { + return OpCodes.Ble_Un; // ble.un InlineBrTarget + } + case 0x44: + { + return OpCodes.Blt_Un; // blt.un InlineBrTarget + } + case 0x45: + { + return OpCodes.Switch; // switch InlineSwitch + } + case 0x46: + { + return OpCodes.Ldind_I1; // ldind.i1 InlineNone + } + case 0x47: + { + return OpCodes.Ldind_U1; // ldind.u1 InlineNone + } + case 0x48: + { + return OpCodes.Ldind_I2; // ldind.i2 InlineNone + } + case 0x49: + { + return OpCodes.Ldind_U2; // ldind.u2 InlineNone + } + case 0x4A: + { + return OpCodes.Ldind_I4; // ldind.i4 InlineNone + } + case 0x4B: + { + return OpCodes.Ldind_U4; // ldind.u4 InlineNone + } + case 0x4C: + { + return OpCodes.Ldind_I8; // ldind.i8 InlineNone + } + case 0x4D: + { + return OpCodes.Ldind_I; // ldind.i InlineNone + } + case 0x4E: + { + return OpCodes.Ldind_R4; // ldind.r4 InlineNone + } + case 0x4F: + { + return OpCodes.Ldind_R8; // ldind.r8 InlineNone + } + case 0x50: + { + return OpCodes.Ldind_Ref; // ldind.ref InlineNone + } + case 0x51: + { + return OpCodes.Stind_Ref; // stind.ref InlineNone + } + case 0x52: + { + return OpCodes.Stind_I1; // stind.i1 InlineNone + } + case 0x53: + { + return OpCodes.Stind_I2; // stind.i2 InlineNone + } + case 0x54: + { + return OpCodes.Stind_I4; // stind.i4 InlineNone + } + case 0x55: + { + return OpCodes.Stind_I8; // stind.i8 InlineNone + } + case 0x56: + { + return OpCodes.Stind_R4; // stind.r4 InlineNone + } + case 0x57: + { + return OpCodes.Stind_R8; // stind.r8 InlineNone + } + case 0x58: + { + return OpCodes.Add; // add InlineNone + } + case 0x59: + { + return OpCodes.Sub; // sub InlineNone + } + case 0x5A: + { + return OpCodes.Mul; // mul InlineNone + } + case 0x5B: + { + return OpCodes.Div; // div InlineNone + } + case 0x5C: + { + return OpCodes.Div_Un; // div.un InlineNone + } + case 0x5D: + { + return OpCodes.Rem; // rem InlineNone + } + case 0x5E: + { + return OpCodes.Rem_Un; // rem.un InlineNone + } + case 0x5F: + { + return OpCodes.And; // and InlineNone + } + case 0x60: + { + return OpCodes.Or; // or InlineNone + } + case 0x61: + { + return OpCodes.Xor; // xor InlineNone + } + case 0x62: + { + return OpCodes.Shl; // shl InlineNone + } + case 0x63: + { + return OpCodes.Shr; // shr InlineNone + } + case 0x64: + { + return OpCodes.Shr_Un; // shr.un InlineNone + } + case 0x65: + { + return OpCodes.Neg; // neg InlineNone + } + case 0x66: + { + return OpCodes.Not; // not InlineNone + } + case 0x67: + { + return OpCodes.Conv_I1; // conv.i1 InlineNone + } + case 0x68: + { + return OpCodes.Conv_I2; // conv.i2 InlineNone + } + case 0x69: + { + return OpCodes.Conv_I4; // conv.i4 InlineNone + } + case 0x6A: + { + return OpCodes.Conv_I8; // conv.i8 InlineNone + } + case 0x6B: + { + return OpCodes.Conv_R4; // conv.r4 InlineNone + } + case 0x6C: + { + return OpCodes.Conv_R8; // conv.r8 InlineNone + } + case 0x6D: + { + return OpCodes.Conv_U4; // conv.u4 InlineNone + } + case 0x6E: + { + return OpCodes.Conv_U8; // conv.u8 InlineNone + } + case 0x6F: + { + return OpCodes.Callvirt; // callvirt InlineMethod + } + case 0x70: + { + return OpCodes.Cpobj; // cpobj InlineType + } + case 0x71: + { + return OpCodes.Ldobj; // ldobj InlineType + } + case 0x72: + { + return OpCodes.Ldstr; // ldstr InlineString + } + case 0x73: + { + return OpCodes.Newobj; // newobj InlineMethod + } + case 0x74: + { + return OpCodes.Castclass; // castclass InlineType + } + case 0x75: + { + return OpCodes.Isinst; // isinst InlineType + } + case 0x76: + { + return OpCodes.Conv_R_Un; // conv.r.un InlineNone + } + case 0x79: + { + return OpCodes.Unbox; // unbox InlineType + } + case 0x7A: + { + return OpCodes.Throw; // throw InlineNone + } + case 0x7B: + { + return OpCodes.Ldfld; // ldfld InlineField + } + case 0x7C: + { + return OpCodes.Ldflda; // ldflda InlineField + } + case 0x7D: + { + return OpCodes.Stfld; // stfld InlineField + } + case 0x7E: + { + return OpCodes.Ldsfld; // ldsfld InlineField + } + case 0x7F: + { + return OpCodes.Ldsflda; // ldsflda InlineField + } + case 0x80: + { + return OpCodes.Stsfld; // stsfld InlineField + } + case 0x81: + { + return OpCodes.Stobj; // stobj InlineType + } + case 0x82: + { + return OpCodes.Conv_Ovf_I1_Un; // conv.ovf.i1.un InlineNone + } + case 0x83: + { + return OpCodes.Conv_Ovf_I2_Un; // conv.ovf.i2.un InlineNone + } + case 0x84: + { + return OpCodes.Conv_Ovf_I4_Un; // conv.ovf.i4.un InlineNone + } + case 0x85: + { + return OpCodes.Conv_Ovf_I8_Un; // conv.ovf.i8.un InlineNone + } + case 0x86: + { + return OpCodes.Conv_Ovf_U1_Un; // conv.ovf.u1.un InlineNone + } + case 0x87: + { + return OpCodes.Conv_Ovf_U2_Un; // conv.ovf.u2.un InlineNone + } + case 0x88: + { + return OpCodes.Conv_Ovf_U4_Un; // conv.ovf.u4.un InlineNone + } + case 0x89: + { + return OpCodes.Conv_Ovf_U8_Un; // conv.ovf.u8.un InlineNone + } + case 0x8A: + { + return OpCodes.Conv_Ovf_I_Un; // conv.ovf.i.un InlineNone + } + case 0x8B: + { + return OpCodes.Conv_Ovf_U_Un; // conv.ovf.u.un InlineNone + } + case 0x8C: + { + return OpCodes.Box; // box InlineType + } + case 0x8D: + { + return OpCodes.Newarr; // newarr InlineType + } + case 0x8E: + { + return OpCodes.Ldlen; // ldlen InlineNone + } + case 0x8F: + { + return OpCodes.Ldelema; // ldelema InlineType + } + case 0x90: + { + return OpCodes.Ldelem_I1; // ldelem.i1 InlineNone + } + case 0x91: + { + return OpCodes.Ldelem_U1; // ldelem.u1 InlineNone + } + case 0x92: + { + return OpCodes.Ldelem_I2; // ldelem.i2 InlineNone + } + case 0x93: + { + return OpCodes.Ldelem_U2; // ldelem.u2 InlineNone + } + case 0x94: + { + return OpCodes.Ldelem_I4; // ldelem.i4 InlineNone + } + case 0x95: + { + return OpCodes.Ldelem_U4; // ldelem.u4 InlineNone + } + case 0x96: + { + return OpCodes.Ldelem_I8; // ldelem.i8 InlineNone + } + case 0x97: + { + return OpCodes.Ldelem_I; // ldelem.i InlineNone + } + case 0x98: + { + return OpCodes.Ldelem_R4; // ldelem.r4 InlineNone + } + case 0x99: + { + return OpCodes.Ldelem_R8; // ldelem.r8 InlineNone + } + case 0x9A: + { + return OpCodes.Ldelem_Ref; // ldelem.ref InlineNone + } + case 0x9B: + { + return OpCodes.Stelem_I; // stelem.i InlineNone + } + case 0x9C: + { + return OpCodes.Stelem_I1; // stelem.i1 InlineNone + } + case 0x9D: + { + return OpCodes.Stelem_I2; // stelem.i2 InlineNone + } + case 0x9E: + { + return OpCodes.Stelem_I4; // stelem.i4 InlineNone + } + case 0x9F: + { + return OpCodes.Stelem_I8; // stelem.i8 InlineNone + } + case 0xA0: + { + return OpCodes.Stelem_R4; // stelem.r4 InlineNone + } + case 0xA1: + { + return OpCodes.Stelem_R8; // stelem.r8 InlineNone + } + case 0xA2: + { + return OpCodes.Stelem_Ref; // stelem.ref InlineNone + } + case 0xA3: + { + return OpCodes.Ldelem; // ldelem InlineType + } + case 0xA4: + { + return OpCodes.Stelem; // stelem InlineType + } + case 0xA5: + { + return OpCodes.Unbox_Any; // unbox.any InlineType + } + case 0xB3: + { + return OpCodes.Conv_Ovf_I1; // conv.ovf.i1 InlineNone + } + case 0xB4: + { + return OpCodes.Conv_Ovf_U1; // conv.ovf.u1 InlineNone + } + case 0xB5: + { + return OpCodes.Conv_Ovf_I2; // conv.ovf.i2 InlineNone + } + case 0xB6: + { + return OpCodes.Conv_Ovf_U2; // conv.ovf.u2 InlineNone + } + case 0xB7: + { + return OpCodes.Conv_Ovf_I4; // conv.ovf.i4 InlineNone + } + case 0xB8: + { + return OpCodes.Conv_Ovf_U4; // conv.ovf.u4 InlineNone + } + case 0xB9: + { + return OpCodes.Conv_Ovf_I8; // conv.ovf.i8 InlineNone + } + case 0xBA: + { + return OpCodes.Conv_Ovf_U8; // conv.ovf.u8 InlineNone + } + case 0xC2: + { + return OpCodes.Refanyval; // refanyval InlineType + } + case 0xC3: + { + return OpCodes.Ckfinite; // ckfinite InlineNone + } + case 0xC6: + { + return OpCodes.Mkrefany; // mkrefany InlineType + } + case 0xD0: + { + return OpCodes.Ldtoken; // ldtoken InlineTok + } + case 0xD1: + { + return OpCodes.Conv_U2; // conv.u2 InlineNone + } + case 0xD2: + { + return OpCodes.Conv_U1; // conv.u1 InlineNone + } + case 0xD3: + { + return OpCodes.Conv_I; // conv.i InlineNone + } + case 0xD4: + { + return OpCodes.Conv_Ovf_I; // conv.ovf.i InlineNone + } + case 0xD5: + { + return OpCodes.Conv_Ovf_U; // conv.ovf.u InlineNone + } + case 0xD6: + { + return OpCodes.Add_Ovf; // add.ovf InlineNone + } + case 0xD7: + { + return OpCodes.Add_Ovf_Un; // add.ovf.un InlineNone + } + case 0xD8: + { + return OpCodes.Mul_Ovf; // mul.ovf InlineNone + } + case 0xD9: + { + return OpCodes.Mul_Ovf_Un; // mul.ovf.un InlineNone + } + case 0xDA: + { + return OpCodes.Sub_Ovf; // sub.ovf InlineNone + } + case 0xDB: + { + return OpCodes.Sub_Ovf_Un; // sub.ovf.un InlineNone + } + case 0xDC: + { + return OpCodes.Endfinally; // endfinally InlineNone + } + case 0xDD: + { + return OpCodes.Leave; // leave InlineBrTarget + } + case 0xDE: + { + return OpCodes.Leave_S; // leave.s ShortInlineBrTarget + } + case 0xDF: + { + return OpCodes.Stind_I; // stind.i InlineNone + } + case 0xE0: + { + return OpCodes.Conv_U; // conv.u InlineNone + } + default: + { + var opCode = BinaryPrimitives.ReadUInt16BigEndian(il); + switch (opCode) + { + case 0xFE00: + { + return OpCodes.Arglist; // arglist InlineNone + } + case 0xFE01: + { + return OpCodes.Ceq; // ceq InlineNone + } + case 0xFE02: + { + return OpCodes.Cgt; // cgt InlineNone + } + case 0xFE03: + { + return OpCodes.Cgt_Un; // cgt.un InlineNone + } + case 0xFE04: + { + return OpCodes.Clt; // clt InlineNone + } + case 0xFE05: + { + return OpCodes.Clt_Un; // clt.un InlineNone + } + case 0xFE06: + { + return OpCodes.Ldftn; // ldftn InlineMethod + } + case 0xFE07: + { + return OpCodes.Ldvirtftn; // ldvirtftn InlineMethod + } + case 0xFE09: + { + return OpCodes.Ldarg; // ldarg InlineVar + } + case 0xFE0A: + { + return OpCodes.Ldarga; // ldarga InlineVar + } + case 0xFE0B: + { + return OpCodes.Starg; // starg InlineVar + } + case 0xFE0C: + { + return OpCodes.Ldloc; // ldloc InlineVar + } + case 0xFE0D: + { + return OpCodes.Ldloca; // ldloca InlineVar + } + case 0xFE0E: + { + return OpCodes.Stloc; // stloc InlineVar + } + case 0xFE0F: + { + return OpCodes.Localloc; // localloc InlineNone + } + case 0xFE11: + { + return OpCodes.Endfilter; // endfilter InlineNone + } + case 0xFE12: + { + return OpCodes.Unaligned; // unaligned. ShortInlineI + } + case 0xFE13: + { + return OpCodes.Volatile; // volatile. InlineNone + } + case 0xFE14: + { + return OpCodes.Tailcall; // tail. InlineNone + } + case 0xFE15: + { + return OpCodes.Initobj; // initobj InlineType + } + case 0xFE16: + { + return OpCodes.Constrained; // constrained. InlineType + } + case 0xFE17: + { + return OpCodes.Cpblk; // cpblk InlineNone + } + case 0xFE18: + { + return OpCodes.Initblk; // initblk InlineNone + } + case 0xFE1A: + { + return OpCodes.Rethrow; // rethrow InlineNone + } + case 0xFE1C: + { + return OpCodes.Sizeof; // sizeof InlineType + } + case 0xFE1D: + { + return OpCodes.Refanytype; // refanytype InlineNone + } + case 0xFE1E: + { + return OpCodes.Readonly; // readonly. InlineNone + } + default: + { + throw new Exception($"Unknown opcode 0x{opCode}."); + } + } + } + } + } + } +} diff --git a/src/MsgPack.Serialization.ILGeneration/Serialization/Reflection/ILTranslator.Parse.tt b/src/MsgPack.Serialization.ILGeneration/Serialization/Reflection/ILTranslator.Parse.tt new file mode 100644 index 000000000..36c108a0e --- /dev/null +++ b/src/MsgPack.Serialization.ILGeneration/Serialization/Reflection/ILTranslator.Parse.tt @@ -0,0 +1,82 @@ +<#@ template debug="false" hostspecific="false" language="C#" #> +<#@ assembly name="System.Core" #> +<#@ import namespace="System.Collections.Generic" #> +<#@ import namespace="System.Linq" #> +<#@ import namespace="System.Reflection" #> +<#@ import namespace="System.Reflection.Emit" #> +<#@ output extension=".cs" #> +<# +var opCodes = typeof(OpCodes).GetFields(BindingFlags.Public | BindingFlags.Static).Select(f => (OpCode)f.GetValue(null)).ToArray(); +var opCode1Bytes = opCodes.Where(o => o.Value >= 0 && !o.Name.StartsWith("prefix")).ToArray(); +var opCode2Bytes = opCodes.Where(o => o.Value < 0).ToArray(); +#> +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +// +// This file is generated from acompanying .tt file. +// DO NOT edit this file directly, edit .tt file instead. + +using System; +using System.Buffers.Binary; +using System.Reflection.Emit; + +namespace MsgPack.Serialization.Reflection +{ + partial class ILTranspiler + { + private static OpCode Parse(Span il) + { + switch(il[0]) + { +<# +foreach (var opCode in opCode1Bytes) +{ +#> + case 0x<#= opCode.Value.ToString("X2") #>: + { + return OpCodes.<#= Pascalize(opCode.Name) #>; // <#= opCode.Name #> <#= opCode.OperandType #> + } +<# +} +#> + default: + { + var opCode = BinaryPrimitives.ReadUInt16BigEndian(il); + switch (opCode) + { +<# +foreach (var opCode in opCode2Bytes) +{ +#> + case 0x<#= opCode.Value.ToString("X4") #>: + { + return OpCodes.<#= Pascalize(opCode.Name) #>; // <#= opCode.Name #> <#= opCode.OperandType #> + } +<# +} +#> + default: + { + throw new Exception($"Unknown opcode 0x{opCode}."); + } + } + } + } + } + } +} +<#+ +static readonly char[] SplitDelimiters = { '.' }; +static readonly Dictionary OpCodeNameMap = + new Dictionary + { + ["tail."] = "Tailcall", + }; + +string Pascalize(string name) + => OpCodeNameMap.TryGetValue(name, out var mapped) ? + mapped : + String.Join("_", name.Split(SplitDelimiters, StringSplitOptions.RemoveEmptyEntries).Select(n => Char.ToUpperInvariant(n[0]) + n.Substring(1))); +#> diff --git a/src/MsgPack.Serialization.ILGeneration/Serialization/Reflection/ILTranspiler.cs b/src/MsgPack.Serialization.ILGeneration/Serialization/Reflection/ILTranspiler.cs new file mode 100644 index 000000000..ca208fbad --- /dev/null +++ b/src/MsgPack.Serialization.ILGeneration/Serialization/Reflection/ILTranspiler.cs @@ -0,0 +1,308 @@ +// Copyright (c) FUJIWARA, Yusuke and all contributors. +// This file is licensed under Apache2 license. +// See the LICENSE in the project root for more information. + +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; + +namespace MsgPack.Serialization.Reflection +{ + /// + /// Defines IL transpilation logics. + /// + internal static partial class ILTranspiler + { + /// + /// Ports specified method IL instructions to the specified in context of specified . + /// + /// A context for current code generation. + /// Real which contains IL source. + /// which will be emitted ported IL instructions. + public static void Port(ILGenerationContext context, MethodBase source, ILGenerator destination) + { + var methodBody = source.GetMethodBody(); + if (methodBody == null) + { + throw new ArgumentException($"The method {source} does not have method body.", nameof(source)); + } + + var tries = methodBody.ExceptionHandlingClauses.ToLookup(e => e.TryOffset); + var filters = methodBody.ExceptionHandlingClauses.ToLookup(e => e.FilterOffset); + var handlers = methodBody.ExceptionHandlingClauses.ToLookup(e => e.HandlerOffset); + var endClauses = methodBody.ExceptionHandlingClauses.ToLookup(e => e.HandlerOffset); + + var locals = methodBody.LocalVariables.OrderBy(l => l.LocalIndex).Select(l => destination.DeclareLocal(l.LocalType, l.IsPinned)).ToArray(); + var labels = new Dictionary>(); + + var ilStream = methodBody.GetILAsByteArray()!.AsSpan(); + var emits = new List<(int Offset, List> Emitters)>(); + + // Scan + ParseILStream(context, source, destination, locals, labels, ilStream, emits); + + // Real emit + EmitILStream(destination, tries, filters, handlers, endClauses, labels, emits); + } + + private static void ParseILStream(ILGenerationContext context, MethodBase source, ILGenerator destination, LocalBuilder[] locals, Dictionary> labels, Span ilStream, List<(int Offset, List> Emitters)> emits) + { + var offset = 0; + while (offset < ilStream.Length) + { + var currentOffset = offset; + var currentEmits = new List>(); + + var opCode = Parse(ilStream); + ilStream = ilStream.Slice(1); + offset++; + switch (opCode.OperandType) + { + case OperandType.InlineBrTarget: + { + var branchOffset = BinaryPrimitives.ReadInt32LittleEndian(ilStream); + var label = destination.DefineLabel(); + ilStream = ilStream.Slice(sizeof(int)); + offset += sizeof(int); + AddLabel(labels, branchOffset + offset, label); + currentEmits.Add(il => il.Emit(opCode, label)); + break; + } + case OperandType.InlineField: + { + var token = BinaryPrimitives.ReadInt32LittleEndian(ilStream); + var field = context.ResolveField(source, token); + ilStream = ilStream.Slice(sizeof(int)); + offset += sizeof(int); + currentEmits.Add(il => il.Emit(opCode, field!)); + break; + } + case OperandType.InlineI: + { + var immediate = BinaryPrimitives.ReadInt32LittleEndian(ilStream); + ilStream = ilStream.Slice(sizeof(int)); + offset += sizeof(int); + currentEmits.Add(il => il.Emit(opCode, immediate)); + break; + } + case OperandType.InlineI8: + { + var immediate = BinaryPrimitives.ReadInt64LittleEndian(ilStream); + ilStream = ilStream.Slice(sizeof(long)); + offset += sizeof(long); + currentEmits.Add(il => il.Emit(opCode, immediate)); + break; + } + case OperandType.InlineMethod: + { + var token = BinaryPrimitives.ReadInt32LittleEndian(ilStream); + ilStream = ilStream.Slice(sizeof(int)); + offset += sizeof(int); + + if (opCode == OpCodes.Call || opCode == OpCodes.Callvirt) + { + var method = context.ResolveMethod(source, token); + currentEmits.Add(il => il.EmitCall(opCode, method, null)); + } + else if (opCode == OpCodes.Newobj) + { + var constructor = context.ResolveConstructor(source, token); + currentEmits.Add(il => il.Emit(opCode, constructor)); + } + else if (opCode == OpCodes.Ldftn || opCode == OpCodes.Ldvirtftn || opCode == OpCodes.Jmp) + { + var method = context.ResolveMethod(source, token); + currentEmits.Add(il => il.Emit(opCode, method)); + } + else + { + throw new NotSupportedException($"Unknown opcode '{opCode.Name}'(0x{opCode.Value:X}) at offset {offset}."); + } + + break; + } + case OperandType.InlineNone: + { + currentEmits.Add(il => il.Emit(opCode)); + break; + } + case OperandType.InlineR: + { + var immediate = BitConverter.Int64BitsToDouble(BinaryPrimitives.ReadInt64LittleEndian(ilStream)); + ilStream = ilStream.Slice(sizeof(long)); + offset += sizeof(long); + currentEmits.Add(il => il.Emit(opCode, immediate)); + break; + } + case OperandType.InlineSig: + { + throw new NotImplementedException($"OpCode '{opCode.Name}' at offset {offset} in {source} is not implemented."); + } + case OperandType.InlineString: + { + var token = BinaryPrimitives.ReadInt32LittleEndian(ilStream); + ilStream = ilStream.Slice(sizeof(int)); + offset += sizeof(int); + var literal = source.Module.ResolveString(token); + currentEmits.Add(il => il.Emit(opCode, literal)); + break; + } + case OperandType.InlineSwitch: + { + var caseCount = BinaryPrimitives.ReadInt32LittleEndian(ilStream); + ilStream = ilStream.Slice(sizeof(int)); + offset += sizeof(int); + + var caseLabels = new List