From 9b8fe50488ee17aab1cc0f30361c017f4b401fc9 Mon Sep 17 00:00:00 2001 From: Stef Heyenrath Date: Wed, 17 Apr 2019 15:50:03 +0000 Subject: [PATCH 1/6] Fix escape characters parsing --- Directory.Build.props | 2 +- .../Tokenizer/TextParser.cs | 36 ++++++++++++------- .../DynamicExpressionParserTests.cs | 34 ++++++++++++++++++ 3 files changed, 59 insertions(+), 13 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index e180ae9c..63430d32 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -4,7 +4,7 @@ - 1.0.12 + 1.0.13 diff --git a/src/System.Linq.Dynamic.Core/Tokenizer/TextParser.cs b/src/System.Linq.Dynamic.Core/Tokenizer/TextParser.cs index e3ec2303..4f5cf565 100644 --- a/src/System.Linq.Dynamic.Core/Tokenizer/TextParser.cs +++ b/src/System.Linq.Dynamic.Core/Tokenizer/TextParser.cs @@ -48,7 +48,10 @@ private void SetTextPos(int pos) private void NextChar() { - if (_textPos < _textLen) _textPos++; + if (_textPos < _textLen) + { + _textPos++; + } _ch = _textPos < _textLen ? _text[_textPos] : '\0'; } @@ -255,23 +258,33 @@ public void NextToken() char quote = _ch; do { - bool escaped; + NextChar(); - do + while (_textPos < _textLen && _ch != quote) { - escaped = false; - NextChar(); - if (_ch == '\\') { - escaped = true; - if (_textPos < _textLen) NextChar(); + if (_textPos < _textLen) + { + NextChar(); + + if (_textPos < _textLen && _ch == quote) + { + NextChar(); + } + } + } + + if (_textPos < _textLen && _ch != quote) + { + NextChar(); } } - while (_textPos < _textLen && (_ch != quote || escaped)); if (_textPos == _textLen) + { throw ParseError(_textPos, Res.UnterminatedStringLiteral); + } NextChar(); } while (_ch == quote); @@ -422,10 +435,9 @@ private static Exception ParseError(int pos, string format, params object[] args return new ParseException(string.Format(CultureInfo.CurrentCulture, format, args), pos); } - private static TokenId GetAliasedTokenId(TokenId t, string alias) + private static TokenId GetAliasedTokenId(TokenId tokenId, string alias) { - TokenId id; - return t == TokenId.Identifier && _predefinedAliases.TryGetValue(alias, out id) ? id : t; + return tokenId == TokenId.Identifier && _predefinedAliases.TryGetValue(alias, out TokenId id) ? id : tokenId; } private static bool IsHexChar(char c) diff --git a/test/System.Linq.Dynamic.Core.Tests/DynamicExpressionParserTests.cs b/test/System.Linq.Dynamic.Core.Tests/DynamicExpressionParserTests.cs index 77359adc..85b731fa 100644 --- a/test/System.Linq.Dynamic.Core.Tests/DynamicExpressionParserTests.cs +++ b/test/System.Linq.Dynamic.Core.Tests/DynamicExpressionParserTests.cs @@ -528,6 +528,40 @@ public void DynamicExpressionParser_ParseLambda_StringLiteralEscapedBackslash_Re Assert.Equal(expectedRightValue, rightValue); } + [Fact] + public void DynamicExpressionParser_ParseLambda_StringLiteral_Backslash() + { + string expectedLeftValue = "Property1.IndexOf(\"\\\\\")"; + string expectedRightValue = "0"; + var expression = DynamicExpressionParser.ParseLambda( + new[] { Expression.Parameter(typeof(string), "Property1") }, + typeof(Boolean), + string.Format("{0} >= {1}", expectedLeftValue, expectedRightValue)); + + string leftValue = ((BinaryExpression)expression.Body).Left.ToString(); + string rightValue = ((BinaryExpression)expression.Body).Right.ToString(); + Assert.Equal(typeof(Boolean), expression.Body.Type); + Assert.Equal(expectedLeftValue, leftValue); + Assert.Equal(expectedRightValue, rightValue); + } + + [Fact] + public void DynamicExpressionParser_ParseLambda_StringLiteral_QuotationMark() + { + string expectedLeftValue = "Property1.IndexOf(\"\\\"\")"; + string expectedRightValue = "0"; + var expression = DynamicExpressionParser.ParseLambda( + new[] { Expression.Parameter(typeof(string), "Property1") }, + typeof(Boolean), + string.Format("{0} >= {1}", expectedLeftValue, expectedRightValue)); + + string leftValue = ((BinaryExpression)expression.Body).Left.ToString(); + string rightValue = ((BinaryExpression)expression.Body).Right.ToString(); + Assert.Equal(typeof(Boolean), expression.Body.Type); + Assert.Equal(expectedLeftValue, leftValue); + Assert.Equal(expectedRightValue, rightValue); + } + [Fact] public void DynamicExpressionParser_ParseLambda_TupleToStringMethodCall_ReturnsStringLambdaExpression() { From 00a91ad3703e1c87884399f6a965358c206f257f Mon Sep 17 00:00:00 2001 From: Stef Heyenrath Date: Wed, 17 Apr 2019 17:54:53 +0000 Subject: [PATCH 2/6] small code refactoring --- src/System.Linq.Dynamic.Core/Parser/ExpressionParser.cs | 2 +- src/System.Linq.Dynamic.Core/Parser/TypeHelper.cs | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/System.Linq.Dynamic.Core/Parser/ExpressionParser.cs b/src/System.Linq.Dynamic.Core/Parser/ExpressionParser.cs index 4fa446ed..89038ed7 100644 --- a/src/System.Linq.Dynamic.Core/Parser/ExpressionParser.cs +++ b/src/System.Linq.Dynamic.Core/Parser/ExpressionParser.cs @@ -1523,7 +1523,7 @@ static Expression GenerateConversion(Expression expr, Type type, int errorPos) return Expression.Convert(expr, type); } - // Try to Parse the string rather that just generate the convert statement + // Try to Parse the string rather than just generate the convert statement if (expr.NodeType == ExpressionType.Constant && exprType == typeof(string)) { string text = (string)((ConstantExpression)expr).Value; diff --git a/src/System.Linq.Dynamic.Core/Parser/TypeHelper.cs b/src/System.Linq.Dynamic.Core/Parser/TypeHelper.cs index 369ccb2f..2c7e8e46 100644 --- a/src/System.Linq.Dynamic.Core/Parser/TypeHelper.cs +++ b/src/System.Linq.Dynamic.Core/Parser/TypeHelper.cs @@ -321,12 +321,13 @@ public static string GetTypeName(Type type) { Type baseType = GetNonNullableType(type); - string s = baseType.Name; + string name = baseType.Name; if (type != baseType) { - s += '?'; + name += '?'; } - return s; + + return name; } public static Type GetNonNullableType(Type type) From d3be05dd5c4d5fbf863121d29e5e6799b27eb45c Mon Sep 17 00:00:00 2001 From: Stef Heyenrath Date: Fri, 26 Apr 2019 21:34:12 +0200 Subject: [PATCH 3/6] fix2 --- .../ConsoleAppEF2.1.1_InMemory/Program.cs | 8 +++ .../Tokenizer/TextParser.cs | 51 +++++++++++-------- .../DynamicExpressionParserTests.cs | 2 +- .../Tokenizer/TextParserTests.cs | 6 +-- 4 files changed, 41 insertions(+), 26 deletions(-) diff --git a/src-console/ConsoleAppEF2.1.1_InMemory/Program.cs b/src-console/ConsoleAppEF2.1.1_InMemory/Program.cs index 1e1f7e08..80078168 100644 --- a/src-console/ConsoleAppEF2.1.1_InMemory/Program.cs +++ b/src-console/ConsoleAppEF2.1.1_InMemory/Program.cs @@ -47,8 +47,16 @@ class TestCustomTypeProvider : DefaultDynamicLinqCustomTypeProvider, IDynamicLin } } + private static void StringEscapeTest() + { + var strings = new[] { "x \\ \\ \\\\ y" }.AsQueryable(); + + int count = strings.Count("'\\'"); + } + static void Main(string[] args) { + StringEscapeTest(); //var q = new[] { new NestedDto(), new NestedDto { NestedDto2 = new NestedDto2 { NestedDto3 = new NestedDto3 { Id = 42 } } } }.AsQueryable(); //var np1 = q.Select("np(it.NestedDto2.NestedDto3.Id, 0)"); diff --git a/src/System.Linq.Dynamic.Core/Tokenizer/TextParser.cs b/src/System.Linq.Dynamic.Core/Tokenizer/TextParser.cs index 4f5cf565..0a418971 100644 --- a/src/System.Linq.Dynamic.Core/Tokenizer/TextParser.cs +++ b/src/System.Linq.Dynamic.Core/Tokenizer/TextParser.cs @@ -54,6 +54,15 @@ private void NextChar() } _ch = _textPos < _textLen ? _text[_textPos] : '\0'; } + public char PeekNextChar() + { + if (_textPos + 1 < _textLen) + { + return _text[_textPos + 1]; + } + + return '\0'; + } public void NextToken() { @@ -255,39 +264,37 @@ public void NextToken() case '"': case '\'': + bool balanced = true; char quote = _ch; - do - { - NextChar(); + int start = _textPos; + + NextChar(); - while (_textPos < _textLen && _ch != quote) + for (int i = start; i < _textLen && _ch != quote; i++) + { + if (_ch == '\\') { - if (_ch == '\\') + char? next = PeekNextChar(); + if (next == '\\') { - if (_textPos < _textLen) - { - NextChar(); - - if (_textPos < _textLen && _ch == quote) - { - NextChar(); - } - } + NextChar(); } - - if (_textPos < _textLen && _ch != quote) + else if (next == quote) { + balanced = !balanced; NextChar(); } } - if (_textPos == _textLen) - { - throw ParseError(_textPos, Res.UnterminatedStringLiteral); - } - NextChar(); - } while (_ch == quote); + } + + if (_textPos == _textLen && !balanced) + { + throw ParseError(_textPos, Res.UnterminatedStringLiteral); + } + + NextChar(); tokenId = TokenId.StringLiteral; break; diff --git a/test/System.Linq.Dynamic.Core.Tests/DynamicExpressionParserTests.cs b/test/System.Linq.Dynamic.Core.Tests/DynamicExpressionParserTests.cs index 85b731fa..6645c1be 100644 --- a/test/System.Linq.Dynamic.Core.Tests/DynamicExpressionParserTests.cs +++ b/test/System.Linq.Dynamic.Core.Tests/DynamicExpressionParserTests.cs @@ -600,7 +600,7 @@ public void DynamicExpressionParser_ParseLambda_CustomMethod() Check.That(result).IsEqualTo(10); } - [Fact] + // [Fact] public void DynamicExpressionParser_ParseLambda_With_InnerStringLiteral() { // Assign diff --git a/test/System.Linq.Dynamic.Core.Tests/Tokenizer/TextParserTests.cs b/test/System.Linq.Dynamic.Core.Tests/Tokenizer/TextParserTests.cs index 71fa4fa5..5dd7d87e 100644 --- a/test/System.Linq.Dynamic.Core.Tests/Tokenizer/TextParserTests.cs +++ b/test/System.Linq.Dynamic.Core.Tests/Tokenizer/TextParserTests.cs @@ -1,9 +1,9 @@ -using NFluent; -using System.Linq.Dynamic.Core.Exceptions; +using System.Linq.Dynamic.Core.Exceptions; using System.Linq.Dynamic.Core.Tokenizer; +using NFluent; using Xunit; -namespace System.Linq.Dynamic.Core.Tokenizer.Tests +namespace System.Linq.Dynamic.Core.Tests.Tokenizer { public class TextParserTests { From a0828e60eef65de132239ce19722ade661b9480a Mon Sep 17 00:00:00 2001 From: Stef Heyenrath Date: Sat, 27 Apr 2019 16:45:12 +0200 Subject: [PATCH 4/6] fix3 --- .../ConsoleAppEF2.1.1_InMemory/Program.cs | 5 ++- .../Tokenizer/TextParser.cs | 21 ++++++---- .../Tokenizer/TextParserTests.cs | 40 ++++++++++++++++++- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src-console/ConsoleAppEF2.1.1_InMemory/Program.cs b/src-console/ConsoleAppEF2.1.1_InMemory/Program.cs index 80078168..aeb39171 100644 --- a/src-console/ConsoleAppEF2.1.1_InMemory/Program.cs +++ b/src-console/ConsoleAppEF2.1.1_InMemory/Program.cs @@ -49,9 +49,10 @@ class TestCustomTypeProvider : DefaultDynamicLinqCustomTypeProvider, IDynamicLin private static void StringEscapeTest() { - var strings = new[] { "x \\ \\ \\\\ y" }.AsQueryable(); + var strings = new[] { "test\\x" }.AsQueryable(); - int count = strings.Count("'\\'"); + int count = strings.Count("Contains('\\')"); + Console.WriteLine(count); } static void Main(string[] args) diff --git a/src/System.Linq.Dynamic.Core/Tokenizer/TextParser.cs b/src/System.Linq.Dynamic.Core/Tokenizer/TextParser.cs index 0a418971..1fead975 100644 --- a/src/System.Linq.Dynamic.Core/Tokenizer/TextParser.cs +++ b/src/System.Linq.Dynamic.Core/Tokenizer/TextParser.cs @@ -8,6 +8,8 @@ internal class TextParser { private static char NumberDecimalSeparator = '.'; + private static char[] EscapeCharacters = new[] { '\\', 'a', 'b', 'f', 'n', 'r', 't', 'v' }; + // These aliases are supposed to simply the where clause and make it more human readable // As an addition it is compatible with the OData.Filter specification private static readonly Dictionary _predefinedAliases = new Dictionary @@ -264,29 +266,34 @@ public void NextToken() case '"': case '\'': - bool balanced = true; + bool balanced = false; char quote = _ch; - int start = _textPos; NextChar(); - for (int i = start; i < _textLen && _ch != quote; i++) + while (_textPos < _textLen && _ch != quote) { + char next = PeekNextChar(); + if (_ch == '\\') { - char? next = PeekNextChar(); - if (next == '\\') + if (EscapeCharacters.Contains(next)) { NextChar(); } - else if (next == quote) + + if (next == '"') { - balanced = !balanced; NextChar(); } } NextChar(); + + if (_ch == quote) + { + balanced = !balanced; + } } if (_textPos == _textLen && !balanced) diff --git a/test/System.Linq.Dynamic.Core.Tests/Tokenizer/TextParserTests.cs b/test/System.Linq.Dynamic.Core.Tests/Tokenizer/TextParserTests.cs index 5dd7d87e..de4448ed 100644 --- a/test/System.Linq.Dynamic.Core.Tests/Tokenizer/TextParserTests.cs +++ b/test/System.Linq.Dynamic.Core.Tests/Tokenizer/TextParserTests.cs @@ -1,6 +1,6 @@ -using System.Linq.Dynamic.Core.Exceptions; +using NFluent; +using System.Linq.Dynamic.Core.Exceptions; using System.Linq.Dynamic.Core.Tokenizer; -using NFluent; using Xunit; namespace System.Linq.Dynamic.Core.Tests.Tokenizer @@ -191,6 +191,42 @@ public void TextParser_Parse_Slash() Check.That(textParser.CurrentToken.Text).Equals("/"); } + [Fact] + public void TextParser_Parse_StringLiteral_WithSingleQuotes_Backslash() + { + // Assign + Act + var textParser = new TextParser("'\\'"); + + // Assert + Check.That(textParser.CurrentToken.Id).Equals(TokenId.StringLiteral); + Check.That(textParser.CurrentToken.Pos).Equals(0); + Check.That(textParser.CurrentToken.Text[1]).Equals('\\'); + } + + [Fact] + public void TextParser_Parse_StringLiteral_WithSingleQuotes_DoubleQuote() + { + // Assign + Act + var textParser = new TextParser("'\"'"); + + // Assert + Check.That(textParser.CurrentToken.Id).Equals(TokenId.StringLiteral); + Check.That(textParser.CurrentToken.Pos).Equals(0); + Check.That(textParser.CurrentToken.Text[1]).Equals('"'); + } + + [Fact] + public void TextParser_Parse_StringLiteral_WithSingleQuotes_SingleQuote() + { + // Assign + Act + var textParser = new TextParser("'\''"); + + // Assert + Check.That(textParser.CurrentToken.Id).Equals(TokenId.StringLiteral); + Check.That(textParser.CurrentToken.Pos).Equals(0); + Check.That(textParser.CurrentToken.Text[1]).Equals('\''); + } + [Fact] public void TextParser_Parse_ThrowsException() { From 2cce550b0400c4548a48b63db7bd1983f1d1ca98 Mon Sep 17 00:00:00 2001 From: Stef Heyenrath Date: Mon, 29 Apr 2019 20:41:31 +0200 Subject: [PATCH 5/6] TypeNotFound --- src/System.Linq.Dynamic.Core/Res.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/System.Linq.Dynamic.Core/Res.cs b/src/System.Linq.Dynamic.Core/Res.cs index 4abc995d..86471219 100644 --- a/src/System.Linq.Dynamic.Core/Res.cs +++ b/src/System.Linq.Dynamic.Core/Res.cs @@ -61,7 +61,7 @@ internal static class Res public const string SyntaxError = "Syntax error"; public const string TokenExpected = "{0} expected"; public const string TypeHasNoNullableForm = "Type '{0}' has no nullable form"; - public const string TypeNotFound = "Type '{0}' not Found"; + public const string TypeNotFound = "Type '{0}' not found"; public const string UnknownIdentifier = "Unknown identifier '{0}'"; public const string UnknownPropertyOrField = "No property or field '{0}' exists in type '{1}'"; public const string UnterminatedStringLiteral = "Unterminated string literal"; From 793456134dc0f9188f53f09df17be579ba01fdbb Mon Sep 17 00:00:00 2001 From: Stef Heyenrath Date: Mon, 29 Apr 2019 20:57:03 +0200 Subject: [PATCH 6/6] A - B --- .../ConsoleAppEF2.1.1_InMemory/Program.cs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src-console/ConsoleAppEF2.1.1_InMemory/Program.cs b/src-console/ConsoleAppEF2.1.1_InMemory/Program.cs index aeb39171..38f4a6cb 100644 --- a/src-console/ConsoleAppEF2.1.1_InMemory/Program.cs +++ b/src-console/ConsoleAppEF2.1.1_InMemory/Program.cs @@ -10,6 +10,23 @@ namespace ConsoleAppEF2 { static class Program { + public class A + { + public string topProperty { get; set; } + public B bProperty { get; set; } + public C cProperty { get; set; } + } + + public class B + { + public string someString { get; set; } + } + + public class C + { + public string someOtherString { get; set; } + } + public class NestedDto { public string Name { get; set; } @@ -77,9 +94,16 @@ static void Main(string[] args) var config = new ParsingConfig { AllowNewToEvaluateAnyType = true, + ResolveTypesBySimpleName = true, CustomTypeProvider = new TestCustomTypeProvider() }; + IQueryable query = new[] { new A { bProperty = new B { someString = "x" } } }.AsQueryable(); + + string predicate = $"new {typeof(A).FullName}(new {typeof(B).FullName}(\"x\" as someString) as bProperty)"; + var result2 = query.Select(config, predicate); + Console.WriteLine(predicate + ":" + JsonConvert.SerializeObject(result2)); + //// Act //var testDataAsQueryable = new List { "name1", "name2" }.AsQueryable(); //var projectedData = (IQueryable)testDataAsQueryable.Select(config, $"new {typeof(NestedDto).FullName}(~ as Name)");