From 3e648b5822aeeea3fe13c218772667ead991c80f Mon Sep 17 00:00:00 2001 From: Bruce Dunwiddie Date: Fri, 15 Jul 2022 06:01:23 -0500 Subject: [PATCH] Fixed "SELECT COUNT(DISTINCT ...)" and "SELECT NULL..." by adding new expression types. #101 #104 --- ...er.cs => TSQLOperationExpressionParser.cs} | 6 +-- .../Parsers/TSQLSelectExpressionParser.cs | 2 +- .../Parsers/TSQLValueExpressionParser.cs | 47 ++++++++++++++++- .../TSQLVariableAssignmentExpressionParser.cs | 2 +- .../TSQLDuplicateSpecificationExpression.cs | 31 ++++++++++++ .../TSQL_Parser/Expressions/TSQLExpression.cs | 20 +++++++- .../Expressions/TSQLExpressionType.cs | 14 +++++- .../Expressions/TSQLNullExpression.cs | 21 ++++++++ ...pression.cs => TSQLOperationExpression.cs} | 4 +- TSQL_Parser/TSQL_Parser/TSQL_Parser.csproj | 6 ++- .../Tests/Clauses/SelectClauseTests.cs | 18 +++---- ...onTests.cs => OperationExpressionTests.cs} | 10 ++-- .../Tests/Statements/SelectStatementTests.cs | 50 ++++++++++++++++--- TSQL_Parser/Tests/Tests.csproj | 2 +- 14 files changed, 196 insertions(+), 37 deletions(-) rename TSQL_Parser/TSQL_Parser/Expressions/Parsers/{TSQLOperatorExpressionParser.cs => TSQLOperationExpressionParser.cs} (86%) create mode 100644 TSQL_Parser/TSQL_Parser/Expressions/TSQLDuplicateSpecificationExpression.cs create mode 100644 TSQL_Parser/TSQL_Parser/Expressions/TSQLNullExpression.cs rename TSQL_Parser/TSQL_Parser/Expressions/{TSQLOperatorExpression.cs => TSQLOperationExpression.cs} (81%) rename TSQL_Parser/Tests/Expressions/{OperatorExpressionTests.cs => OperationExpressionTests.cs} (89%) diff --git a/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLOperatorExpressionParser.cs b/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLOperationExpressionParser.cs similarity index 86% rename from TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLOperatorExpressionParser.cs rename to TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLOperationExpressionParser.cs index e7c2bad..c50051a 100644 --- a/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLOperatorExpressionParser.cs +++ b/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLOperationExpressionParser.cs @@ -8,13 +8,13 @@ namespace TSQL.Expressions.Parsers { - internal class TSQLOperatorExpressionParser + internal class TSQLOperationExpressionParser { - public TSQLOperatorExpression Parse( + public TSQLOperationExpression Parse( ITSQLTokenizer tokenizer, TSQLExpression leftSide) { - TSQLOperatorExpression opExpression = new TSQLOperatorExpression(); + TSQLOperationExpression opExpression = new TSQLOperationExpression(); opExpression.LeftSide = leftSide; opExpression.Operator = tokenizer.Current.AsOperator; diff --git a/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLSelectExpressionParser.cs b/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLSelectExpressionParser.cs index 8901e88..5152d1a 100644 --- a/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLSelectExpressionParser.cs +++ b/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLSelectExpressionParser.cs @@ -51,7 +51,7 @@ public TSQLExpression Parse(ITSQLTokenizer tokenizer) } else { - return new TSQLOperatorExpressionParser().Parse( + return new TSQLOperationExpressionParser().Parse( tokenizer, expression); } diff --git a/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLValueExpressionParser.cs b/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLValueExpressionParser.cs index 2be6776..92de3d9 100644 --- a/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLValueExpressionParser.cs +++ b/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLValueExpressionParser.cs @@ -22,7 +22,7 @@ public TSQLExpression Parse(ITSQLTokenizer tokenizer) tokenizer.Current.Type.In( TSQLTokenType.Operator)) { - return new TSQLOperatorExpressionParser().Parse( + return new TSQLOperationExpressionParser().Parse( tokenizer, expression); } @@ -125,6 +125,39 @@ public TSQLExpression ParseNext( #endregion } } + else if ( + tokenizer.Current.IsKeyword(TSQLKeywords.DISTINCT) || + tokenizer.Current.IsKeyword(TSQLKeywords.ALL)) + { + #region parse rest of expression contained inside parenthesis + + TSQLDuplicateSpecificationExpression distinct = new TSQLDuplicateSpecificationExpression(); + + distinct.Tokens.Add(tokenizer.Current); + + if (tokenizer.Current.IsKeyword(TSQLKeywords.ALL)) + { + distinct.DuplicateSpecificationType = TSQLDuplicateSpecificationExpression.TSQLDuplicateSpecificationType.All; + } + else + { + distinct.DuplicateSpecificationType = TSQLDuplicateSpecificationExpression.TSQLDuplicateSpecificationType.Distinct; + } + + TSQLTokenParserHelper.ReadThroughAnyCommentsOrWhitespace( + tokenizer, + distinct.Tokens); + + distinct.InnerExpression = + new TSQLValueExpressionParser().Parse( + tokenizer); + + distinct.Tokens.AddRange(distinct.InnerExpression.Tokens); + + return distinct; + + #endregion + } else if (tokenizer.Current.Type.In( TSQLTokenType.Variable, TSQLTokenType.SystemVariable)) @@ -158,6 +191,18 @@ public TSQLExpression ParseNext( return constant; } + else if (tokenizer.Current.IsKeyword(TSQLKeywords.NULL)) + { + TSQLNullExpression nullExp = new TSQLNullExpression(); + + nullExp.Tokens.Add(tokenizer.Current); + + TSQLTokenParserHelper.ReadThroughAnyCommentsOrWhitespace( + tokenizer, + nullExp.Tokens); + + return nullExp; + } else if (tokenizer.Current.IsKeyword(TSQLKeywords.CASE)) { return new TSQLCaseExpressionParser().Parse(tokenizer); diff --git a/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLVariableAssignmentExpressionParser.cs b/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLVariableAssignmentExpressionParser.cs index d1d03d4..8bd3e40 100644 --- a/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLVariableAssignmentExpressionParser.cs +++ b/TSQL_Parser/TSQL_Parser/Expressions/Parsers/TSQLVariableAssignmentExpressionParser.cs @@ -44,7 +44,7 @@ public TSQLVariableAssignmentExpression Parse( tokenizer.Current.Type.In( TSQLTokenType.Operator)) { - rightSide = new TSQLOperatorExpressionParser().Parse( + rightSide = new TSQLOperationExpressionParser().Parse( tokenizer, rightSide); } diff --git a/TSQL_Parser/TSQL_Parser/Expressions/TSQLDuplicateSpecificationExpression.cs b/TSQL_Parser/TSQL_Parser/Expressions/TSQLDuplicateSpecificationExpression.cs new file mode 100644 index 0000000..46577c2 --- /dev/null +++ b/TSQL_Parser/TSQL_Parser/Expressions/TSQLDuplicateSpecificationExpression.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using TSQL.Tokens; + +namespace TSQL.Expressions +{ + public class TSQLDuplicateSpecificationExpression : TSQLExpression + { + public override TSQLExpressionType Type + { + get + { + return TSQLExpressionType.DuplicateSpecification; + } + } + + public TSQLDuplicateSpecificationType DuplicateSpecificationType { get; internal set; } + + public TSQLExpression InnerExpression { get; internal set; } + + public enum TSQLDuplicateSpecificationType + { + Distinct, + All + } + } +} diff --git a/TSQL_Parser/TSQL_Parser/Expressions/TSQLExpression.cs b/TSQL_Parser/TSQL_Parser/Expressions/TSQLExpression.cs index 0b65bce..f9fba39 100644 --- a/TSQL_Parser/TSQL_Parser/Expressions/TSQLExpression.cs +++ b/TSQL_Parser/TSQL_Parser/Expressions/TSQLExpression.cs @@ -72,11 +72,11 @@ public TSQLMulticolumnExpression AsMulticolumn } } - public TSQLOperatorExpression AsOperator + public TSQLOperationExpression AsOperation { get { - return this as TSQLOperatorExpression; + return this as TSQLOperationExpression; } } @@ -119,5 +119,21 @@ public TSQLValueAsTypeExpression AsValueAsType return this as TSQLValueAsTypeExpression; } } + + public TSQLNullExpression AsNull + { + get + { + return this as TSQLNullExpression; + } + } + + public TSQLDuplicateSpecificationExpression AsDuplicateSpecification + { + get + { + return this as TSQLDuplicateSpecificationExpression; + } + } } } diff --git a/TSQL_Parser/TSQL_Parser/Expressions/TSQLExpressionType.cs b/TSQL_Parser/TSQL_Parser/Expressions/TSQLExpressionType.cs index 1a1b2c8..7195219 100644 --- a/TSQL_Parser/TSQL_Parser/Expressions/TSQLExpressionType.cs +++ b/TSQL_Parser/TSQL_Parser/Expressions/TSQLExpressionType.cs @@ -29,7 +29,7 @@ public enum TSQLExpressionType /// Multicolumn, - Operator, + Operation, /// /// i.e. an expression surrounded by parenthesis, but not containing a subquery @@ -54,6 +54,16 @@ public enum TSQLExpressionType /// /// e.g. 123.45 AS INT (only used as an argument to CAST function) /// - ValueAsType + ValueAsType, + + /// + /// e.g. NULL + /// + Null, + + /// + /// e.g. DISTINCT or ALL + /// + DuplicateSpecification } } diff --git a/TSQL_Parser/TSQL_Parser/Expressions/TSQLNullExpression.cs b/TSQL_Parser/TSQL_Parser/Expressions/TSQLNullExpression.cs new file mode 100644 index 0000000..c44d66f --- /dev/null +++ b/TSQL_Parser/TSQL_Parser/Expressions/TSQLNullExpression.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using TSQL.Tokens; + +namespace TSQL.Expressions +{ + public class TSQLNullExpression : TSQLExpression + { + public override TSQLExpressionType Type + { + get + { + return TSQLExpressionType.Null; + } + } + } +} diff --git a/TSQL_Parser/TSQL_Parser/Expressions/TSQLOperatorExpression.cs b/TSQL_Parser/TSQL_Parser/Expressions/TSQLOperationExpression.cs similarity index 81% rename from TSQL_Parser/TSQL_Parser/Expressions/TSQLOperatorExpression.cs rename to TSQL_Parser/TSQL_Parser/Expressions/TSQLOperationExpression.cs index 40dc815..d19322b 100644 --- a/TSQL_Parser/TSQL_Parser/Expressions/TSQLOperatorExpression.cs +++ b/TSQL_Parser/TSQL_Parser/Expressions/TSQLOperationExpression.cs @@ -8,13 +8,13 @@ namespace TSQL.Expressions { - public class TSQLOperatorExpression : TSQLExpression + public class TSQLOperationExpression : TSQLExpression { public override TSQLExpressionType Type { get { - return TSQLExpressionType.Operator; + return TSQLExpressionType.Operation; } } diff --git a/TSQL_Parser/TSQL_Parser/TSQL_Parser.csproj b/TSQL_Parser/TSQL_Parser/TSQL_Parser.csproj index 4227243..be17de6 100644 --- a/TSQL_Parser/TSQL_Parser/TSQL_Parser.csproj +++ b/TSQL_Parser/TSQL_Parser/TSQL_Parser.csproj @@ -118,7 +118,7 @@ - + TSQLArgumentList.cs @@ -126,10 +126,11 @@ + - + @@ -140,6 +141,7 @@ + diff --git a/TSQL_Parser/Tests/Clauses/SelectClauseTests.cs b/TSQL_Parser/Tests/Clauses/SelectClauseTests.cs index 256622f..f99ec5e 100644 --- a/TSQL_Parser/Tests/Clauses/SelectClauseTests.cs +++ b/TSQL_Parser/Tests/Clauses/SelectClauseTests.cs @@ -59,18 +59,18 @@ public void SelectClause_Comments() Assert.AreEqual(1, select.Columns.Count); Assert.IsNull(select.Columns[0].ColumnAlias); - Assert.AreEqual(TSQLExpressionType.Operator, select.Columns[0].Expression.Type); + Assert.AreEqual(TSQLExpressionType.Operation, select.Columns[0].Expression.Type); - TSQLOperatorExpression operatorExpression = select.Columns[0].Expression.AsOperator; - Assert.AreEqual("/", operatorExpression.Operator.Text); - Assert.AreEqual(TSQLExpressionType.Column, operatorExpression.LeftSide.Type); + TSQLOperationExpression operationExpression = select.Columns[0].Expression.AsOperation; + Assert.AreEqual("/", operationExpression.Operator.Text); + Assert.AreEqual(TSQLExpressionType.Column, operationExpression.LeftSide.Type); - TSQLColumnExpression leftSide = operatorExpression.LeftSide.AsColumn; + TSQLColumnExpression leftSide = operationExpression.LeftSide.AsColumn; Assert.AreEqual("oh", leftSide.TableReference.Single().AsIdentifier.Name); Assert.AreEqual("TaxAmt", leftSide.Column.Name); - Assert.AreEqual(TSQLExpressionType.Column, operatorExpression.RightSide.Type); + Assert.AreEqual(TSQLExpressionType.Column, operationExpression.RightSide.Type); - TSQLColumnExpression rightSide = operatorExpression.RightSide.AsColumn; + TSQLColumnExpression rightSide = operationExpression.RightSide.AsColumn; Assert.AreEqual("oh", rightSide.TableReference.Single().AsIdentifier.Name); Assert.AreEqual("SubTotal", rightSide.Column.Name); Assert.AreEqual(" tax percent ", select.Columns.Last().Tokens.Last().AsMultilineComment.Comment); @@ -327,9 +327,9 @@ public void SelectClause_UnaryOperator() TSQLSelectColumn column = select.Columns[0]; Assert.IsNull(column.ColumnAlias); - Assert.AreEqual(TSQLExpressionType.Operator, column.Expression.Type); + Assert.AreEqual(TSQLExpressionType.Operation, column.Expression.Type); - TSQLOperatorExpression tsqlOperator = column.Expression.AsOperator; + TSQLOperationExpression tsqlOperator = column.Expression.AsOperation; Assert.AreEqual("+", tsqlOperator.Operator.Text); Assert.IsNull(tsqlOperator.LeftSide); diff --git a/TSQL_Parser/Tests/Expressions/OperatorExpressionTests.cs b/TSQL_Parser/Tests/Expressions/OperationExpressionTests.cs similarity index 89% rename from TSQL_Parser/Tests/Expressions/OperatorExpressionTests.cs rename to TSQL_Parser/Tests/Expressions/OperationExpressionTests.cs index 77020db..414cda9 100644 --- a/TSQL_Parser/Tests/Expressions/OperatorExpressionTests.cs +++ b/TSQL_Parser/Tests/Expressions/OperationExpressionTests.cs @@ -16,10 +16,10 @@ namespace Tests.Expressions { [TestFixture(Category = "Expression Parsing")] - public class OperatorExpressionTests + public class OperationExpressionTests { [Test] - public void OperatorExpression_Simple() + public void OperationExpression_Simple() { TSQLTokenizer tokenizer = new TSQLTokenizer( "+ 2 - 3") @@ -38,7 +38,7 @@ public void OperatorExpression_Simple() Assert.IsTrue(tokenizer.MoveNext()); - TSQLOperatorExpression op = new TSQLOperatorExpressionParser().Parse( + TSQLOperationExpression op = new TSQLOperationExpressionParser().Parse( tokenizer, leftSide); @@ -61,8 +61,8 @@ public void OperatorExpression_Simple() Assert.AreEqual("+", op.Operator.Text); - Assert.AreEqual(TSQLExpressionType.Operator, op.RightSide.Type); - TSQLOperatorExpression rightSide = op.RightSide.AsOperator; + Assert.AreEqual(TSQLExpressionType.Operation, op.RightSide.Type); + TSQLOperationExpression rightSide = op.RightSide.AsOperation; TokenComparisons.CompareTokenLists( new List() { diff --git a/TSQL_Parser/Tests/Statements/SelectStatementTests.cs b/TSQL_Parser/Tests/Statements/SelectStatementTests.cs index a50f681..923aed6 100644 --- a/TSQL_Parser/Tests/Statements/SelectStatementTests.cs +++ b/TSQL_Parser/Tests/Statements/SelectStatementTests.cs @@ -267,23 +267,23 @@ public void SelectStatement_MultiLevelParens() Assert.AreEqual(TSQLExpressionType.Grouped, lvl1Expression.Type); // contents of outer parens TSQLExpression lvl2Expression = lvl1Expression.AsGrouped.InnerExpression; - Assert.AreEqual(TSQLExpressionType.Operator, lvl2Expression.Type); - Assert.AreEqual("-", lvl2Expression.AsOperator.Operator.Text); + Assert.AreEqual(TSQLExpressionType.Operation, lvl2Expression.Type); + Assert.AreEqual("-", lvl2Expression.AsOperation.Operator.Text); // (A/B) - TSQLExpression lvl2aExpression = lvl2Expression.AsOperator.LeftSide; + TSQLExpression lvl2aExpression = lvl2Expression.AsOperation.LeftSide; // 1 - TSQLExpression lvl2bExpression = lvl2Expression.AsOperator.RightSide; + TSQLExpression lvl2bExpression = lvl2Expression.AsOperation.RightSide; Assert.AreEqual(TSQLExpressionType.Grouped, lvl2aExpression.Type); Assert.AreEqual(TSQLExpressionType.Constant, lvl2bExpression.Type); Assert.AreEqual(1, lvl2bExpression.AsConstant.Literal.AsNumericLiteral.Value); // A/B TSQLExpression lvl3Expression = lvl2aExpression.AsGrouped.InnerExpression; - Assert.AreEqual(TSQLExpressionType.Operator, lvl3Expression.Type); - Assert.AreEqual("/", lvl3Expression.AsOperator.Operator.Text); + Assert.AreEqual(TSQLExpressionType.Operation, lvl3Expression.Type); + Assert.AreEqual("/", lvl3Expression.AsOperation.Operator.Text); // A - TSQLExpression lvl3aExpression = lvl3Expression.AsOperator.LeftSide; + TSQLExpression lvl3aExpression = lvl3Expression.AsOperation.LeftSide; // B - TSQLExpression lvl3bExpression = lvl3Expression.AsOperator.RightSide; + TSQLExpression lvl3bExpression = lvl3Expression.AsOperation.RightSide; Assert.AreEqual(TSQLExpressionType.Column, lvl3aExpression.Type); Assert.AreEqual("A", lvl3aExpression.AsColumn.Column.Name); Assert.IsNull(lvl3aExpression.AsColumn.TableReference); @@ -844,5 +844,39 @@ INNER JOIN Sales.SalesOrderDetail AS sod Assert.AreEqual("NonDiscountSales", select.Select.Columns[1].ColumnAlias.Name); Assert.AreEqual("Discounts", select.Select.Columns[2].ColumnAlias.Name); } + + [Test] + public void SelectStatement_SELECT_DISTINCT() + { + // regression test for https://github.com/bruce-dunwiddie/tsql-parser/issues/101 + List statements = TSQLStatementReader.ParseStatements( + @"SELECT + COUNT(DISTINCT id) AssetId_changes + FROM [gs].[ESG_ResolvedGSID_Merged]", + includeWhitespace: false); + + Assert.AreEqual(1, statements.Count); + TSQLSelectStatement select = statements.Single().AsSelect; + Assert.AreEqual(11, select.Tokens.Count); + Assert.AreEqual(1, select.Select.Columns.Count); + Assert.AreEqual("AssetId_changes", select.Select.Columns[0].ColumnAlias.Name); + Assert.AreEqual("id", select.Select.Columns[0].Expression.AsDuplicateSpecification.InnerExpression.AsColumn.Column.Name) + } + + [Test] + public void SelectStatement_SELECT_NULL() + { + // regression test for https://github.com/bruce-dunwiddie/tsql-parser/issues/104 + List statements = TSQLStatementReader.ParseStatements( + @"SELECT 1, 2, 3, NULL, 5 FROM MyTable", + includeWhitespace: false); + + Assert.AreEqual(1, statements.Count); + TSQLSelectStatement select = statements.Single().AsSelect; + Assert.AreEqual(12, select.Tokens.Count); + Assert.AreEqual(5, select.Select.Columns.Count); + Assert.AreEqual(TSQLExpressionType.Null, select.Select.Columns[3].Expression.Type); + Assert.AreEqual(5, select.Select.Columns[4].Expression.AsConstant.Literal.AsNumericLiteral.Value); + } } } diff --git a/TSQL_Parser/Tests/Tests.csproj b/TSQL_Parser/Tests/Tests.csproj index fba23c3..6a59a9a 100644 --- a/TSQL_Parser/Tests/Tests.csproj +++ b/TSQL_Parser/Tests/Tests.csproj @@ -72,7 +72,7 @@ - +